R-2.14.6

simplified oneall dname handling

CHANGELOG

@@ -1,3 +1,11 @@
+## 2.14.6
+
+- Increased test coverage (thanks Richard)
+- Fixed some newly discovered security issues in admin:
+  CSRF vulnerability in admin that allows disabling apps
+  Brute force password attack vulnerability in admin
+  (thanks Narendra and Leonel)  
+
 ## 2.14.1-5
 
 - fixed two major security issues that caused the examples app to leak information

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.5-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

VERSION

@@ -1 +1 @@
-Version 2.14.5-stable+timestamp.2016.04.13.22.22.13
+Version 2.14.6-stable+timestamp.2016.05.09.19.18.48

applications/admin/controllers/default.py

@@ -32,15 +32,15 @@ from gluon.languages import (read_possible_languages, read_dict, write_dict,
 
 
 if DEMO_MODE and request.function in ['change_password', 'pack',
-'pack_custom','pack_plugin', 'upgrade_web2py', 'uninstall',
-'cleanup', 'compile_app', 'remove_compiled_app', 'delete',
-'delete_plugin', 'create_file', 'upload_file', 'update_languages',
-'reload_routes', 'git_push', 'git_pull', 'install_plugin']:
+                                      'pack_custom', 'pack_plugin', 'upgrade_web2py', 'uninstall',
+                                      'cleanup', 'compile_app', 'remove_compiled_app', 'delete',
+                                      'delete_plugin', 'create_file', 'upload_file', 'update_languages',
+                                      'reload_routes', 'git_push', 'git_pull', 'install_plugin']:
     session.flash = T('disabled in demo mode')
     redirect(URL('site'))
 
 if is_gae and request.function in ('edit', 'edit_language',
-'edit_plurals', 'update_languages', 'create_file', 'install_plugin'):
+                                   'edit_plurals', 'update_languages', 'create_file', 'install_plugin'):
     session.flash = T('disabled in GAE mode')
     redirect(URL('site'))
 
@@ -74,8 +74,10 @@ def log_progress(app, mode='EDIT', filename=None, progress=0):
 def safe_open(a, b):
     if (DEMO_MODE or is_gae) and ('w' in b or 'a' in b):
         class tmp:
+
             def write(self, data):
                 pass
+
             def close(self):
                 pass
         return tmp()
@@ -119,6 +121,9 @@ def index():
         send = URL('site')
     if session.authorized:
         redirect(send)
+    elif failed_login_count() >= allowed_number_of_attempts:
+        time.sleep(2 ** allowed_number_of_attempts)
+        raise HTTP(403)
     elif request.vars.password:
         if verify_password(request.vars.password[:1024]):
             session.authorized = True
@@ -208,6 +213,7 @@ def site():
     file_or_appurl = 'file' in request.vars or 'appurl' in request.vars
 
     class IS_VALID_APPNAME(object):
+
         def __call__(self, value):
             if not re.compile('^\w+$').match(value):
                 return (value, T('Invalid application name'))
@@ -325,7 +331,7 @@ def report_progress(app):
         if not m:
             continue
         days = -(request.now - datetime.datetime.strptime(m[0],
-                 '%Y-%m-%d %H:%M:%S')).days
+                                                          '%Y-%m-%d %H:%M:%S')).days
         counter += int(m[1])
         events.append([days, counter])
     return events
@@ -353,6 +359,7 @@ def pack():
         session.flash = T('internal error: %s', e)
         redirect(URL('site'))
 
+
 def pack_plugin():
     app = get_app()
     if len(request.args) == 2:
@@ -368,7 +375,6 @@ def pack_plugin():
         redirect(URL('plugin', args=request.args))
 
 
-
 def pack_exe(app, base, filenames=None):
     import urllib
     import zipfile
@@ -397,10 +403,20 @@ def pack_exe(app, base, filenames=None):
 def pack_custom():
     app = get_app()
     base = apath(app, r=request)
+
+    def ignore(fs):
+        return [f for f in fs if not (
+                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
+    files = {}
+    for (r, d, f) in os.walk(base):
+        files[r] = {'folders': ignore(d), 'files': ignore(f)}
+
     if request.post_vars.file:
-        
+        valid_set = set(os.path.relpath(os.path.join(r, f), base) for r in files for f in files[r]['files'])
         files = request.post_vars.file
-        files = [files] if not isinstance(files,list) else files
+        files = [files] if not isinstance(files, list) else files
+        files = [file for file in files if file in valid_set]
+
         if request.post_vars.doexe is None:
             fname = 'web2py.app.%s.w2p' % app
             try:
@@ -417,12 +433,7 @@ def pack_custom():
                 redirect(URL(args=request.args))
         else:
             return pack_exe(app, base, files)
-    def ignore(fs):
-        return [f for f in fs if not (
-                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
-    files = {}
-    for (r,d,f) in os.walk(base):
-        files[r] = {'folders':ignore(d),'files':ignore(f)}
+
     return locals()
 
 
@@ -485,14 +496,14 @@ def cleanup():
 def compile_app():
     app = get_app()
     c = app_compile(app, request,
-        skip_failed_views = (request.args(1) == 'skip_failed_views'))
+                    skip_failed_views=(request.args(1) == 'skip_failed_views'))
     if not c:
         session.flash = T('application compiled')
     elif isinstance(c, list):
         session.flash = DIV(*[T('application compiled'), BR(), BR(),
                               T('WARNING: The following views could not be compiled:'), BR()] +
-                             [CAT(BR(), view) for view in c] +
-                             [BR(), BR(), T('DO NOT use the "Pack compiled" feature.')])
+                            [CAT(BR(), view) for view in c] +
+                            [BR(), BR(), T('DO NOT use the "Pack compiled" feature.')])
     else:
         session.flash = DIV(T('Cannot compile: there are errors in your app:'),
                             CODE(c))
@@ -533,8 +544,8 @@ def delete():
         redirect(URL(sender, anchor=request.vars.id2))
     return dict(dialog=dialog, filename=filename)
 
-
 def enable():
+    if not URL.verify(request, hmac_key=session.hmac_key): raise HTTP(401)
     app = get_app()
     filename = os.path.join(apath(app, r=request), 'DISABLED')
     if is_gae:
@@ -546,6 +557,7 @@ def enable():
         safe_open(filename, 'wb').write('disabled: True\ntime-disabled: %s' % request.now)
         return SPAN(T('Enable'), _style='color:red')
 
+
 def peek():
     """ Visualize object code """
     app = get_app(request.vars.app)
@@ -609,7 +621,7 @@ def edit():
     # Load json only if it is ajax edited...
     app = get_app(request.vars.app)
     app_path = apath(app, r=request)
-    preferences={'theme':'web2py', 'editor': 'default', 'closetag': 'true', 'codefolding': 'false', 'tabwidth':'4', 'indentwithtabs':'false', 'linenumbers':'true', 'highlightline':'true'}
+    preferences = {'theme': 'web2py', 'editor': 'default', 'closetag': 'true', 'codefolding': 'false', 'tabwidth': '4', 'indentwithtabs': 'false', 'linenumbers': 'true', 'highlightline': 'true'}
     config = Config(os.path.join(request.folder, 'settings.cfg'),
                     section='editor', default_values={})
     preferences.update(config.read())
@@ -617,14 +629,14 @@ def edit():
     if not(request.ajax) and not(is_mobile):
         # return the scaffolding, the rest will be through ajax requests
         response.title = T('Editing %s') % app
-        return response.render ('default/edit.html', dict(app=app, editor_settings=preferences))
+        return response.render('default/edit.html', dict(app=app, editor_settings=preferences))
 
     # show settings tab and save prefernces
     if 'settings' in request.vars:
-        if request.post_vars:        #save new preferences
+        if request.post_vars:  # save new preferences
             post_vars = request.post_vars.items()
             # Since unchecked checkbox are not serialized, we must set them as false by hand to store the correct preference in the settings
-            post_vars+= [(opt, 'false') for opt in preferences if opt not in request.post_vars ]
+            post_vars += [(opt, 'false') for opt in preferences if opt not in request.post_vars]
             if config.save(post_vars):
                 response.headers["web2py-component-flash"] = T('Preferences saved correctly')
             else:
@@ -632,8 +644,8 @@ def edit():
             response.headers["web2py-component-command"] = "update_editor(%s);$('a[href=#editor_settings] button.close').click();" % response.json(config.read())
             return
         else:
-            details = {'realfilename':'settings', 'filename':'settings', 'id':'editor_settings', 'force': False}
-            details['plain_html'] = response.render('default/editor_settings.html', {'editor_settings':preferences})
+            details = {'realfilename': 'settings', 'filename': 'settings', 'id': 'editor_settings', 'force': False}
+            details['plain_html'] = response.render('default/editor_settings.html', {'editor_settings': preferences})
             return response.json(details)
 
     """ File edit handler """
@@ -764,8 +776,8 @@ def edit():
             view = request.args[3].replace('.html', '')
             view_link = URL(request.args[0], request.args[2], view)
     elif filetype == 'python' and request.args[1] == 'controllers':
-        ## it's a controller file.
-        ## Create links to all of the associated view files.
+        # it's a controller file.
+        # Create links to all of the associated view files.
         app = get_app()
         viewname = os.path.splitext(request.args[2])[0]
         viewpath = os.path.join(app, 'views', viewname)
@@ -796,22 +808,22 @@ def edit():
         return response.json({'file_hash': file_hash, 'saved_on': saved_on, 'functions': functions, 'controller': controller, 'application': request.args[0], 'highlight': highlight})
     else:
         file_details = dict(app=request.args[0],
-                    lineno=request.vars.lineno or 1,
-                    editor_settings=preferences,
-                    filename=filename,
-                    realfilename=realfilename,
-                    filetype=filetype,
-                    data=data,
-                    edit_controller=edit_controller,
-                    file_hash=file_hash,
-                    saved_on=saved_on,
-                    controller=controller,
-                    functions=functions,
-                    view_link=view_link,
-                    editviewlinks=editviewlinks,
-                    id=IS_SLUG()(filename)[0],
-                    force= True if (request.vars.restore or
-                                    request.vars.revert) else False)
+                            lineno=request.vars.lineno or 1,
+                            editor_settings=preferences,
+                            filename=filename,
+                            realfilename=realfilename,
+                            filetype=filetype,
+                            data=data,
+                            edit_controller=edit_controller,
+                            file_hash=file_hash,
+                            saved_on=saved_on,
+                            controller=controller,
+                            functions=functions,
+                            view_link=view_link,
+                            editviewlinks=editviewlinks,
+                            id=IS_SLUG()(filename)[0],
+                            force=True if (request.vars.restore or
+                                           request.vars.revert) else False)
         plain_html = response.render('default/edit_js.html', file_details)
         file_details['plain_html'] = plain_html
         if is_mobile:
@@ -820,14 +832,16 @@ def edit():
         else:
             return response.json(file_details)
 
+
 def todolist():
     """ Returns all TODO of the requested app
     """
     app = request.vars.app or ''
-    app_path = apath('%(app)s' % {'app':app}, r=request)
-    dirs=['models', 'controllers', 'modules', 'private' ]
+    app_path = apath('%(app)s' % {'app': app}, r=request)
+    dirs = ['models', 'controllers', 'modules', 'private']
+
     def listfiles(app, dir, regexp='.*\.py$'):
-        files = sorted( listdir(apath('%(app)s/%(dir)s/' % {'app':app, 'dir':dir}, r=request), regexp))
+        files = sorted(listdir(apath('%(app)s/%(dir)s/' % {'app': app, 'dir': dir}, r=request), regexp))
         files = [x.replace(os.path.sep, '/') for x in files if not x.endswith('.bak')]
         return files
 
@@ -838,17 +852,18 @@ def todolist():
     for d in dirs:
         for f in listfiles(app, d):
             matches = []
-            filename= apath(os.path.join(app, d, f), r=request)
+            filename = apath(os.path.join(app, d, f), r=request)
             with open(filename, 'r') as f_s:
                 src = f_s.read()
                 for m in regex.finditer(src):
                     start = m.start()
                     lineno = src.count('\n', 0, start) + 1
-                    matches.append({'text':m.group(0), 'lineno':lineno})
+                    matches.append({'text': m.group(0), 'lineno': lineno})
             if len(matches) != 0:
-                output.append({'filename':f,'matches':matches, 'dir':d})
+                output.append({'filename': f, 'matches': matches, 'dir': d})
+
+    return {'todo': output, 'app': app}
 
-    return {'todo':output, 'app': app}
 
 def editor_sessions():
     config = Config(os.path.join(request.folder, 'settings.cfg'),
@@ -858,13 +873,14 @@ def editor_sessions():
     if request.vars.session_name and request.vars.files:
         session_name = request.vars.session_name
         files = request.vars.files
-        preferences.update({session_name:','.join(files)})
+        preferences.update({session_name: ','.join(files)})
         if config.save(preferences.items()):
             response.headers["web2py-component-flash"] = T('Session saved correctly')
         else:
             response.headers["web2py-component-flash"] = T('Session saved on session only')
 
-    return response.render('default/editor_sessions.html', {'editor_sessions':preferences})
+    return response.render('default/editor_sessions.html', {'editor_sessions': preferences})
+
 
 def resolve():
     """
@@ -901,8 +917,8 @@ def resolve():
 
     def getclass(item):
         """ Determine item class """
-        operators = {' ':'normal', '+':'plus', '-':'minus'}
-        
+        operators = {' ': 'normal', '+': 'plus', '-': 'minus'}
+
         return operators[item[0]]
 
     if request.vars:
@@ -921,7 +937,7 @@ def resolve():
         diff = TABLE(*[TR(TD(gen_data(i, item)),
                           TD(item[0]),
                           TD(leading(item[2:]),
-                          TT(item[2:].rstrip())),
+                             TT(item[2:].rstrip())),
                           _class=getclass(item))
                        for (i, item) in enumerate(d) if item[0] != '?'])
 
@@ -968,11 +984,11 @@ def edit_language():
 
         new_row = DIV(LABEL(prefix, k, _style="font-weight:normal;"),
                       CAT(elem, '\n', TAG.BUTTON(
-                    T('delete'),
-                    _onclick='return delkey("%s")' % name,
-                    _class='btn')), _id=name, _class='span6 well well-small')
+                          T('delete'),
+                          _onclick='return delkey("%s")' % name,
+                          _class='btn')), _id=name, _class='span6 well well-small')
 
-        rows.append(DIV(new_row,_class="row-fluid"))
+        rows.append(DIV(new_row, _class="row-fluid"))
     rows.append(DIV(INPUT(_type='submit', _value=T('update'), _class="btn btn-primary"), _class='controls'))
     form = FORM(*rows)
     if form.accepts(request.vars, keepvalues=True):
@@ -1128,18 +1144,18 @@ def design():
 
     # Get all static files
     statics = listdir(apath('%s/static/' % app, r=request), '[^\.#].*',
-                      maxnum = MAXNFILES)
+                      maxnum=MAXNFILES)
     statics = [x.replace(os.path.sep, '/') for x in statics]
     statics.sort()
 
     # Get all languages
-    langpath = os.path.join(apath(app, r=request),'languages')
+    langpath = os.path.join(apath(app, r=request), 'languages')
     languages = dict([(lang, info) for lang, info
                       in read_possible_languages(langpath).iteritems()
                       if info[2] != 0])  # info[2] is langfile_mtime:
-                                         # get only existed files
+    # get only existed files
 
-    #Get crontab
+    # Get crontab
     cronfolder = apath('%s/cron' % app, r=request)
     crontab = apath('%s/cron/crontab' % app, r=request)
     if not is_gae:
@@ -1265,7 +1281,7 @@ def plugin():
 
     # Get all static files
     statics = listdir(apath('%s/static/' % app, r=request), '[^\.#].*',
-                      maxnum = MAXNFILES)
+                      maxnum=MAXNFILES)
     statics = [x.replace(os.path.sep, '/') for x in statics]
     statics.sort()
 
@@ -1273,9 +1289,9 @@ def plugin():
     languages = sorted([lang + '.py' for lang, info in
                         T.get_possible_languages_info().iteritems()
                         if info[2] != 0])  # info[2] is langfile_mtime:
-                                    # get only existed files
+    # get only existed files
 
-    #Get crontab
+    # Get crontab
     crontab = apath('%s/cron/crontab' % app, r=request)
     if not os.path.exists(crontab):
         safe_write(crontab, '#crontab')
@@ -1298,6 +1314,7 @@ def plugin():
                 languages=languages,
                 crontab=crontab)
 
+
 def create_file():
     """ Create files handler """
     if request.vars and not request.vars.token == session.token:
@@ -1309,7 +1326,7 @@ def create_file():
             path = abspath(request.vars.location)
         else:
             if request.vars.dir:
-                    request.vars.location += request.vars.dir + '/'
+                request.vars.location += request.vars.dir + '/'
             app = get_app(name=request.vars.location.split('/')[0])
             path = apath(request.vars.location, r=request)
         filename = re.sub('[^\w./-]+', '_', request.vars.filename)
@@ -1419,7 +1436,7 @@ def create_file():
 
         elif (path[-8:] == '/static/') or (path[-9:] == '/private/'):
             if (request.vars.plugin and
-                not filename.startswith('plugin_%s/' % request.vars.plugin)):
+                    not filename.startswith('plugin_%s/' % request.vars.plugin)):
                 filename = 'plugin_%s/%s' % (request.vars.plugin, filename)
             text = ''
 
@@ -1439,17 +1456,17 @@ def create_file():
         log_progress(app, 'CREATE', filename)
         if request.vars.dir:
             result = T('file "%(filename)s" created',
-                          dict(filename=full_filename[len(path):]))
+                       dict(filename=full_filename[len(path):]))
         else:
             session.flash = T('file "%(filename)s" created',
-                          dict(filename=full_filename[len(path):]))
+                              dict(filename=full_filename[len(path):]))
         vars = {}
         if request.vars.id:
             vars['id'] = request.vars.id
         if request.vars.app:
             vars['app'] = request.vars.app
         redirect(URL('edit',
-                 args=[os.path.join(request.vars.location, filename)], vars=vars))
+                     args=[os.path.join(request.vars.location, filename)], vars=vars))
 
     except Exception, e:
         if not isinstance(e, HTTP):
@@ -1460,7 +1477,7 @@ def create_file():
         response.headers['web2py-component-content'] = 'append'
         response.headers['web2py-component-command'] = "%s %s %s" % (
             "$.web2py.invalidate('#files_menu');",
-            "load_file('%s');" % URL('edit', args=[app,request.vars.dir,filename]),
+            "load_file('%s');" % URL('edit', args=[app, request.vars.dir, filename]),
             "$.web2py.enableElement($('#form form').find($.web2py.formInputClickSelector));")
         return ''
     else:
@@ -1468,32 +1485,35 @@ def create_file():
 
 
 def listfiles(app, dir, regexp='.*\.py$'):
-        files = sorted(
-         listdir(apath('%(app)s/%(dir)s/' % {'app':app, 'dir':dir}, r=request), regexp))
-        files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')]
-        return files
+    files = sorted(
+        listdir(apath('%(app)s/%(dir)s/' % {'app': app, 'dir': dir}, r=request), regexp))
+    files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')]
+    return files
+
+
+def editfile(path, file, vars={}, app=None):
+    args = (path, file) if 'app' in vars else (app, path, file)
+    url = URL('edit', args=args, vars=vars)
+    return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;')
 
-def editfile(path,file,vars={}, app = None):
-        args=(path,file) if 'app' in vars else (app,path,file)
-        url = URL('edit', args=args, vars=vars)
-        return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;')
 
 def files_menu():
-        app = request.vars.app or 'welcome'
-        dirs=[{'name':'models', 'reg':'.*\.py$'},
-              {'name':'controllers', 'reg':'.*\.py$'},
-              {'name':'views', 'reg':'[\w/\-]+(\.\w+)+$'},
-              {'name':'modules', 'reg':'.*\.py$'},
-              {'name':'static', 'reg': '[^\.#].*'},
-              {'name':'private', 'reg':'.*\.py$'}]
-        result_files = []
-        for dir in dirs:
-                result_files.append(TAG[''](LI(dir['name'], _class="nav-header component", _onclick="collapse('" + dir['name'] + "_files');"),
-                                      LI(UL(*[LI(editfile(dir['name'], f, dict(id=dir['name'] + f.replace('.','__')), app), _style="overflow:hidden", _id=dir['name']+"__"+f.replace('.','__'))
-                                                      for f in listfiles(app, dir['name'], regexp=dir['reg'])],
-                                                      _class="nav nav-list small-font"),
-                                               _id=dir['name'] + '_files', _style="display: none;")))
-        return dict(result_files = result_files)
+    app = request.vars.app or 'welcome'
+    dirs = [{'name': 'models', 'reg': '.*\.py$'},
+            {'name': 'controllers', 'reg': '.*\.py$'},
+            {'name': 'views', 'reg': '[\w/\-]+(\.\w+)+$'},
+            {'name': 'modules', 'reg': '.*\.py$'},
+            {'name': 'static', 'reg': '[^\.#].*'},
+            {'name': 'private', 'reg': '.*\.py$'}]
+    result_files = []
+    for dir in dirs:
+        result_files.append(TAG[''](LI(dir['name'], _class="nav-header component", _onclick="collapse('" + dir['name'] + "_files');"),
+                                    LI(UL(*[LI(editfile(dir['name'], f, dict(id=dir['name'] + f.replace('.', '__')), app), _style="overflow:hidden", _id=dir['name'] + "__" + f.replace('.', '__'))
+                                            for f in listfiles(app, dir['name'], regexp=dir['reg'])],
+                                          _class="nav nav-list small-font"),
+                                       _id=dir['name'] + '_files', _style="display: none;")))
+    return dict(result_files=result_files)
+
 
 def upload_file():
     """ File uploading handler """
@@ -1556,7 +1576,7 @@ def errors():
     app = get_app()
     if is_gae:
         method = 'dbold' if ('old' in
-                     (request.args(1) or '')) else 'dbnew'
+                             (request.args(1) or '')) else 'dbnew'
     else:
         method = request.args(1) or 'new'
     db_ready = {}
@@ -1599,7 +1619,7 @@ def errors():
                     hash2error[hash]['count'] += 1
                 except KeyError:
                     error_lines = error['traceback'].split("\n")
-                    last_line = error_lines[-2] if len(error_lines)>1 else 'unknown'
+                    last_line = error_lines[-2] if len(error_lines) > 1 else 'unknown'
                     error_causer = os.path.split(error['layer'])[1]
                     hash2error[hash] = dict(count=1, pickel=error,
                                             causer=error_causer,
@@ -1638,9 +1658,9 @@ def errors():
                         last_line = error_lines[-2]
                         error_causer = os.path.split(error['layer'])[1]
                         hash2error[hash] = dict(count=1,
-                            pickel=error, causer=error_causer,
-                            last_line=last_line, hash=hash,
-                            ticket=fn.ticket_id)
+                                                pickel=error, causer=error_causer,
+                                                last_line=last_line, hash=hash,
+                                                ticket=fn.ticket_id)
             except AttributeError, e:
                 tk_db(tk_table.id == fn.id).delete()
                 tk_db.commit()
@@ -1657,11 +1677,11 @@ def errors():
                 tk_db(tk_table.ticket_id == item[7:]).delete()
                 tk_db.commit()
         tickets_ = tk_db(tk_table.id > 0).select(tk_table.ticket_id,
-            tk_table.created_datetime,
-            orderby=~tk_table.created_datetime)
+                                                 tk_table.created_datetime,
+                                                 orderby=~tk_table.created_datetime)
         tickets = [row.ticket_id for row in tickets_]
         times = dict([(row.ticket_id, row.created_datetime) for
-            row in tickets_])
+                      row in tickets_])
         return dict(app=app, tickets=tickets, method=method,
                     times=times, db_ready=db_ready)
 
@@ -1721,7 +1741,7 @@ def make_link(path):
             if ext.lower() == editable[key] and check_extension:
                 return A('"' + tryFile + '"',
                          _href=URL(r=request,
-                         f='edit/%s/%s/%s' % (app, key, filename))).xml()
+                                   f='edit/%s/%s/%s' % (app, key, filename))).xml()
     return ''
 
 
@@ -1867,7 +1887,7 @@ def bulk_register():
         redirect(URL('site'))
     return locals()
 
-### Begin experimental stuff need fixes:
+# Begin experimental stuff need fixes:
 # 1) should run in its own process - cannot os.chdir
 # 2) should not prompt user at console
 # 3) should give option to force commit and not reuqire manual merge
@@ -1934,6 +1954,7 @@ def git_push():
             redirect(URL('site'))
     return dict(app=app, form=form)
 
+
 def plugins():
     app = request.args(0)
     from serializers import loads_json
@@ -1948,12 +1969,16 @@ def plugins():
             session.plugins = []
     return dict(plugins=session.plugins["results"], app=request.args(0))
 
+
 def install_plugin():
     app = request.args(0)
     source = request.vars.source
     plugin = request.vars.plugin
     if not (source and app):
         raise HTTP(500, T("Invalid request"))
+    # make sure no XSS attacks in source
+    if not source.lower().split('://')[0] in ('http','https'):
+        raise HTTP(500, T("Invalid request"))
     form = SQLFORM.factory()
     result = None
     if form.process().accepted:
@@ -1969,5 +1994,5 @@ def install_plugin():
         else:
             session.flash = \
                 T('unable to install plugin "%s"', filename)
-        redirect(URL(f="plugins", args=[app,]))
+        redirect(URL(f="plugins", args=[app, ]))
     return dict(form=form, app=app, plugin=plugin, source=source)

applications/admin/models/access.py

@@ -4,6 +4,7 @@ import time
 from gluon import portalocker
 from gluon.admin import apath
 from gluon.fileutils import read_file
+from gluon.utils import web2py_uuid
 # ###########################################################
 # ## make sure administrator is on localhost or https
 # ###########################################################
@@ -49,15 +50,18 @@ except IOError:
 def verify_password(password):
     session.pam_user = None
     if DEMO_MODE:
-        return True
+        ret = True
     elif not _config.get('password'):
-        return False
+        ret - False
     elif _config['password'].startswith('pam_user:'):
         session.pam_user = _config['password'][9:].strip()
         import gluon.contrib.pam
-        return gluon.contrib.pam.authenticate(session.pam_user, password)
+        ret = gluon.contrib.pam.authenticate(session.pam_user, password)
     else:
-        return _config['password'] == CRYPT()(password)[0]
+        ret = _config['password'] == CRYPT()(password)[0]
+    if ret:
+        session.hmac_key = web2py_uuid()
+    return ret
 
 
 # ###########################################################
@@ -100,13 +104,12 @@ def write_hosts_deny(denied_hosts):
     portalocker.unlock(f)
     f.close()
 
-
 def login_record(success=True):
     denied_hosts = read_hosts_deny()
     val = (0, 0)
     if success and request.client in denied_hosts:
         del denied_hosts[request.client]
-    elif not success and not request.is_local:
+    elif not success:
         val = denied_hosts.get(request.client, (0, 0))
         if time.time() - val[1] < expiration_failed_logins \
             and val[0] >= allowed_number_of_attempts:
@@ -117,6 +120,11 @@ def login_record(success=True):
     write_hosts_deny(denied_hosts)
     return val[0]
 
+def failed_login_count():
+    denied_hosts = read_hosts_deny()
+    val = denied_hosts.get(request.client, (0, 0))
+    return val[0]
+
 
 # ###########################################################
 # ## session expiration

applications/admin/views/default/site.html

@@ -56,7 +56,7 @@
         {{pass}}
           </ul>
         </div>
-        {{=button_enable(URL('enable',args=a), a) if a!='admin' else ''}}
+        {{=button_enable(URL('enable',args=a, hmac_key=session.hmac_key), a) if a!='admin' else ''}}
       </td>
     </tr>
     {{pass}}

applications/examples/static/css/stupid.css

@@ -68,7 +68,7 @@ header, main, footer {display:block; with:100%} /* IE fix */
 input:not([type]), input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]), [type=file]:before {outline:none; padding:0.5em 1em; margin:0.5px; border-bottom:1px solid #ddd; width:100%}
 textarea {width:100%; border:1px solid #ddd; padding:4px 8px; outline:none; outline:none}
 select {-webkit-appearance:none; outline:none; padding:0.5em 1em; border-radius:0; margin:0.5px; border-bottom:1px solid #ddd; width:100%;background-color:transparent}
-input, textarea, select, button {font-size:12px}
+input, textarea, select, button, .btn {font-size:12px}
 input:not([type]):hover, input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]):hover, select:hover, textarea:hover {background-color:#fbf6d9; transition:background-color 1s ease}
 input:invalid, input.error {background:#cc1f00;color:white}
 

applications/examples/views/default/who.html

@@ -100,6 +100,7 @@
 </li><li>Michael Willis (shell)
 </li><li>Michele Comitini (facebook)
 </li><li>Michael Toomim (scheduler)
+</li><li>Narendra Bhati (security)
 </li><li>Nathan Freeze (admin design, IS_STRONG, DAL features, <a href="http://web2pyslices.com">web2pyslices.com</a>)
 </li><li>Niall Sweeny (MSSQL support)
 </li><li>Niccolo Polo (epydoc)

fabfile.py

@@ -115,8 +115,9 @@ def deploy(appname=None, all=False):
     """fab -H username@host deploy:appname,all"""
     appname = appname or os.path.split(os.getcwd())[-1]
     appfolder = applications+'/'+appname
-    if os.path.exists('_update.zip'):
-        os.unlink('_update.zip')
+    zipfile = os.path.join(appfolder, '_update.zip')
+    if os.path.exists(zipfile):
+        os.unlink(zipfile)
 
     backup = mkdir_or_backup(appname)
             

gluon/compileapp.py

@@ -676,8 +676,8 @@ def run_view_in(environment):
     else:
         filename = pjoin(folder, 'views', view)
         if os.path.exists(path): # compiled views
-            x = view.replace('/', '.')
-            files = ['views.%s.pyc' % x]
+            x = view.replace('/', '_')
+            files = ['views_%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(path, files[0]))
             # Don't use a generic view if the non-compiled view exists.
             if is_compiled or (not is_compiled and not os.path.exists(filename)):

gluon/contrib/login_methods/oneall_account.py

@@ -51,7 +51,7 @@ class OneallAccount(object):
             reg_id=profile.get('identity_token','')
             username=profile.get('preferredUsername',email)
             first_name=name.get('givenName', dname.split(' ')[0])
-            last_name=profile.get('familyName', dname.split(' ')[1] if(len(dname.split(' ')) > 1) else None)
+            last_name=profile.get('familyName', dname.split(' ')[1] if(dname.count(' ') > 0) else None)
             return dict(registration_id=reg_id,username=username,email=email,
                         first_name=first_name,last_name=last_name)
         self.mappings.default = defaultmapping

gluon/contrib/markdown/markdown2.py

@@ -53,8 +53,9 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * header-ids: Adds "id" attributes to headers. The id value is a slug of
   the header text.
 * html-classes: Takes a dict mapping html tag names (lowercase) to a
-  string to use for a "class" tag attribute. Currently only supports
-  "pre" and "code" tags. Add an issue if you require this for other tags.
+  string to use for a "class" tag attribute. Currently only supports "img",
+  "table", "pre" and "code" tags. Add an issue if you require this for other
+  tags.
 * markdown-in-html: Allow the use of `markdown="1"` in a block HTML tag to
   have markdown processing be done on its contents. Similar to
   <http://michelf.com/projects/php-markdown/extra/#markdown-attr> but with
@@ -70,9 +71,14 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * smarty-pants: Replaces ' and " with curly quotation marks or curly
   apostrophes.  Replaces --, ---, ..., and . . . with en dashes, em dashes,
   and ellipses.
+* spoiler: A special kind of blockquote commonly hidden behind a
+  click on SO. Syntax per <http://meta.stackexchange.com/a/72878>.
 * toc: The returned HTML string gets a new "toc_html" attribute which is
   a Table of Contents for the document. (experimental)
 * xml: Passes one-liner processing instructions and namespaced XML tags.
+* tables: Tables using the same format as GFM
+  <https://help.github.com/articles/github-flavored-markdown#tables> and
+  PHP-Markdown Extra <https://michelf.ca/projects/php-markdown/extra/#table>.
 * wiki-tables: Google Code Wiki-style tables. See
   <http://code.google.com/p/support/wiki/WikiSyntax#Tables>.
 """
@@ -82,13 +88,11 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 #   not yet sure if there implications with this. Compare 'pydoc sre'
 #   and 'perldoc perlre'.
 
-__version_info__ = (2, 2, 4)
+__version_info__ = (2, 3, 1)
 __version__ = '.'.join(map(str, __version_info__))
 __author__ = "Trent Mick"
 
-import os
 import sys
-from pprint import pprint
 import re
 import logging
 try:
@@ -102,13 +106,7 @@ import codecs
 
 #---- Python version compat
 
-try:
-    from urllib.parse import quote # python3
-except ImportError:
-    from urllib import quote # python2
-
 if sys.version_info[:2] < (2,4):
-    from sets import Set as set
     def reversed(sequence):
         for i in sequence[::-1]:
             yield i
@@ -804,6 +802,8 @@ class Markdown(object):
             text = self._prepare_pyshell_blocks(text)
         if "wiki-tables" in self.extras:
             text = self._do_wiki_tables(text)
+        if "tables" in self.extras:
+            text = self._do_tables(text)
 
         text = self._do_code_blocks(text)
 
@@ -844,6 +844,79 @@ class Markdown(object):
 
         return _pyshell_block_re.sub(self._pyshell_block_sub, text)
 
+    def _table_sub(self, match):
+        trim_space_re = '^[ \t\n]+|[ \t\n]+$'
+        trim_bar_re = '^\||\|$'
+
+        head, underline, body = match.groups()
+
+        # Determine aligns for columns.
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", underline)).split('|')]
+        align_from_col_idx = {}
+        for col_idx, col in enumerate(cols):
+            if col[0] == ':' and col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="center"'
+            elif col[0] == ':':
+                align_from_col_idx[col_idx] = ' align="left"'
+            elif col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="right"'
+
+        # thead
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<thead>', '<tr>']
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", head)).split('|')]
+        for col_idx, col in enumerate(cols):
+            hlines.append('  <th%s>%s</th>' % (
+                align_from_col_idx.get(col_idx, ''),
+                self._run_span_gamut(col)
+            ))
+        hlines.append('</tr>')
+        hlines.append('</thead>')
+
+        # tbody
+        hlines.append('<tbody>')
+        for line in body.strip('\n').split('\n'):
+            hlines.append('<tr>')
+            cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", line)).split('|')]
+            for col_idx, col in enumerate(cols):
+                hlines.append('  <td%s>%s</td>' % (
+                    align_from_col_idx.get(col_idx, ''),
+                    self._run_span_gamut(col)
+                ))
+            hlines.append('</tr>')
+        hlines.append('</tbody>')
+        hlines.append('</table>')
+
+        return '\n'.join(hlines) + '\n'
+
+    def _do_tables(self, text):
+        """Copying PHP-Markdown and GFM table syntax. Some regex borrowed from
+        https://github.com/michelf/php-markdown/blob/lib/Michelf/Markdown.php#L2538
+        """
+        less_than_tab = self.tab_width - 1
+        table_re = re.compile(r'''
+                (?:(?<=\n\n)|\A\n?)             # leading blank line
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (.*[|].*)  \n                   # $1: header row (at least one pipe)
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (                               # $2: underline row
+                    # underline row with leading bar
+                    (?:  \|\ *:?-+:?\ *  )+  \|?  \n
+                    |
+                    # or, underline row without leading bar
+                    (?:  \ *:?-+:?\ *\|  )+  (?:  \ *:?-+:?\ *  )?  \n
+                )
+
+                (                               # $3: data rows
+                    (?:
+                        ^[ ]{0,%d}(?!\ )         # ensure line begins with 0 to less_than_tab spaces
+                        .*\|.*  \n
+                    )+
+                )
+            ''' % (less_than_tab, less_than_tab, less_than_tab), re.M | re.X)
+        return table_re.sub(self._table_sub, text)
+
     def _wiki_table_sub(self, match):
         ttext = match.group(0).strip()
         #print 'wiki table: %r' % match.group(0)
@@ -853,7 +926,7 @@ class Markdown(object):
             row = [c.strip() for c in re.split(r'(?<!\\)\|\|', line)]
             rows.append(row)
         #pprint(rows)
-        hlines = ['<table>', '<tbody>']
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<tbody>']
         for row in rows:
             hrow = ['<tr>']
             for cell in row:
@@ -899,6 +972,9 @@ class Markdown(object):
 
         text = self._encode_amps_and_angles(text)
 
+        if "strike" in self.extras:
+            text = self._do_strike(text)
+
         text = self._do_italics_and_bold(text)
 
         if "smarty-pants" in self.extras:
@@ -1206,7 +1282,6 @@ class Markdown(object):
                                  .replace('_', self._escape_table['_'])
                         title = self.titles.get(link_id)
                         if title:
-                            before = title
                             title = _xml_escape_attr(title) \
                                 .replace('*', self._escape_table['*']) \
                                 .replace('_', self._escape_table['_'])
@@ -1418,7 +1493,6 @@ class Markdown(object):
     def _list_item_sub(self, match):
         item = match.group(4)
         leading_line = match.group(1)
-        leading_space = match.group(2)
         if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
             item = self._run_block_gamut(self._outdent(item))
         else:
@@ -1654,6 +1728,11 @@ class Markdown(object):
         self._escape_table[text] = hashed
         return hashed
 
+    _strike_re = re.compile(r"~~(?=\S)(.+?)(?<=\S)~~", re.S)
+    def _do_strike(self, text):
+        text = self._strike_re.sub(r"<strike>\1</strike>", text)
+        return text
+
     _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
     _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
     _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
@@ -1714,38 +1793,53 @@ class Markdown(object):
         text = text.replace(". . .", "&#8230;")
         return text
 
-    _block_quote_re = re.compile(r'''
+    _block_quote_base = r'''
         (                           # Wrap whole match in \1
           (
-            ^[ \t]*>[ \t]?          # '>' at the start of a line
+            ^[ \t]*>%s[ \t]?        # '>' at the start of a line
               .+\n                  # rest of the first line
             (.+\n)*                 # subsequent consecutive lines
             \n*                     # blanks
           )+
         )
-        ''', re.M | re.X)
+    '''
+    _block_quote_re = re.compile(_block_quote_base % '', re.M | re.X)
+    _block_quote_re_spoiler = re.compile(_block_quote_base % '[ \t]*?!?', re.M | re.X)
     _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M);
-
+    _bq_one_level_re_spoiler = re.compile('^[ \t]*>[ \t]*?![ \t]?', re.M);
+    _bq_all_lines_spoilers = re.compile(r'\A(?:^[ \t]*>[ \t]*?!.*[\n\r]*)+\Z', re.M)
     _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
     def _dedent_two_spaces_sub(self, match):
         return re.sub(r'(?m)^  ', '', match.group(1))
 
     def _block_quote_sub(self, match):
         bq = match.group(1)
-        bq = self._bq_one_level_re.sub('', bq)  # trim one level of quoting
-        bq = self._ws_only_line_re.sub('', bq)  # trim whitespace-only lines
+        is_spoiler = 'spoiler' in self.extras and self._bq_all_lines_spoilers.match(bq)
+        # trim one level of quoting
+        if is_spoiler:
+            bq = self._bq_one_level_re_spoiler.sub('', bq)
+        else:
+            bq = self._bq_one_level_re.sub('', bq)
+        # trim whitespace-only lines
+        bq = self._ws_only_line_re.sub('', bq)
         bq = self._run_block_gamut(bq)          # recurse
 
         bq = re.sub('(?m)^', '  ', bq)
         # These leading spaces screw with <pre> content, so we need to fix that:
         bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
 
-        return "<blockquote>\n%s\n</blockquote>\n\n" % bq
+        if is_spoiler:
+            return '<blockquote class="spoiler">\n%s\n</blockquote>\n\n' % bq
+        else:
+            return '<blockquote>\n%s\n</blockquote>\n\n' % bq
 
     def _do_block_quotes(self, text):
         if '>' not in text:
             return text
-        return self._block_quote_re.sub(self._block_quote_sub, text)
+        if 'spoiler' in self.extras:
+            return self._block_quote_re_spoiler.sub(self._block_quote_sub, text)
+        else:
+            return self._block_quote_re.sub(self._block_quote_sub, text)
 
     def _form_paragraphs(self, text):
         # Strip leading and trailing lines:
@@ -2053,7 +2147,6 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False):
     if DEBUG:
         print("dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
               % (tabsize, skip_first_line))
-    indents = []
     margin = None
     for i, line in enumerate(lines):
         if i == 0 and skip_first_line: continue
@@ -2362,4 +2455,4 @@ def main(argv=None):
 
 
 if __name__ == "__main__":
-    sys.exit( main(sys.argv) )
+    sys.exit( main(sys.argv) )

gluon/contrib/redis_scheduler.py

@@ -9,6 +9,17 @@ Scheduler with redis backend
 ---------------------------------
 """
 
+import os
+import time
+import socket
+import datetime
+import logging
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+from gluon.scheduler import *
+from gluon.scheduler import _decode_dict
+from gluon.contrib.redis_utils import RWatchError
+
 USAGE = """
 ## Example
 
@@ -35,11 +46,6 @@ mysched = RScheduler(db, dict(demo1=demo1,demo2=demo2), ...., redis_conn=rconn)
 
 """
 
-import os
-import time
-import socket
-import datetime
-import logging
 
 path = os.getcwd()
 
@@ -47,20 +53,19 @@ if 'WEB2PY_PATH' not in os.environ:
     os.environ['WEB2PY_PATH'] = path
 
 try:
-    from gluon.contrib.simplejson import loads, dumps
-except:
+    # try external module
     from simplejson import loads, dumps
+except ImportError:
+    try:
+        # try stdlib (Python >= 2.6)
+        from json import loads, dumps
+    except:
+        # fallback to pure-Python module
+        from gluon.contrib.simplejson import loads, dumps
 
 IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
-logger = logging.getLogger('web2py.rscheduler.%s' % IDENTIFIER)
-
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-from gluon.scheduler import *
-from gluon.scheduler import _decode_dict
-from gluon.contrib.redis_utils import RWatchError
-
+logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
 POLLING = 'POLLING'
 
@@ -111,8 +116,7 @@ class RScheduler(Scheduler):
         self._application = current.request.application or 'appname'
 
     def _nkey(self, key):
-        """Helper to restrict all keys to a namespace
-        and track them"""
+        """Helper to restrict all keys to a namespace and track them."""
         prefix = 'w2p:rsched:%s' % self._application
         allkeys = '%s:allkeys' % prefix
         newkey = "%s:%s" % (prefix, key)
@@ -120,10 +124,7 @@ class RScheduler(Scheduler):
         return newkey
 
     def prune_all(self):
-        """
-        Just to be fair and implement a method
-        that does housekeeping
-        """
+        """Global housekeeping."""
         all_keys = self._nkey('allkeys')
         with self.r_server.pipeline() as pipe:
             while True:
@@ -148,8 +149,9 @@ class RScheduler(Scheduler):
 
     def send_heartbeat(self, counter):
         """
-        workers coordination has evolved into something is not that
-        easy. Here we try to do what we need in a single transaction,
+        Workers coordination in redis.
+        It has evolved into something is not that easy.
+        Here we try to do what we need in a single transaction,
         and retry that transaction if something goes wrong
         """
         with self.r_server.pipeline() as pipe:
@@ -167,7 +169,9 @@ class RScheduler(Scheduler):
 
     def inner_send_heartbeat(self, counter, pipe):
         """
-        Does a few things:
+        Do a few things in the "maintenance" thread.
+
+        Specifically:
         - registers the workers
         - accepts commands sent to workers (KILL, TERMINATE, PICK, DISABLED, etc)
         - adjusts sleep
@@ -269,6 +273,8 @@ class RScheduler(Scheduler):
 
     def being_a_ticker(self, pipe):
         """
+        Elects a ticker.
+
         This is slightly more convoluted than the original
         but if far more efficient
         """
@@ -311,7 +317,9 @@ class RScheduler(Scheduler):
 
     def assign_tasks(self, db):
         """
-        The real beauty. We don't need to ASSIGN tasks, we just put
+        The real beauty.
+
+        We don't need to ASSIGN tasks, we just put
         them into the relevant queue
         """
         st, sd = db.scheduler_task, db.scheduler_task_deps
@@ -375,9 +383,6 @@ class RScheduler(Scheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -437,6 +442,7 @@ class RScheduler(Scheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def pop_task(self, db):
+        """Lift a task off a queue."""
         r_server = self.r_server
         st = self.db.scheduler_task
         task = None
@@ -533,7 +539,9 @@ class RScheduler(Scheduler):
 
     def report_task(self, task, task_report):
         """
-        Needs overwriting only because we need to pop from the
+        Override.
+
+        Needs it only because we need to pop from the
         running tasks
         """
         r_server = self.r_server
@@ -558,12 +566,12 @@ class RScheduler(Scheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
-                      and task.next_run_time > task.stop_time
-                      and True or False)
-        status = (task.run_again and is_expired and EXPIRED
-                  or task.run_again and not is_expired
-                  and QUEUED or COMPLETED)
+        is_expired = (task.stop_time and
+                      task.next_run_time > task.stop_time and
+                      True or False)
+        status = (task.run_again and is_expired and EXPIRED or
+                  task.run_again and not is_expired and
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             # assigned calculations
             d = dict(status=status,
@@ -579,10 +587,10 @@ class RScheduler(Scheduler):
             st_mapping = {'FAILED': 'FAILED',
                           'TIMEOUT': 'TIMEOUT',
                           'STOPPED': 'FAILED'}[task_report.status]
-            status = (task.retry_failed
-                      and task.times_failed < task.retry_failed
-                      and QUEUED or task.retry_failed == -1
-                      and QUEUED or st_mapping)
+            status = (task.retry_failed and
+                      task.times_failed < task.retry_failed and
+                      QUEUED or task.retry_failed == -1 and
+                      QUEUED or st_mapping)
             db(st.id == task.task_id).update(
                 times_failed=st.times_failed + 1,
                 next_run_time=task.next_run_time,
@@ -596,7 +604,7 @@ class RScheduler(Scheduler):
         r_server.hdel(running_dict, task.task_id)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -620,8 +628,8 @@ class RScheduler(Scheduler):
                 time.sleep(0.5)
 
     def get_workers(self, only_ticker=False):
-        """ Returns a dict holding worker_name : {**columns}
-        representing all "registered" workers
+        """Return a dict holding worker_name : {**columns}
+        representing all "registered" workers.
         only_ticker returns only the worker running as a TICKER,
         if there is any
         """

gluon/html.py

@@ -317,7 +317,7 @@ def URL(a=None,
                 response = current.response
                 if response.static_version and response.static_version_urls:
                     args = [function] + args
-                    function = '_'+str(response.static_version)
+                    function = '_' + str(response.static_version)
 
         if '.' in function:
             function, extension = function.rsplit('.', 1)
@@ -330,18 +330,16 @@ def URL(a=None,
     if args:
         if url_encode:
             if encode_embedded_slash:
-                other = '/' + '/'.join([urllib.quote(str(
-                    x), '') for x in args])
+                other = '/' + '/'.join([urllib.quote(str(x), '') for x in args])
             else:
-                other = args and urllib.quote(
-                    '/' + '/'.join([str(x) for x in args]))
+                other = args and urllib.quote('/' + '/'.join([str(x) for x in args]))
         else:
             other = args and ('/' + '/'.join([str(x) for x in args]))
     else:
         other = ''
 
     if other.endswith('/'):
-        other += '/'    # add trailing slash to make last trailing empty arg explicit
+        other += '/'  # add trailing slash to make last trailing empty arg explicit
 
     list_vars = []
     for (key, vals) in sorted(vars.items()):
@@ -364,11 +362,11 @@ def URL(a=None,
         h_args = '/%s/%s/%s%s' % (application, controller, function2, other)
 
         # how many of the vars should we include in our hash?
-        if hash_vars is True:       # include them all
+        if hash_vars is True:  # include them all
             h_vars = list_vars
-        elif hash_vars is False:    # include none of them
+        elif hash_vars is False:  # include none of them
             h_vars = ''
-        else:                       # include just those specified
+        else:  # include just those specified
             if hash_vars and not isinstance(hash_vars, (list, tuple)):
                 hash_vars = [hash_vars]
             h_vars = [(k, v) for (k, v) in list_vars if k in hash_vars]

gluon/languages.py

@@ -1002,17 +1002,19 @@ def update_all_languages(application_path):
             findT(application_path, language[:-3])
 
 
-def update_from_langfile(target, source):
+def update_from_langfile(target, source, force_update=False):
     """this will update untranslated messages in target from source (where both are language files)
     this can be used as first step when creating language file for new but very similar language
         or if you want update your app from welcome app of newer web2py version
         or in non-standard scenarios when you work on target and from any reason you have partial translation in source
+    Args:
+        force_update: if False existing translations remain unchanged, if True existing translations will update from source
     """
     src = read_dict(source)
     sentences = read_dict(target)
     for key in sentences:
         val = sentences[key]
-        if not val or val == key:
+        if not val or val == key or force_update:
             new_val = src.get(key)
             if new_val and new_val != val:
                 sentences[key] = new_val

gluon/scheduler.py

@@ -9,6 +9,26 @@ Background processes made simple
 ---------------------------------
 """
 
+
+import os
+import time
+import multiprocessing
+import sys
+import threading
+import traceback
+import signal
+import socket
+import datetime
+import logging
+import optparse
+import tempfile
+import types
+import Queue
+from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
+from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+
 USAGE = """
 ## Example
 
@@ -67,20 +87,6 @@ sudo restart web2py-scheduler
 sudo status web2py-scheduler
 """
 
-import os
-import time
-import multiprocessing
-import sys
-import threading
-import traceback
-import signal
-import socket
-import datetime
-import logging
-import optparse
-import types
-import Queue
-
 path = os.getcwd()
 
 if 'WEB2PY_PATH' not in os.environ:
@@ -101,12 +107,6 @@ IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
 logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
-from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
-from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-
-
 QUEUED = 'QUEUED'
 ASSIGNED = 'ASSIGNED'
 RUNNING = 'RUNNING'
@@ -168,24 +168,25 @@ class TaskReport(object):
 
 
 class JobGraph(object):
-    """Experimental: with JobGraph you can specify
-    dependencies amongs tasks"""
+    """Experimental: dependencies amongs tasks"""
 
     def __init__(self, db, job_name):
         self.job_name = job_name or 'job_0'
         self.db = db
 
     def add_deps(self, task_parent, task_child):
-        """Creates a dependency between task_parent and task_child"""
+        """Create a dependency between task_parent and task_child."""
         self.db.scheduler_task_deps.insert(task_parent=task_parent,
                                            task_child=task_child,
                                            job_name=self.job_name)
 
     def validate(self, job_name=None):
-        """Validates if all tasks job_name can be completed, i.e. there
-        are no mutual dependencies among tasks.
+        """Validate if all tasks job_name can be completed.
+
+        Checks if there are no mutual dependencies among tasks.
         Commits at the end if successfull, or it rollbacks the entire
-        transaction. Handle with care!"""
+        transaction. Handle with care!
+        """
         db = self.db
         sd = db.scheduler_task_deps
         if job_name:
@@ -223,14 +224,6 @@ class JobGraph(object):
             db.rollback()
             return None
 
-
-def demo_function(*argv, **kwargs):
-    """ test function """
-    for i in range(argv[0]):
-        print 'click', i
-        time.sleep(1)
-    return 'done'
-
 # the two functions below deal with simplejson decoding as unicode, esp for the dict decode
 # and subsequent usage as function Keyword arguments unicode variable names won't work!
 # borrowed from http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-unicode-ones-from-json-in-python
@@ -261,11 +254,12 @@ def _decode_dict(dct):
 
 
 def executor(queue, task, out):
-    """The function used to execute tasks in the background process"""
+    """The function used to execute tasks in the background process."""
     logger.debug('    task started')
 
     class LogOutput(object):
-        """Facility to log output at intervals"""
+        """Facility to log output at intervals."""
+
         def __init__(self, out_queue):
             self.out_queue = out_queue
             self.stdout = sys.stdout
@@ -280,7 +274,11 @@ def executor(queue, task, out):
         def write(self, data):
             self.out_queue.put(data)
 
-    W2P_TASK = Storage({'id': task.task_id, 'uuid': task.uuid})
+    W2P_TASK = Storage({
+                       'id': task.task_id,
+                       'uuid': task.uuid,
+                       'run_id': task.run_id
+                       })
     stdout = LogOutput(out)
     try:
         if task.app:
@@ -318,6 +316,11 @@ def executor(queue, task, out):
             result = eval(task.function)(
                 *loads(task.args, object_hook=_decode_dict),
                 **loads(task.vars, object_hook=_decode_dict))
+        if len(result) >= 1024:
+            fd, temp_path = tempfile.mkstemp(suffix='.w2p_sched')
+            with os.fdopen(fd, 'w') as f:
+                f.write(result)
+            result = 'w2p_special:%s' % temp_path
         queue.put(TaskReport('COMPLETED', result=result))
     except BaseException, e:
         tb = traceback.format_exc()
@@ -335,7 +338,7 @@ class MetaScheduler(threading.Thread):
         self.empty_runs = 0
 
     def async(self, task):
-        """Starts the background process
+        """Start the background process.
 
         Args:
             task : a `Task` object
@@ -410,6 +413,12 @@ class MetaScheduler(threading.Thread):
             else:
                 logger.debug('  task completed or failed')
                 tr = queue.get()
+        result = tr.result
+        if result and result.startswith('w2p_special'):
+            temp_path = result.replace('w2p_special:', '', 1)
+            with open(temp_path) as f:
+                tr.result = f.read()
+            os.unlink(temp_path)
         tr.output = task_output
         return tr
 
@@ -444,50 +453,23 @@ class MetaScheduler(threading.Thread):
         self.start()
 
     def send_heartbeat(self, counter):
-        print 'thum'
-        time.sleep(1)
+        raise NotImplementedError
 
     def pop_task(self):
         """Fetches a task ready to be executed"""
-        return Task(
-            app=None,
-            function='demo_function',
-            timeout=7,
-            args='[2]',
-            vars='{}')
+        raise NotImplementedError
 
     def report_task(self, task, task_report):
         """Creates a task report"""
-        print 'reporting task'
-        pass
+        raise NotImplementedError
 
     def sleep(self):
-        pass
+        raise NotImplementedError
 
     def loop(self):
         """Main loop, fetching tasks and starting executor's background
         processes"""
-        try:
-            self.start_heartbeats()
-            while True and self.have_heartbeat:
-                logger.debug('looping...')
-                task = self.pop_task()
-                if task:
-                    self.empty_runs = 0
-                    self.report_task(task, self.async(task))
-                else:
-                    self.empty_runs += 1
-                    logger.debug('sleeping...')
-                    if self.max_empty_runs != 0:
-                        logger.debug('empty runs %s/%s',
-                                     self.empty_runs, self.max_empty_runs)
-                        if self.empty_runs >= self.max_empty_runs:
-                            logger.info(
-                                'empty runs limit reached, killing myself')
-                            self.die()
-                    self.sleep()
-        except KeyboardInterrupt:
-            self.die()
+        raise NotImplementedError
 
 
 TASK_STATUS = (QUEUED, RUNNING, COMPLETED, FAILED, TIMEOUT, STOPPED, EXPIRED)
@@ -594,11 +576,11 @@ class Scheduler(MetaScheduler):
         return True
 
     def now(self):
-        """Shortcut that fetches current time based on UTC preferences"""
+        """Shortcut that fetches current time based on UTC preferences."""
         return self.utc_time and datetime.datetime.utcnow() or datetime.datetime.now()
 
     def set_requirements(self, scheduler_task):
-        """Called to set defaults for lazy_tables connections"""
+        """Called to set defaults for lazy_tables connections."""
         from gluon import current
         if hasattr(current, 'request'):
             scheduler_task.application_name.default = '%s/%s' % (
@@ -606,7 +588,7 @@ class Scheduler(MetaScheduler):
             )
 
     def define_tables(self, db, migrate):
-        """Defines Scheduler tables structure"""
+        """Define Scheduler tables structure."""
         from pydal.base import DEFAULT
         logger.debug('defining tables (migrate=%s)', migrate)
         now = self.now
@@ -693,14 +675,14 @@ class Scheduler(MetaScheduler):
 
     @staticmethod
     def total_seconds(td):
-        # backport for py2.6
+        """Backport for py2.6."""
         if hasattr(td, 'total_seconds'):
             return td.total_seconds()
         else:
             return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6
 
     def loop(self, worker_name=None):
-        """Main loop
+        """Main loop.
 
         This works basically as a neverending loop that:
 
@@ -752,7 +734,8 @@ class Scheduler(MetaScheduler):
             self.die()
 
     def wrapped_assign_tasks(self, db):
-        """Commodity function to call `assign_tasks` and trap exceptions
+        """Commodity function to call `assign_tasks` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `assign_task` after 0.5 seconds
         """
@@ -773,7 +756,8 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -793,7 +777,7 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def pop_task(self, db):
-        """Grabs a task ready to be executed from the queue"""
+        """Grab a task ready to be executed from the queue."""
         now = self.now()
         st = self.db.scheduler_task
         if self.is_a_ticker and self.do_assign_tasks:
@@ -874,7 +858,8 @@ class Scheduler(MetaScheduler):
             uuid=task.uuid)
 
     def wrapped_report_task(self, task, task_report):
-        """Commodity function to call `report_task` and trap exceptions
+        """Commodity function to call `report_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -891,8 +876,10 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def report_task(self, task, task_report):
-        """Takes care of storing the result according to preferences
-        and deals with logic for repeating tasks"""
+        """Take care of storing the result according to preferences.
+
+        Deals with logic for repeating tasks.
+        """
         db = self.db
         now = self.now()
         st = db.scheduler_task
@@ -914,12 +901,12 @@ class Scheduler(MetaScheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
-                      and task.next_run_time > task.stop_time
-                      and True or False)
-        status = (task.run_again and is_expired and EXPIRED
-                  or task.run_again and not is_expired
-                  and QUEUED or COMPLETED)
+        is_expired = (task.stop_time and
+                      task.next_run_time > task.stop_time and
+                      True or False)
+        status = (task.run_again and is_expired and EXPIRED or
+                  task.run_again and not is_expired and
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             d = dict(status=status,
                      next_run_time=task.next_run_time,
@@ -945,27 +932,26 @@ class Scheduler(MetaScheduler):
         logger.info('task completed (%s)', task_report.status)
 
     def update_dependencies(self, db, task_id):
+        """Unblock execution paths for Jobs."""
         db(db.scheduler_task_deps.task_child == task_id).update(can_visit=True)
 
     def adj_hibernation(self):
-        """Used to increase the "sleep" interval for DISABLED workers"""
+        """Used to increase the "sleep" interval for DISABLED workers."""
         if self.w_stats.status == DISABLED:
             wk_st = self.w_stats.sleep
             hibernation = wk_st + HEARTBEAT if wk_st < MAXHIBERNATION else MAXHIBERNATION
             self.w_stats.sleep = hibernation
 
     def send_heartbeat(self, counter):
-        """This function is vital for proper coordination among available
-        workers.
-        It:
+        """Coordination among available workers.
 
+        It:
         - sends the heartbeat
         - elects a ticker among available workers (the only process that
             effectively dispatch tasks to workers)
         - deals with worker's statuses
         - does "housecleaning" for dead workers
         - triggers tasks assignment to workers
-
         """
         if not self.db_thread:
             logger.debug('thread building own DAL object')
@@ -1053,7 +1039,8 @@ class Scheduler(MetaScheduler):
         self.sleep()
 
     def being_a_ticker(self):
-        """Elects a TICKER process that assigns tasks to available workers.
+        """Elect a TICKER process that assigns tasks to available workers.
+
         Does its best to elect a worker that is not busy processing other tasks
         to allow a proper distribution of tasks among all active workers ASAP
         """
@@ -1087,7 +1074,7 @@ class Scheduler(MetaScheduler):
             return False
 
     def assign_tasks(self, db):
-        """Assigns task to workers, that can then pop them from the queue
+        """Assign task to workers, that can then pop them from the queue.
 
         Deals with group_name(s) logic, in order to assign linearly tasks
         to available workers for those groups
@@ -1137,9 +1124,6 @@ class Scheduler(MetaScheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -1151,8 +1135,8 @@ class Scheduler(MetaScheduler):
         # intelligence (like esteeming how many tasks will a worker complete
         # before the ticker reassign them around, but the gain is quite small
         # 50 is a sweet spot also for fast tasks, with sane heartbeat values
-        # NB: ticker reassign tasks every 5 cycles, so if a worker completes its
-        # 50 tasks in less than heartbeat*5 seconds,
+        # NB: ticker reassign tasks every 5 cycles, so if a worker completes
+        # its 50 tasks in less than heartbeat*5 seconds,
         # it won't pick new tasks until heartbeat*5 seconds pass.
 
         # If a worker is currently elaborating a long task, its tasks needs to
@@ -1165,7 +1149,7 @@ class Scheduler(MetaScheduler):
         x = 0
         for group in wkgroups.keys():
             tasks = all_available(st.group_name == group).select(
-                limitby=(0, limit), orderby = st.next_run_time)
+                limitby=(0, limit), orderby=st.next_run_time)
             # let's break up the queue evenly among workers
             for task in tasks:
                 x += 1
@@ -1183,8 +1167,6 @@ class Scheduler(MetaScheduler):
                         status=ASSIGNED,
                         assigned_worker_name=assigned_wn
                     )
-                    if not task.task_name:
-                        d['task_name'] = task.function_name
                     db(
                         (st.id == task.id) &
                         (st.status.belongs((QUEUED, ASSIGNED)))
@@ -1204,14 +1186,13 @@ class Scheduler(MetaScheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def sleep(self):
-        """Calculates the number of seconds to sleep according to worker's
-        status and `heartbeat` parameter"""
+        """Calculate the number of seconds to sleep."""
         time.sleep(self.w_stats.sleep)
         # should only sleep until next available task
 
     def set_worker_status(self, group_names=None, action=ACTIVE,
                           exclude=None, limit=None, worker_name=None):
-        """Internal function to set worker's status"""
+        """Internal function to set worker's status."""
         ws = self.db.scheduler_worker
         if not group_names:
             group_names = self.group_names
@@ -1235,10 +1216,12 @@ class Scheduler(MetaScheduler):
                 self.db(ws.id.belongs(workers)).update(status=action)
 
     def disable(self, group_names=None, limit=None, worker_name=None):
-        """Sets DISABLED on the workers processing `group_names` tasks.
+        """Set DISABLED on the workers processing `group_names` tasks.
+
         A DISABLED worker will be kept alive but it won't be able to process
         any waiting tasks, essentially putting it to sleep.
-        By default, all group_names of Scheduler's instantation are selected"""
+        By default, all group_names of Scheduler's instantation are selected
+        """
         self.set_worker_status(
             group_names=group_names,
             action=DISABLED,
@@ -1283,8 +1266,9 @@ class Scheduler(MetaScheduler):
             pvars: "raw" kwargs to be passed to the function. Automatically
                 jsonified
             kwargs: all the parameters available (basically, every
-                `scheduler_task` column). If args and vars are here, they should
-                be jsonified already, and they will override pargs and pvars
+                `scheduler_task` column). If args and vars are here, they
+                should be jsonified already, and they will override pargs
+                and pvars
 
         Returns:
             a dict just as a normal validate_and_insert(), plus a uuid key

gluon/tests/__init__.py

@@ -21,6 +21,7 @@ from test_contribs import *
 from test_web import *
 from test_dal import *
 from test_tools import *
+from test_appadmin import *
 from test_scheduler import *
 
 if sys.version[:3] == '2.7':

gluon/tests/test_appadmin.py

@@ -0,0 +1,175 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.sqlhtml
+"""
+import os
+import sys
+if sys.version < "2.7":
+    import unittest2 as unittest
+else:
+    import unittest
+
+from fix_path import fix_sys_path
+
+fix_sys_path(__file__)
+
+
+from compileapp import run_controller_in, run_view_in
+from languages import translator
+from gluon.storage import Storage, List
+import gluon.fileutils
+from gluon.dal import DAL, Field, Table
+from gluon.http import HTTP
+
+DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
+
+try:
+    import json
+except ImportError:
+    from gluon.contrib import simplejson as json
+
+
+def fake_check_credentials(foo):
+    return True
+
+
+class TestAppAdmin(unittest.TestCase):
+
+    def setUp(self):
+        from gluon.globals import Request, Response, Session, current
+        from gluon.html import A, DIV, FORM, MENU, TABLE, TR, INPUT, URL, XML
+        from gluon.validators import IS_NOT_EMPTY
+        from compileapp import LOAD
+        from gluon.http import HTTP, redirect
+        from gluon.tools import Auth
+        from gluon.sql import SQLDB
+        from gluon.sqlhtml import SQLTABLE, SQLFORM
+        self.original_check_credentials = gluon.fileutils.check_credentials
+        gluon.fileutils.check_credentials = fake_check_credentials
+        request = Request(env={})
+        request.application = 'welcome'
+        request.controller = 'appadmin'
+        request.function = self._testMethodName.split('_')[1]
+        request.folder = 'applications/welcome'
+        request.env.http_host = '127.0.0.1:8000'
+        request.env.remote_addr = '127.0.0.1'
+        response = Response()
+        session = Session()
+        T = translator('', 'en')
+        session.connect(request, response)
+        current.request = request
+        current.response = response
+        current.session = session
+        current.T = T
+        db = DAL(DEFAULT_URI, check_reserved=['all'])
+        auth = Auth(db)
+        auth.define_tables(username=True, signature=False)
+        db.define_table('t0', Field('tt'), auth.signature)
+        # Create a user
+        db.auth_user.insert(first_name='Bart',
+                            last_name='Simpson',
+                            username='user1',
+                            email='user1@test.com',
+                            password='password_123',
+                            registration_key=None,
+                            registration_id=None)
+        self.env = locals()
+
+    def tearDown(self):
+        gluon.fileutils.check_credentials = self.original_check_credentials
+
+    def run_function(self):
+        return run_controller_in(self.env['request'].controller, self.env['request'].function, self.env)
+
+    def run_view(self):
+        return run_view_in(self.env)
+
+    def test_index(self):
+        result = self.run_function()
+        self.assertTrue('db' in result['databases'])
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_select(self):
+        request = self.env['request']
+        request.args = List(['db'])
+        request.env.query_string = 'query=db.auth_user.id>0'
+        result = self.run_function()
+        self.assertTrue('table' in result and 'query' in result)
+        self.assertTrue(result['table'] == 'auth_user')
+        self.assertTrue(result['query'] == 'db.auth_user.id>0')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        result = self.run_function()
+        self.assertTrue('table' in result)
+        self.assertTrue('form' in result)
+        self.assertTrue(str(result['table']) is 'auth_user')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        data['first_name'] = 'Lisa'
+        data['last_name'] = 'Simpson'
+        data['username'] = 'lisasimpson'
+        data['password'] = 'password_123'
+        data['email'] = 'lisa@example.com'
+        request._vars = data
+        result = self.run_function()
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+        db = self.env['db']
+        lisa_record = db(db.auth_user.username == 'lisasimpson').select().first()
+        self.assertIsNotNone(lisa_record)
+        del data['_formkey']
+        del data['_formname']
+        del data['password']
+        for key in data:
+            self.assertEqual(data[key], lisa_record[key])
+
+    def test_update_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user', '1'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        for element in form.elements('input'):
+            data[element['_name']] = element['_value']
+        data['email'] = 'user1@example.com'
+        data['id'] = '1'
+        request._vars = data
+        self.assertRaises(HTTP, self.run_function)
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_html.py

@@ -98,6 +98,11 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
         rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'p': (1, 3), 'q': 2}, hmac_key='key', hash_vars='p')
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
+        # test url_encode
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=False)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li\xc3\xa9=2&ma\xc3\xaf=1&ma\xc3\xaf=3')
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=True)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li%C3%A9=2&ma%C3%AF=1&ma%C3%AF=3')
         # test CRLF detection
         self.assertRaises(SyntaxError, URL, *['a\n', 'c', 'f'])
         self.assertRaises(SyntaxError, URL, *['a\r', 'c', 'f'])
@@ -394,18 +399,12 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(HR(_a='1', _b='2').xml(), '<hr a="1" b="2" />')
 
     def test_A(self):
-        self.assertEqual(
-            A('<>', _a='1', _b='2').xml(),
-            '<a a="1" b="2">&lt;&gt;</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>'
-            )
+        self.assertEqual(A('<>', _a='1', _b='2').xml(),
+                         '<a a="1" b="2">&lt;&gt;</a>')
+        self.assertEqual(A('a', cid='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
+        self.assertEqual(A('a', callback='b', _id='c').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>')
         # Callback with no id trigger web2py_uuid() call
         from html import web2pyHTMLParser
         a = A('a', callback='b').xml()
@@ -413,38 +412,22 @@ class TestBareHelpers(unittest.TestCase):
             uuid_generated = tag.attributes['_id']
         self.assertEqual(a,
                          '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="{id}">a</a>'.format(id=uuid_generated))
-        self.assertEqual(
-            A('a', delete='tr').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>'
-            )
-        self.assertEqual(
-            A('a', _id='b', target='<self>').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', component='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', _id='b', callback='c', noconfirm=True).xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b', _disable_with='processing...').xml(),
-            '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
-            '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
+        self.assertEqual(A('a', delete='tr').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>')
+        self.assertEqual(A('a', _id='b', target='<self>').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>')
+        self.assertEqual(A('a', component='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>')
+        self.assertEqual(A('a', _id='b', callback='c', noconfirm=True).xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>')
+        self.assertEqual(A('a', cid='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
+        self.assertEqual(A('a', cid='b', _disable_with='processing...').xml(),
+                         '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>')
+        self.assertEqual(A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
+        self.assertEqual(A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
+                         '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
 
     def test_BUTTON(self):
         self.assertEqual(BUTTON('test', _type='button').xml(),

gluon/tests/test_recfile.py

@@ -23,8 +23,8 @@ class TestRecfile(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree('tests')
 
-    def testgeneration(self):
-        for k in range(20):
+    def test_generation(self):
+        for k in range(10):
             teststring = 'test%s' % k
             filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
             with recfile.open(filename, "w") as g:
@@ -35,6 +35,40 @@ class TestRecfile(unittest.TestCase):
             recfile.remove(filename)
             is_there = recfile.exists(filename)
             self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = str(uuid.uuid4()) + '.test'
+            with recfile.open(filename, "w", path='tests') as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r", path='tests').read(), teststring)
+            is_there = recfile.exists(filename, path='tests')
+            self.assertTrue(is_there)
+            recfile.remove(filename, path='tests')
+            is_there = recfile.exists(filename, path='tests')
+            self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = os.path.join('tests', str(uuid.uuid4()), str(uuid.uuid4()) + '.test')
+            with recfile.open(filename, "w") as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r").read(), teststring)
+            is_there = recfile.exists(filename)
+            self.assertTrue(is_there)
+            recfile.remove(filename)
+            is_there = recfile.exists(filename)
+            self.assertFalse(is_there)
+
+    def test_existing(self):
+        filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
+        with open(filename, 'w') as g:
+            g.write('this file exists')
+        self.assertTrue(recfile.exists(filename))
+        self.assertTrue(hasattr(recfile.open(filename, "r"), 'read'))
+        recfile.remove(filename, path='tests')
+        self.assertFalse(recfile.exists(filename))
+        self.assertRaises(IOError, recfile.remove, filename)
+        self.assertRaises(IOError, recfile.open, filename, "r")
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_scheduler.py

@@ -9,6 +9,7 @@ import unittest
 import glob
 import datetime
 import sys
+
 from fix_path import fix_sys_path
 
 
@@ -255,6 +256,14 @@ class testForSchedulerRunnerBase(BaseTestScheduler):
         from gluon import current
         fdest = os.path.join(current.request.folder, 'models', 'scheduler.py')
         os.unlink(fdest)
+        additional_files = [
+            os.path.join(current.request.folder, 'private', 'demo8.pholder')
+        ]
+        for f in additional_files:
+            try:
+                os.unlink(f)
+            except:
+                pass
 
     def writefunction(self, content, initlines=None):
         from gluon import current
@@ -266,7 +275,7 @@ import time
 from gluon.scheduler import Scheduler
 db_dal = os.path.abspath(os.path.join(request.folder, '..', '..', 'dummy2.db'))
 sched_dal = DAL('sqlite://%s' % db_dal, folder=os.path.dirname(db_dal))
-sched = Scheduler(sched_dal, max_empty_runs=20, migrate=False, heartbeat=1)
+sched = Scheduler(sched_dal, max_empty_runs=15, migrate=False, heartbeat=1)
             """
         with open(fdest, 'w') as q:
             q.write(initlines)
@@ -274,34 +283,179 @@ sched = Scheduler(sched_dal, max_empty_runs=20, migrate=False, heartbeat=1)
 
     def exec_sched(self):
         import subprocess
-        call_args = [sys.executable, 'web2py.py', '-K', 'welcome']
+        call_args = [sys.executable, 'web2py.py', '--no-banner', '-D', '20','-K', 'welcome']
         ret = subprocess.call(call_args, env=dict(os.environ))
         return ret
 
+    def fetch_results(self, sched, task):
+        info = sched.task_status(task.id)
+        task_runs = self.db(self.db.scheduler_run.task_id == task.id).select()
+        return info, task_runs
+
+    def exec_asserts(self, stmts, tag):
+        for stmt in stmts:
+            self.assertEqual(stmt[1], True, msg="%s - %s" % (tag, stmt[0]))
+
 
 class TestsForSchedulerRunner(testForSchedulerRunnerBase):
 
-    def testBasic(self):
+    def testRepeats_and_Expired_and_Prio(self):
         s = Scheduler(self.db)
-        foo = s.queue_task('foo')
+        repeats = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), repeats=2, period=5)
+        a_while_ago = datetime.datetime.now() - datetime.timedelta(seconds=60)
+        expired = s.queue_task('demo4', stop_time=a_while_ago)
+        prio1 = s.queue_task('demo1', ['scheduled_first'])
+        prio2 = s.queue_task('demo1', ['scheduled_second'], next_run_time=a_while_ago)
         self.db.commit()
         self.writefunction(r"""
-def foo():
-    return 'a'
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
 """)
         ret = self.exec_sched()
-        # process finished just fine
         self.assertEqual(ret, 0)
-        info = s.task_status(foo.id, output=True)
-        self.assertEqual(info.result, 'a')
+        # repeats check
+        task, task_run = self.fetch_results(s, repeats)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are COMPLETED ", (task_run[0].status == task_run[1].status == 'COMPLETED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'REPEATS')
+
+        # expired check
+        task, task_run = self.fetch_results(s, expired)
+        res = [
+            ("task status expired", task.status == 'EXPIRED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task didn't run at all", len(task_run) == 0)
+        ]
+        self.exec_asserts(res, 'EXPIRATION')
+
+        # prio check
+        task1 = s.task_status(prio1.id, output=True)
+        task2 = s.task_status(prio2.id, output=True)
+        res = [
+            ("tasks status completed", task1.scheduler_task.status == task2.scheduler_task.status == 'COMPLETED'),
+            ("priority2 was executed before priority1" , task1.scheduler_run.id > task2.scheduler_run.id)
+        ]
+        self.exec_asserts(res, 'PRIORITY')
+
+    def testNoReturn_and_Timeout_and_Progress(self):
+        s = Scheduler(self.db)
+        noret1 = s.queue_task('demo5')
+        noret2 = s.queue_task('demo3')
+        timeout1 = s.queue_task('demo4', timeout=5)
+        timeout2 = s.queue_task('demo4')
+        progress = s.queue_task('demo6', sync_output=2)
+        self.db.commit()
+        self.writefunction(r"""
+def demo3():
+    time.sleep(15)
+    print 1/0
+    return None
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
+
+def demo5():
+    time.sleep(15)
+    print "I'm printing something"
+    rtn = dict(a=1, b=2)
+
+def demo6():
+    time.sleep(5)
+    print '50%'
+    time.sleep(5)
+    print '!clear!100%'
+    return 1
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # noreturn check
+        task1, task_run1 = self.fetch_results(s, noret1)
+        task2, task_run2 = self.fetch_results(s, noret2)
+        res = [
+            ("tasks no_returns1 completed", task1.status == 'COMPLETED'),
+            ("tasks no_returns2 failed", task2.status == 'FAILED'),
+            ("no_returns1 doesn't have a scheduler_run record", len(task_run1) == 0),
+            ("no_returns2 has a scheduler_run record FAILED", (len(task_run2) == 1 and task_run2[0].status == 'FAILED')),
+        ]
+        self.exec_asserts(res, 'NO_RETURN')
+
+        # timeout check
+        task1 = s.task_status(timeout1.id, output=True)
+        task2 = s.task_status(timeout2.id, output=True)
+        res = [
+            ("tasks timeouts1 timeoutted", task1.scheduler_task.status == 'TIMEOUT'),
+            ("tasks timeouts2 completed", task2.scheduler_task.status == 'COMPLETED')
+        ]
+        self.exec_asserts(res, 'TIMEOUT')
+
+        # progress check
+        task1 = s.task_status(progress.id, output=True)
+        res = [
+            ("tasks percentages completed", task1.scheduler_task.status == 'COMPLETED'),
+            ("output contains only 100%", task1.scheduler_run.run_output.strip() == "100%")
+        ]
+        self.exec_asserts(res, 'PROGRESS')
+
+    def testDrift_and_env_and_immediate(self):
+        s = Scheduler(self.db)
+        immediate = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), immediate=True)
+        env = s.queue_task('demo7')
+        drift = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), period=93, prevent_drift=True)
+        self.db.commit()
+        self.writefunction(r"""
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+import random
+def demo7():
+    time.sleep(random.randint(1,5))
+    print W2P_TASK, request.now
+    return W2P_TASK.id, W2P_TASK.uuid, W2P_TASK.run_id
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # immediate check, can only check that nothing breaks
+        task1 = s.task_status(immediate.id)
+        res = [
+            ("tasks status completed", task1.status == 'COMPLETED'),
+        ]
+        self.exec_asserts(res, 'IMMEDIATE')
+
+        # drift check
+        task, task_run = self.fetch_results(s, drift)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("next_run_time is exactly start_time + period", (task.next_run_time == task.start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'DRIFT')
+
+        # env check
+        task1 = s.task_status(env.id, output=True)
+        res = [
+            ("task %s returned W2P_TASK correctly" % (task1.scheduler_task.id),  task1.result == [task1.scheduler_task.id, task1.scheduler_task.uuid, task1.scheduler_run.id]),
+        ]
+        self.exec_asserts(res, 'ENV')
+
 
     def testRetryFailed(self):
         s = Scheduler(self.db)
-        failed = s.queue_task('demo2', retry_failed=1, period=5)
-        failed_consecutive = s.queue_task('demo8', retry_failed=2, repeats=2, period=5)
+        failed = s.queue_task('demo2', retry_failed=1, period=1)
+        failed_consecutive = s.queue_task('demo8', retry_failed=2, repeats=2, period=1)
         self.db.commit()
         self.writefunction(r"""
-
 def demo2():
     1/0
 
@@ -323,32 +477,50 @@ def demo8():
         # process finished just fine
         self.assertEqual(ret, 0)
         # failed - checks
-        info = s.task_status(failed.id)
-        task_runs = self.db(self.db.scheduler_run.task_id == info.id).select()
+        task, task_run = self.fetch_results(s, failed)
         res = [
-            ("task status failed", info.status == 'FAILED'),
-            ("task times_run is 0", info.times_run == 0),
-            ("task times_failed is 2", info.times_failed == 2),
-            ("task ran 2 times only", len(task_runs) == 2),
-            ("scheduler_run records are FAILED", (task_runs[0].status == task_runs[1].status == 'FAILED')),
-            ("period is respected", (task_runs[1].start_time > task_runs[0].start_time + datetime.timedelta(seconds=info.period)))
+            ("task status failed", task.status == 'FAILED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task times_failed is 2", task.times_failed == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are FAILED", (task_run[0].status == task_run[1].status == 'FAILED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
         ]
-        for a in res:
-            self.assertEqual(a[1], True, msg=a[0])
+        self.exec_asserts(res, 'FAILED')
 
         # failed consecutive - checks
-        info = s.task_status(failed_consecutive.id)
-        task_runs = self.db(self.db.scheduler_run.task_id == info.id).select()
+        task, task_run = self.fetch_results(s, failed_consecutive)
         res = [
-            ("task status completed", info.status == 'COMPLETED'),
-            ("task times_run is 2", info.times_run == 2),
-            ("task times_failed is 0", info.times_failed == 0),
-            ("task ran 6 times", len(task_runs) == 6),
-            ("scheduler_run records for COMPLETED is 2", len([run.status for run in task_runs if run.status == 'COMPLETED']) == 2),
-            ("scheduler_run records for FAILED is 4", len([run.status for run in task_runs if run.status == 'FAILED']) == 4),
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task times_failed is 0", task.times_failed == 0),
+            ("task ran 6 times", len(task_run) == 6),
+            ("scheduler_run records for COMPLETED is 2", len([run.status for run in task_run if run.status == 'COMPLETED']) == 2),
+            ("scheduler_run records for FAILED is 4", len([run.status for run in task_run if run.status == 'FAILED']) == 4),
         ]
-        for a in res:
-            self.assertEqual(a[1], True, msg=a[0])
+        self.exec_asserts(res, 'FAILED_CONSECUTIVE')
+
+    def testHugeResult(self):
+        s = Scheduler(self.db)
+        huge_result = s.queue_task('demo10', retry_failed=1, period=1)
+        self.db.commit()
+        self.writefunction(r"""
+def demo10():
+    res = 'a' * 99999
+    return dict(res=res)
+""")
+        ret = self.exec_sched()
+        # process finished just fine
+        self.assertEqual(ret, 0)
+        # huge_result - checks
+        task = s.task_status(huge_result.id, output=True)
+        res = [
+            ("task status completed", task.scheduler_task.status == 'COMPLETED'),
+            ("task times_run is 1", task.scheduler_task.times_run == 1),
+            ("result is the correct one", task.result == dict(res='a' * 99999))
+        ]
+        self.exec_asserts(res, 'HUGE_RESULT')
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_tools.py

@@ -7,6 +7,7 @@
 import os
 import sys
 import smtplib
+import datetime
 if sys.version < "2.7":
     import unittest2 as unittest
 else:
@@ -20,7 +21,7 @@ DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
 from gluon.dal import DAL, Field
 from pydal.objects import Table
-from tools import Auth, Mail, Recaptcha, Recaptcha2
+from tools import Auth, Mail, Recaptcha, Recaptcha2, prettydate
 from gluon.globals import Request, Response, Session
 from storage import Storage
 from languages import translator
@@ -229,23 +230,242 @@ class TestMail(unittest.TestCase):
 
 
 @unittest.skipIf(IS_IMAP, "TODO: Imap raises 'Connection refused'")
+# class TestAuth(unittest.TestCase):
+#
+#     def setUp(self):
+#         request = Request(env={})
+#         request.application = 'a'
+#         request.controller = 'c'
+#         request.function = 'f'
+#         request.folder = 'applications/admin'
+#         response = Response()
+#         session = Session()
+#         T = translator('', 'en')
+#         session.connect(request, response)
+#         from gluon.globals import current
+#         current.request = request
+#         current.response = response
+#         current.session = session
+#         current.T = T
+#         self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+#         self.auth = Auth(self.db)
+#         self.auth.define_tables(username=True, signature=False)
+#         self.db.define_table('t0', Field('tt'), self.auth.signature)
+#         self.auth.enable_record_versioning(self.db)
+#         # Create a user
+#         self.auth.get_or_create_user(dict(first_name='Bart',
+#                                           last_name='Simpson',
+#                                           username='bart',
+#                                           email='bart@simpson.com',
+#                                           password='bart_password',
+#                                           registration_key='bart',
+#                                           registration_id=''
+#                                           ))
+#         # self.auth.settings.registration_requires_verification = False
+#         # self.auth.settings.registration_requires_approval = False
+#
+#     def test_assert_setup(self):
+#         self.assertEqual(self.db(self.db.auth_user.username == 'bart').select().first()['username'], 'bart')
+#         self.assertTrue('auth_user' in self.db)
+#         self.assertTrue('auth_group' in self.db)
+#         self.assertTrue('auth_membership' in self.db)
+#         self.assertTrue('auth_permission' in self.db)
+#         self.assertTrue('auth_event' in self.db)
+#
+#     def test_enable_record_versioning(self):
+#         self.assertTrue('t0_archive' in self.db)
+#
+#     def test_basic_blank_forms(self):
+#         for f in ['login', 'retrieve_password',
+#                   'retrieve_username',
+#                   # 'register'  # register complain about : client_side=self.settings.client_side
+#                   ]:
+#             html_form = getattr(self.auth, f)().xml()
+#             self.assertTrue('name="_formkey"' in html_form)
+#
+#         # NOTE: Not sure it is the proper way to logout_bare() as there is not methods for that and auth.logout() failed
+#         self.auth.logout_bare()
+#         # self.assertTrue(self.auth.is_logged_in())
+#
+#         for f in ['logout', 'verify_email', 'reset_password',
+#                   'change_password', 'profile', 'groups']:
+#             self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
+#
+#         self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate)
+#
+#         try:
+#             for t in ['t0_archive', 't0', 'auth_cas', 'auth_event',
+#                       'auth_membership', 'auth_permission', 'auth_group',
+#                       'auth_user']:
+#                 self.db[t].drop()
+#         except SyntaxError as e:
+#             # GAE doesn't support drop
+#             pass
+#         return
+#
+#     def test_get_or_create_user(self):
+#         self.db.auth_user.insert(email='user1@test.com', username='user1', password='password_123')
+#         self.db.commit()
+#         # True case
+#         self.assertEqual(self.auth.get_or_create_user({'email': 'user1@test.com',
+#                                                        'username': 'user1',
+#                                                        'password': 'password_123'
+#                                                        })['username'], 'user1')
+#         # user2 doesn't exist yet and get created
+#         self.assertEqual(self.auth.get_or_create_user({'email': 'user2@test.com',
+#                                                        'username': 'user2'})['username'], 'user2')
+#         # user3 for corner case
+#         self.assertEqual(self.auth.get_or_create_user({'first_name': 'Omer',
+#                                                        'last_name': 'Simpson',
+#                                                        'email': 'user3@test.com',
+#                                                        'registration_id': 'user3',
+#                                                        'username': 'user3'})['username'], 'user3')
+#         # False case
+#         self.assertEqual(self.auth.get_or_create_user({'email': ''}), None)
+#         self.db.auth_user.truncate()
+#         self.db.commit()
+#
+#     def test_login_bare(self):
+#         # The following test case should succeed but failed as I never received the user record but False
+#         self.auth.login_bare(username='bart@simpson.com', password='bart_password')
+#         self.assertTrue(self.auth.is_logged_in())
+#         # Failing login because bad_password
+#         self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+#         self.db.auth_user.truncate()
+#
+#     def test_register_bare(self):
+#         # corner case empty register call register_bare without args
+#         self.assertRaises(ValueError, self.auth.register_bare)
+#         # failing register_bare user already exist
+#         self.assertEqual(self.auth.register_bare(username='bart', password='wrong_password'), False)
+#         # successful register_bare
+#         self.assertEqual(self.auth.register_bare(username='user2',
+#                                                  email='user2@test.com',
+#                                                  password='password_123')['username'], 'user2')
+#         # raise ValueError
+#         self.assertRaises(ValueError, self.auth.register_bare,
+#                           **dict(wrong_field_name='user3', password='password_123'))
+#         # raise ValueError wrong email
+#         self.assertRaises(ValueError, self.auth.register_bare,
+#                           **dict(email='user4@', password='password_123'))
+#         self.db.auth_user.truncate()
+#         self.db.commit()
+#
+#     def test_bulk_register(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         self.auth.settings.bulk_register_enabled = True
+#         bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
+#         self.assertTrue('name="_formkey"' in bulk_register_form)
+#
+#     def test_change_password(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         change_password_form = getattr(self.auth, 'change_password')().xml()
+#         self.assertTrue('name="_formkey"' in change_password_form)
+#
+#     def test_profile(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         profile_form = getattr(self.auth, 'profile')().xml()
+#         self.assertTrue('name="_formkey"' in profile_form)
+#
+#     # def test_impersonate(self):
+#     #     # Create a user to be impersonated
+#     #     self.auth.get_or_create_user(dict(first_name='Omer',
+#     #                                       last_name='Simpson',
+#     #                                       username='omer',
+#     #                                       email='omer@test.com',
+#     #                                       password='password_omer',
+#     #                                       registration_key='',
+#     #                                       registration_id=''))
+#     #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+#     #     self.auth.add_group('impersonate')
+#     #     self.auth.add_membership(user_id=1,
+#     #                              group_id=self.db(self.db.auth_user.username == 'bart'
+#     #                                               ).select(self.db.auth_user.id).first().id)
+#     #     self.auth.add_permission(group_id=self.db(self.db.auth_group.role == 'impersonate'
+#     #                                               ).select(self.db.auth_group.id).first().id,
+#     #                              name='impersonate',
+#     #                              table_name='auth_user',
+#     #                              record_id=0)
+#     #     # Bart login
+#     #     self.auth.login_bare(username='bart', password='bart_password')
+#     #     self.assertTrue(self.auth.is_logged_in())
+#     #     # Bart impersonate Omer
+#     #     omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
+#     #     impersonate_form = self.auth.impersonate(user_id=omer_id)
+#     #     self.assertTrue(self.auth.is_impersonating())
+#     #     self.assertEqual(impersonate_form, 'test')
+#
+#     # def test_impersonate(self):
+#     #     request = Request(env={})
+#     #     request.application = 'a'
+#     #     request.controller = 'c'
+#     #     request.function = 'f'
+#     #     request.folder = 'applications/admin'
+#     #     response = Response()
+#     #     session = Session()
+#     #     T = translator('', 'en')
+#     #     session.connect(request, response)
+#     #     from gluon.globals import current
+#     #     current.request = request
+#     #     current.response = response
+#     #     current.session = session
+#     #     current.T = T
+#     #     db = DAL(DEFAULT_URI, check_reserved=['all'])
+#     #     auth = Auth(db)
+#     #     auth.define_tables(username=True, signature=False)
+#     #     db.define_table('t0', Field('tt'), auth.signature)
+#     #     auth.enable_record_versioning(db)
+#     #     # Create a user
+#     #     auth.get_or_create_user(dict(first_name='Bart',
+#     #                                       last_name='Simpson',
+#     #                                       username='bart',
+#     #                                       email='bart@simpson.com',
+#     #                                       password='bart_password',
+#     #                                       registration_key='bart',
+#     #                                       registration_id=''
+#     #                                       ))
+#     #     # Create a user to be impersonated
+#     #     auth.get_or_create_user(dict(first_name='Omer',
+#     #                                       last_name='Simpson',
+#     #                                       username='omer',
+#     #                                       email='omer@test.com',
+#     #                                       password='password_omer',
+#     #                                       registration_key='',
+#     #                                       registration_id=''))
+#     #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+#     #     auth.add_group('impersonate')
+#     #     auth.add_membership(user_id=1,
+#     #                              group_id=db(db.auth_user.username == 'bart'
+#     #                                               ).select(db.auth_user.id).first().id)
+#     #     auth.add_permission(group_id=db(db.auth_group.role == 'impersonate'
+#     #                                               ).select(db.auth_group.id).first().id,
+#     #                              name='impersonate',
+#     #                              table_name='auth_user',
+#     #                              record_id=0)
+#     #     # Bart login
+#     #     auth.login_bare(username='bart', password='bart_password')
+#     #     # Bart impersonate Omer
+#     #     omer_id = db(db.auth_user.username == 'omer').select(db.auth_user.id).first().id
+#     #     impersonate_form = auth.impersonate(user_id=omer_id)
+#     #     self.assertTrue(auth.is_impersonating())
+#     #     self.assertEqual(impersonate_form, 'test')
 class TestAuth(unittest.TestCase):
-
     def setUp(self):
-        request = Request(env={})
-        request.application = 'a'
-        request.controller = 'c'
-        request.function = 'f'
-        request.folder = 'applications/admin'
-        response = Response()
-        session = Session()
+        self.request = Request(env={})
+        self.request.application = 'a'
+        self.request.controller = 'c'
+        self.request.function = 'f'
+        self.request.folder = 'applications/admin'
+        self.response = Response()
+        self.session = Session()
         T = translator('', 'en')
-        session.connect(request, response)
+        self.session.connect(self.request, self.response)
         from gluon.globals import current
-        current.request = request
-        current.response = response
-        current.session = session
-        current.T = T
+        self.current = current
+        self.current.request = self.request
+        self.current.response = self.response
+        self.current.session = self.session
+        self.current.T = T
         self.db = DAL(DEFAULT_URI, check_reserved=['all'])
         self.auth = Auth(self.db)
         self.auth.define_tables(username=True, signature=False)
@@ -259,7 +479,10 @@ class TestAuth(unittest.TestCase):
                                           password='bart_password',
                                           registration_key='bart',
                                           registration_id=''
-                                          ))
+                                          ),
+                                     login=False)
+        self.db.commit()
+        self.assertFalse(self.auth.is_logged_in())
         # self.auth.settings.registration_requires_verification = False
         # self.auth.settings.registration_requires_approval = False
 
@@ -271,23 +494,13 @@ class TestAuth(unittest.TestCase):
         self.assertTrue('auth_permission' in self.db)
         self.assertTrue('auth_event' in self.db)
 
-    def test_enable_record_versioning(self):
-        self.assertTrue('t0_archive' in self.db)
-
+    # Just calling many form functions
     def test_basic_blank_forms(self):
-        for f in ['login', 'retrieve_password',
-                  'retrieve_username',
-                  # 'register'  # register complain about : client_side=self.settings.client_side
-                  ]:
+        for f in ['login', 'retrieve_password', 'retrieve_username', 'register']:
             html_form = getattr(self.auth, f)().xml()
             self.assertTrue('name="_formkey"' in html_form)
 
-        # NOTE: Not sure it is the proper way to logout_bare() as there is not methods for that and auth.logout() failed
-        self.auth.logout_bare()
-        # self.assertTrue(self.auth.is_logged_in())
-
-        for f in ['logout', 'verify_email', 'reset_password',
-                  'change_password', 'profile', 'groups']:
+        for f in ['logout', 'verify_email', 'reset_password', 'change_password', 'profile', 'groups']:
             self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
 
         self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate)
@@ -302,6 +515,63 @@ class TestAuth(unittest.TestCase):
             pass
         return
 
+    def test_get_vars_next(self):
+        self.current.request.vars._next = 'next_test'
+        self.assertEqual(self.auth.get_vars_next(), 'next_test')
+
+    # TODO: def test_navbar(self):
+    # TODO: def test___get_migrate(self):
+
+    def test_enable_record_versioning(self):
+        self.assertTrue('t0_archive' in self.db)
+
+    # TODO: def test_define_signature(self):
+    # TODO: def test_define_signature(self):
+    # TODO: def test_define_table(self):
+
+    def test_log_event(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        # user logged in
+        self.auth.log_event(description='some_log_event_description_%(var1)s',
+                            vars={"var1": "var1"},
+                            origin='log_event_test_1')
+        rtn = self.db(self.db.auth_event.origin == 'log_event_test_1'
+                      ).select(*[self.db.auth_event[f]
+                                 for f in self.db.auth_event.fields if f not in ('id', 'time_stamp')]).first().as_dict()
+        self.assertEqual(set(rtn.items()), set({'origin': 'log_event_test_1',
+                                                'client_ip': None,
+                                                'user_id': bart_id,
+                                                'description': 'some_log_event_description_var1'}.items()))
+        # user not logged
+        self.auth.logout_bare()
+        self.auth.log_event(description='some_log_event_description_%(var2)s',
+                            vars={"var2": "var2"},
+                            origin='log_event_test_2')
+        rtn = self.db(self.db.auth_event.origin == 'log_event_test_2'
+                      ).select(*[self.db.auth_event[f]
+                                 for f in self.db.auth_event.fields if f not in ('id', 'time_stamp')]).first().as_dict()
+        self.assertEqual(set(rtn.items()), set({'origin': 'log_event_test_2',
+                                                'client_ip': None,
+                                                'user_id': None,
+                                                'description': 'some_log_event_description_var2'}.items()))
+        # no logging tests
+        self.auth.settings.logging_enabled = False
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.auth.log_event(description='some_log_event_description_%(var3)s',
+                            vars={"var3": "var3"},
+                            origin='log_event_test_3')
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        self.auth.settings.logging_enabled = True
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.auth.log_event(description=None,
+                            vars={"var4": "var4"},
+                            origin='log_event_test_4')
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # TODO: Corner case translated description...
+
     def test_get_or_create_user(self):
         self.db.auth_user.insert(email='user1@test.com', username='user1', password='password_123')
         self.db.commit()
@@ -324,13 +594,20 @@ class TestAuth(unittest.TestCase):
         self.db.auth_user.truncate()
         self.db.commit()
 
-    def test_login_bare(self):
-        # The following test case should succeed but failed as I never received the user record but False
-        self.auth.login_bare(username='bart@simpson.com', password='bart_password')
-        self.assertTrue(self.auth.is_logged_in())
-        # Failing login because bad_password
-        self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
-        self.db.auth_user.truncate()
+    # TODO: def test_basic(self):
+    # TODO: def test_login_user(self):
+    # TODO: def test__get_login_settings(self):
+
+    # login_bare() seems broken see my post on web2py-developpers
+    # commented for now
+    # def test_login_bare(self):
+    #     # The following test case should succeed but failed as I never received the user record but False
+    #     self.auth.login_bare(username='bart', password='bart_password')
+    #     self.assertTrue(self.auth.is_logged_in())
+    #     # Failing login because bad_password
+    #     self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+    #     self.auth.logout_bare()
+    #     self.db.auth_user.truncate()
 
     def test_register_bare(self):
         # corner case empty register call register_bare without args
@@ -350,104 +627,310 @@ class TestAuth(unittest.TestCase):
         self.db.auth_user.truncate()
         self.db.commit()
 
+    # TODO: def test_cas_login(self):
+    # TODO: def test_cas_validate(self):
+    # TODO: def test__reset_two_factor_auth(self):
+    # TODO: def test_when_is_logged_in_bypass_next_in_url(self):
+    # TODO: def test_login(self):
+    # TODO: def test_logout(self):
+
+    def test_logout_bare(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.logout_bare()
+        self.assertFalse(self.auth.is_logged_in())
+
+    # TODO: def test_register(self):
+
+    def test_is_logged_in(self):
+        self.auth.user = 'logged_in'
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.user = None
+        self.assertFalse(self.auth.is_logged_in())
+
+    # TODO: def test_verify_email(self):
+    # TODO: def test_retrieve_username(self):
+
+    def test_random_password(self):
+        # let just check that the function is callable
+        self.assertTrue(self.auth.random_password())
+
+    # TODO: def test_reset_password_deprecated(self):
+    # TODO: def test_confirm_registration(self):
+    # TODO: def test_email_registration(self):
+
     def test_bulk_register(self):
-        self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         self.auth.settings.bulk_register_enabled = True
         bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
         self.assertTrue('name="_formkey"' in bulk_register_form)
 
+    # TODO: def test_manage_tokens(self):
+    # TODO: def test_reset_password(self):
+    # TODO: def test_request_reset_password(self):
+    # TODO: def test_email_reset_password(self):
+    # TODO: def test_retrieve_password(self):
+
     def test_change_password(self):
-        self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         change_password_form = getattr(self.auth, 'change_password')().xml()
         self.assertTrue('name="_formkey"' in change_password_form)
 
     def test_profile(self):
-        self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         profile_form = getattr(self.auth, 'profile')().xml()
         self.assertTrue('name="_formkey"' in profile_form)
 
-    # def test_impersonate(self):
-    #     # Create a user to be impersonated
-    #     self.auth.get_or_create_user(dict(first_name='Omer',
-    #                                       last_name='Simpson',
-    #                                       username='omer',
-    #                                       email='omer@test.com',
-    #                                       password='password_omer',
-    #                                       registration_key='',
-    #                                       registration_id=''))
-    #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
-    #     self.auth.add_group('impersonate')
-    #     self.auth.add_membership(user_id=1,
-    #                              group_id=self.db(self.db.auth_user.username == 'bart'
-    #                                               ).select(self.db.auth_user.id).first().id)
-    #     self.auth.add_permission(group_id=self.db(self.db.auth_group.role == 'impersonate'
-    #                                               ).select(self.db.auth_group.id).first().id,
-    #                              name='impersonate',
-    #                              table_name='auth_user',
-    #                              record_id=0)
-    #     # Bart login
-    #     self.auth.login_bare(username='bart', password='bart_password')
-    #     self.assertTrue(self.auth.is_logged_in())
-    #     # Bart impersonate Omer
-    #     omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
-    #     impersonate_form = self.auth.impersonate(user_id=omer_id)
-    #     self.assertTrue(self.auth.is_impersonating())
-    #     self.assertEqual(impersonate_form, 'test')
+    # TODO: def test_run_login_onaccept(self):
+    # TODO: def test_jwt(self):
+    # TODO: def test_is_impersonating(self):
 
-    # def test_impersonate(self):
-    #     request = Request(env={})
-    #     request.application = 'a'
-    #     request.controller = 'c'
-    #     request.function = 'f'
-    #     request.folder = 'applications/admin'
-    #     response = Response()
-    #     session = Session()
-    #     T = translator('', 'en')
-    #     session.connect(request, response)
-    #     from gluon.globals import current
-    #     current.request = request
-    #     current.response = response
-    #     current.session = session
-    #     current.T = T
-    #     db = DAL(DEFAULT_URI, check_reserved=['all'])
-    #     auth = Auth(db)
-    #     auth.define_tables(username=True, signature=False)
-    #     db.define_table('t0', Field('tt'), auth.signature)
-    #     auth.enable_record_versioning(db)
-    #     # Create a user
-    #     auth.get_or_create_user(dict(first_name='Bart',
-    #                                       last_name='Simpson',
-    #                                       username='bart',
-    #                                       email='bart@simpson.com',
-    #                                       password='bart_password',
-    #                                       registration_key='bart',
-    #                                       registration_id=''
-    #                                       ))
-    #     # Create a user to be impersonated
-    #     auth.get_or_create_user(dict(first_name='Omer',
-    #                                       last_name='Simpson',
-    #                                       username='omer',
-    #                                       email='omer@test.com',
-    #                                       password='password_omer',
-    #                                       registration_key='',
-    #                                       registration_id=''))
-    #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
-    #     auth.add_group('impersonate')
-    #     auth.add_membership(user_id=1,
-    #                              group_id=db(db.auth_user.username == 'bart'
-    #                                               ).select(db.auth_user.id).first().id)
-    #     auth.add_permission(group_id=db(db.auth_group.role == 'impersonate'
-    #                                               ).select(db.auth_group.id).first().id,
-    #                              name='impersonate',
-    #                              table_name='auth_user',
-    #                              record_id=0)
-    #     # Bart login
-    #     auth.login_bare(username='bart', password='bart_password')
-    #     # Bart impersonate Omer
-    #     omer_id = db(db.auth_user.username == 'omer').select(db.auth_user.id).first().id
-    #     impersonate_form = auth.impersonate(user_id=omer_id)
-    #     self.assertTrue(auth.is_impersonating())
-    #     self.assertEqual(impersonate_form, 'test')
+    def test_impersonate(self):
+        # Create a user to be impersonated
+        self.auth.get_or_create_user(dict(first_name='Omer',
+                                          last_name='Simpson',
+                                          username='omer',
+                                          email='omer@test.com',
+                                          password='password_omer',
+                                          registration_key='',
+                                          registration_id=''),
+                                     login=False)
+        self.db.commit()
+        self.assertFalse(self.auth.is_logged_in())
+        # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+        group_id = self.auth.add_group('impersonate')
+        self.auth.add_membership(user_id=self.db(self.db.auth_user.username == 'bart'
+                                                 ).select(self.db.auth_user.id).first().id,
+                                 group_id=group_id)
+        self.auth.add_permission(group_id=group_id,
+                                 name='impersonate',
+                                 table_name='auth_user',
+                                 record_id=0)
+        # Bart login
+        # self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.is_logged_in())
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.assertEqual(self.auth.user_id, bart_id)
+        # self.session.auth = self.auth
+        # self.assertTrue(self.session.auth)
+
+        # basic impersonate() test that return a read form
+        self.assertEqual(self.auth.impersonate().xml(),
+                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="no_table_user_id__row"><td class="w2p_fl"><label class="" for="no_table_user_id" id="no_table_user_id__label">User Id: </label></td><td class="w2p_fw"><input class="integer" id="no_table_user_id" name="user_id" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
+        # bart impersonate itself
+        self.assertEqual(self.auth.impersonate(bart_id), None)
+        self.assertFalse(self.auth.is_impersonating())  # User shouldn't impersonate itself?
+        # Bart impersonate Omer
+        omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
+        impersonate_form = self.auth.impersonate(user_id=omer_id)
+        self.assertTrue(self.auth.is_impersonating())
+        self.assertEqual(self.auth.user_id, omer_id)  # we make it really sure
+        self.assertEqual(impersonate_form.xml(),
+                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="auth_user_id__row"><td class="w2p_fl"><label class="readonly" for="auth_user_id" id="auth_user_id__label">Id: </label></td><td class="w2p_fw"><span id="auth_user_id">2</span></td><td class="w2p_fc"></td></tr><tr id="auth_user_first_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_first_name" id="auth_user_first_name__label">First name: </label></td><td class="w2p_fw">Omer</td><td class="w2p_fc"></td></tr><tr id="auth_user_last_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_last_name" id="auth_user_last_name__label">Last name: </label></td><td class="w2p_fw">Simpson</td><td class="w2p_fc"></td></tr><tr id="auth_user_email__row"><td class="w2p_fl"><label class="readonly" for="auth_user_email" id="auth_user_email__label">E-mail: </label></td><td class="w2p_fw">omer@test.com</td><td class="w2p_fc"></td></tr><tr id="auth_user_username__row"><td class="w2p_fl"><label class="readonly" for="auth_user_username" id="auth_user_username__label">Username: </label></td><td class="w2p_fw">omer</td><td class="w2p_fc"></td></tr></table><div style="display:none;"><input name="id" type="hidden" value="2" /></div></form>')
+        self.auth.logout_bare()
+        # Failing impersonation
+        # User lacking impersonate membership
+        self.auth.login_user(self.db(self.db.auth_user.username == 'omer').select().first())  # bypass login_bare()
+        # self.assertTrue(self.auth.is_logged_in())  # For developing test
+        # self.assertFalse(self.auth.is_impersonating())  # For developing test
+        self.assertRaisesRegexp(HTTP, "403*", self.auth.impersonate, bart_id)
+        self.auth.logout_bare()
+        # Try impersonate a non existing user
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        # self.assertTrue(self.auth.is_logged_in())  # For developing test
+        # self.assertFalse(self.auth.is_impersonating())  # For developing test
+        self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate, 1000)  # user with id 1000 shouldn't exist
+        # Try impersonate user with id = 0 or '0' when bart impersonating omer
+        self.auth.impersonate(user_id=omer_id)
+        self.assertTrue(self.auth.is_impersonating())
+        self.assertEqual(self.auth.impersonate(user_id=0), None)
+
+    # TODO: def test_update_groups(self):
+
+    def test_groups(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertEqual(self.auth.groups().xml(),
+                         '<table><tr><td><h3>user_1(1)</h3></td></tr><tr><td><p></p></td></tr></table>')
+
+    def test_not_authorized(self):
+        self.current.request.ajax = 'facke_ajax_request'
+        self.assertRaisesRegexp(HTTP, "403*", self.auth.not_authorized)
+        self.current.request.ajax = None
+        self.assertEqual(self.auth.not_authorized(), self.auth.messages.access_denied)
+
+    def test_allows_jwt(self):
+        self.assertRaisesRegexp(HTTP, "400*", self.auth.allows_jwt)
+
+    # TODO: def test_requires(self):
+    # TODO: def test_requires_login(self):
+    # TODO: def test_requires_login_or_token(self):
+    # TODO: def test_requires_membership(self):
+    # TODO: def test_requires_permission(self):
+    # TODO: def test_requires_signature(self):
+
+    def test_add_group(self):
+        self.assertEqual(self.auth.add_group(role='a_group', description='a_group_role_description'),
+                         self.db(self.db.auth_group.role == 'a_group').select(self.db.auth_group.id).first().id)
+
+    def test_del_group(self):
+        bart_group_id = 1  # Should be group 1, 'user_1'
+        self.assertEqual(self.auth.del_group(group_id=bart_group_id), None)
+
+    def test_id_group(self):
+        self.assertEqual(self.auth.id_group(role='user_1'), 1)
+        # If role don't exist it return None
+        self.assertEqual(self.auth.id_group(role='non_existing_role_name'), None)
+
+    def test_user_group(self):
+        self.assertEqual(self.auth.user_group(user_id=1), 1)
+        # Bart should be user 1 and it unique group should be 1, 'user_1'
+
+    def test_user_group_role(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        user_group_role = 'user_%s' % self.db(self.db.auth_user.username == 'bart'
+                                              ).select(self.db.auth_user.id).first().id
+        self.assertEqual(self.auth.user_group_role(), user_group_role)
+        self.auth.logout_bare()
+        # with user_id args
+        self.assertEqual(self.auth.user_group_role(user_id=1), 'user_1')
+        # test None
+        self.auth.settings.create_user_groups = None
+        self.assertEqual(self.auth.user_group_role(user_id=1), None)
+
+    def test_has_membership(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.has_membership('user_1'))
+        self.assertFalse(self.auth.has_membership('user_555'))
+        self.assertTrue(self.auth.has_membership(group_id=1))
+        self.auth.logout_bare()
+        self.assertTrue(self.auth.has_membership(role='user_1', user_id=1))
+        self.assertTrue(self.auth.has_membership(group_id=1, user_id=1))
+        # check that event is logged
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.assertTrue(self.auth.has_membership(group_id=1, user_id=1))
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+
+    # Waiting guidance : https://github.com/web2py/web2py/issues/1300
+    # def test_add_membership(self):
+    #     self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+    #     # failing case
+    #     rtn = self.auth.add_membership('not_existing_role_name')
+    #     # self.assertEqual(rtn, 'test')
+    #     self.assertEqual(self.db(self.db.auth_group.role == 'not_existing_role_name').select().first(), 'test')
+
+    def test_del_membership(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        user_1_role_id = self.db(self.db.auth_membership.group_id == self.auth.id_group('user_1')
+                                 ).select(self.db.auth_membership.id).first().id
+        self.assertEqual(self.auth.del_membership('user_1'), user_1_role_id)
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # not logged in test case
+        group_id = self.auth.add_group('some_test_group')
+        membership_id = self.auth.add_membership('some_test_group')
+        self.assertEqual(self.auth.user_groups[group_id], 'some_test_group')
+        self.auth.logout_bare()
+        # not deleted
+        self.assertFalse(self.auth.del_membership('some_test_group'))
+        self.assertEqual(set(self.db.auth_membership(membership_id).as_dict().items()),
+                         set({'group_id': 2L, 'user_id': 1L, 'id': 2L}.items()))  # is not deleted
+        # deleted
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.assertTrue(self.auth.del_membership('some_test_group', user_id=bart_id))
+        self.assertEqual(self.db.auth_membership(membership_id), None)  # is really deleted
+
+    def test_has_permission(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                 name='some_permission',
+                                 table_name='auth_user',
+                                 record_id=0,
+                                 )
+        # True case
+        self.assertTrue(self.auth.has_permission(name='some_permission',
+                                                 table_name='auth_user',
+                                                 record_id=0,
+                                                 user_id=bart_id,
+                                                 group_id=self.auth.id_group('user_1')))
+        # False case
+        self.assertFalse(self.auth.has_permission(name='some_other_permission',
+                                                  table_name='auth_user',
+                                                  record_id=0,
+                                                  user_id=bart_id,
+                                                  group_id=self.auth.id_group('user_1')))
+
+    def test_add_permission(self):
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        permission_id = \
+            self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                     name='some_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # True case
+        permission_count = \
+            self.db(self.db.auth_permission.id == permission_id).count()
+        self.assertTrue(permission_count)
+        # False case
+        permission_count = \
+            self.db((self.db.auth_permission.group_id == self.auth.id_group('user_1')) &
+                    (self.db.auth_permission.name == 'no_permission') &
+                    (self.db.auth_permission.table_name == 'no_table') &
+                    (self.db.auth_permission.record_id == 0)).count()
+        self.assertFalse(permission_count)
+        # corner case
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        permission_id = \
+            self.auth.add_permission(group_id=0,
+                                     name='user_1_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        permission_name = \
+            self.db(self.db.auth_permission.id == permission_id).select(self.db.auth_permission.name).first().name
+        self.assertEqual(permission_name, 'user_1_permission')
+        # add an existing permission
+        permission_id =\
+            self.auth.add_permission(group_id=0,
+                                     name='user_1_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        self.assertTrue(permission_id)
+
+    def test_del_permission(self):
+        permission_id = \
+            self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                     name='del_permission_test',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.assertTrue(self.auth.del_permission(group_id=self.auth.id_group('user_1'),
+                                                 name='del_permission_test',
+                                                 table_name='auth_user',
+                                                 record_id=0,))
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # really deleted
+        permission_count = \
+            self.db(self.db.auth_permission.id == permission_id).count()
+        self.assertFalse(permission_count)
+
+    # TODO: def test_accessible_query(self):
+    # TODO: def test_archive(self):
+    # TODO: def test_wiki(self):
+    # TODO: def test_wikimenu(self):
+    # End Auth test
 
 
 # TODO: class TestCrud(unittest.TestCase):
@@ -469,8 +952,88 @@ class TestAuth(unittest.TestCase):
 # TODO: class TestConfig(unittest.TestCase):
 
 
-# TODO: class TestToolsFunctions(unittest.TestCase):
-# For all the tools.py functions
+class TestToolsFunctions(unittest.TestCase):
+    """
+    Test suite for all the tools.py functions
+    """
+    def test_prettydate(self):
+        # plain
+        now = datetime.datetime.now()
+        self.assertEqual(prettydate(d=now), 'now')
+        one_second = now - datetime.timedelta(seconds=1)
+        self.assertEqual(prettydate(d=one_second), '1 second ago')
+        more_than_one_second = now - datetime.timedelta(seconds=2)
+        self.assertEqual(prettydate(d=more_than_one_second), '2 seconds ago')
+        one_minute = now - datetime.timedelta(seconds=60)
+        self.assertEqual(prettydate(d=one_minute), '1 minute ago')
+        more_than_one_minute = now - datetime.timedelta(seconds=61)
+        self.assertEqual(prettydate(d=more_than_one_minute), '1 minute ago')
+        two_minutes = now - datetime.timedelta(seconds=120)
+        self.assertEqual(prettydate(d=two_minutes), '2 minutes ago')
+        more_than_two_minutes = now - datetime.timedelta(seconds=121)
+        self.assertEqual(prettydate(d=more_than_two_minutes), '2 minutes ago')
+        one_hour = now - datetime.timedelta(seconds=60 * 60)
+        self.assertEqual(prettydate(d=one_hour), '1 hour ago')
+        more_than_one_hour = now - datetime.timedelta(seconds=3601)
+        self.assertEqual(prettydate(d=more_than_one_hour), '1 hour ago')
+        two_hours = now - datetime.timedelta(seconds=2 * 60 * 60)
+        self.assertEqual(prettydate(d=two_hours), '2 hours ago')
+        more_than_two_hours = now - datetime.timedelta(seconds=2 * 60 * 60 + 1)
+        self.assertEqual(prettydate(d=more_than_two_hours), '2 hours ago')
+        one_day = now - datetime.timedelta(days=1)
+        self.assertEqual(prettydate(d=one_day), '1 day ago')
+        more_than_one_day = now - datetime.timedelta(days=2)
+        self.assertEqual(prettydate(d=more_than_one_day), '2 days ago')
+        one_week = now - datetime.timedelta(days=7)
+        self.assertEqual(prettydate(d=one_week), '1 week ago')
+        more_than_one_week = now - datetime.timedelta(days=8)
+        self.assertEqual(prettydate(d=more_than_one_week), '1 week ago')
+        two_weeks = now - datetime.timedelta(days=14)
+        self.assertEqual(prettydate(d=two_weeks), '2 weeks ago')
+        more_than_two_weeks = now - datetime.timedelta(days=15)
+        self.assertEqual(prettydate(d=more_than_two_weeks), '2 weeks ago')
+        three_weeks = now - datetime.timedelta(days=21)
+        self.assertEqual(prettydate(d=three_weeks), '3 weeks ago')
+        one_month = now - datetime.timedelta(days=27)
+        self.assertEqual(prettydate(d=one_month), '1 month ago')
+        more_than_one_month = now - datetime.timedelta(days=28)
+        self.assertEqual(prettydate(d=more_than_one_month), '1 month ago')
+        two_months = now - datetime.timedelta(days=60)
+        self.assertEqual(prettydate(d=two_months), '2 months ago')
+        three_months = now - datetime.timedelta(days=90)
+        self.assertEqual(prettydate(d=three_months), '3 months ago')
+        one_year = now - datetime.timedelta(days=365)
+        self.assertEqual(prettydate(d=one_year), '1 year ago')
+        more_than_one_year = now - datetime.timedelta(days=366)
+        self.assertEqual(prettydate(d=more_than_one_year), '1 year ago')
+        two_years = now - datetime.timedelta(days=2 * 365)
+        self.assertEqual(prettydate(d=two_years), '2 years ago')
+        more_than_two_years = now - datetime.timedelta(days=2 * 365 + 1)
+        self.assertEqual(prettydate(d=more_than_two_years), '2 years ago')
+        # date()
+        d = now.date()
+        self.assertEqual(prettydate(d=d), 'now')
+        one_day = now.date() - datetime.timedelta(days=1)
+        self.assertEqual(prettydate(d=one_day), '1 day ago')
+        tow_days = now.date() - datetime.timedelta(days=2)
+        self.assertEqual(prettydate(d=tow_days), '2 days ago')
+        # from now
+        # from now is picky depending of the execution time, so we can't use sharp value like 1 second or 1 day
+        in_one_minute = now - datetime.timedelta(seconds=-65)
+        self.assertEqual(prettydate(d=in_one_minute), '1 minute from now')
+        in_twenty_three_hours = now - datetime.timedelta(hours=-23.5)
+        self.assertEqual(prettydate(d=in_twenty_three_hours), '23 hours from now')
+        in_one_year = now - datetime.timedelta(days=-366)
+        self.assertEqual(prettydate(d=in_one_year), '1 year from now')
+        # utc=True
+        now = datetime.datetime.utcnow()
+        self.assertEqual(prettydate(d=now, utc=True), 'now')
+        one_second = now - datetime.timedelta(seconds=1)
+        self.assertEqual(prettydate(d=one_second, utc=True), '1 second ago')
+        # not d or invalid date
+        self.assertEqual(prettydate(d=None), '')
+        self.assertEqual(prettydate(d='invalid_date'), '[invalid date]')
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_validators.py

@@ -178,9 +178,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (('max', 'john'), None))
         rtn = IS_IN_SET(['max', 'john'], multiple=True)(('bill', 'john'))
         self.assertEqual(rtn, (('bill', 'john'), 'Value not allowed'))
-        rtn = IS_IN_SET(('id1','id2'), ['first label','second label'])('id1') # Traditional way
+        rtn = IS_IN_SET(('id1', 'id2'), ['first label', 'second label'])('id1')  # Traditional way
         self.assertEqual(rtn, ('id1', None))
-        rtn = IS_IN_SET({'id1':'first label', 'id2':'second label'})('id1')
+        rtn = IS_IN_SET({'id1': 'first label', 'id2': 'second label'})('id1')
         self.assertEqual(rtn, ('id1', None))
         rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)(None)
         self.assertEqual(rtn, ([], None))
@@ -188,14 +188,14 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ([], None))
         rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)('id1')
         self.assertEqual(rtn, (['id1'], None))
-        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=(1,2))(None)
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=(1, 2))(None)
         self.assertEqual(rtn, ([], 'oops'))
         import itertools
-        rtn = IS_IN_SET(itertools.chain(['1','3','5'],['2','4','6']))('1')
+        rtn = IS_IN_SET(itertools.chain(['1', '3', '5'], ['2', '4', '6']))('1')
         self.assertEqual(rtn, ('1', None))
-        rtn = IS_IN_SET([('id1','first label'), ('id2','second label')])('id1') # Redundant way
+        rtn = IS_IN_SET([('id1', 'first label'), ('id2', 'second label')])('id1')  # Redundant way
         self.assertEqual(rtn, ('id1', None))
-        rtn = IS_IN_SET([('id1','first label'), ('id2','second label')]).options(zero=False)
+        rtn = IS_IN_SET([('id1', 'first label'), ('id2', 'second label')]).options(zero=False)
         self.assertEqual(rtn, [('id1', 'first label'), ('id2', 'second label')])
         rtn = IS_IN_SET(['id1', 'id2']).options(zero=False)
         self.assertEqual(rtn, [('id1', 'id1'), ('id2', 'id2')])
@@ -210,47 +210,79 @@ class TestValidators(unittest.TestCase):
         costanza_id = db.person.insert(name='costanza')
         rtn = IS_IN_DB(db, 'person.id', '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
+        rtn = IS_IN_DB(db, 'person.name', '%(name)s')('george')
+        self.assertEqual(rtn, ('george', None))
         rtn = IS_IN_DB(db, db.person, '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
         rtn = IS_IN_DB(db(db.person.id > 0), db.person, '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
-        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')(george_id+costanza_id)
-        self.assertEqual(rtn, (george_id+costanza_id, 'oops'))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')(george_id + costanza_id)
+        self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
         rtn = IS_IN_DB(db, db.person.id, '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
-        rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops')(george_id+costanza_id)
-        self.assertEqual(rtn, (george_id+costanza_id, 'oops'))
-        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True)([george_id,costanza_id])
-        self.assertEqual(rtn, ([george_id,costanza_id], None))
+        rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops')(george_id + costanza_id)
+        self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True)([george_id, costanza_id])
+        self.assertEqual(rtn, ([george_id, costanza_id], None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, error_message='oops')("I'm not even an id")
+        self.assertEqual(rtn, (["I'm not even an id"], 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, delimiter=',')('%d,%d' % (george_id, costanza_id))
-        self.assertEqual(rtn, ( ('%d,%d' % (george_id, costanza_id)).split(','), None))
-        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1,3), delimiter=',')('%d,%d' % (george_id, costanza_id))
-        self.assertEqual(rtn, ( ('%d,%d' % (george_id, costanza_id)).split(','), None))
-        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1,2), delimiter=',', error_message='oops')('%d,%d' % (george_id, costanza_id))
-        self.assertEqual(rtn, ( ('%d,%d' % (george_id, costanza_id)).split(','), 'oops'))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 3), delimiter=',')('%d,%d' % (george_id, costanza_id))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 2), delimiter=',', error_message='oops')('%d,%d' % (george_id, costanza_id))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), 'oops'))
         rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops').options(zero=False)
         self.assertEqual(sorted(rtn), [('%d' % george_id, 'george'), ('%d' % costanza_id, 'costanza')])
         rtn = IS_IN_DB(db, db.person.id, db.person.name, error_message='oops', sort=True).options(zero=True)
         self.assertEqual(rtn, [('', ''), ('%d' % costanza_id, 'costanza'), ('%d' % george_id, 'george')])
+        # Test using the set it made for options
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops')
+        vldtr.options()
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', multiple=True)
+        vldtr.options()
+        rtn = vldtr(['george', 'costanza'])
+        self.assertEqual(rtn, (['george', 'costanza'], None))
         # Test it works with self reference
         db.define_table('category',
-            Field('parent_id', 'reference category', requires=IS_EMPTY_OR(IS_IN_DB(db, 'category.id', '%(name)s'))),
-            Field('name')
-        )
+                        Field('parent_id', 'reference category', requires=IS_EMPTY_OR(IS_IN_DB(db, 'category.id', '%(name)s'))),
+                        Field('name')
+                        )
         ret = db.category.validate_and_insert(name='seinfeld')
         self.assertFalse(list(ret.errors))
         ret = db.category.validate_and_insert(name='characters', parent_id=ret.id)
         self.assertFalse(list(ret.errors))
         rtn = IS_IN_DB(db, 'category.id', '%(name)s')(ret.id)
         self.assertEqual(rtn, (ret.id, None))
+        # Test _and
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', _and=IS_LENGTH(maxsize=7, error_message='bad'))
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', 'bad'))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr.options() # test theset with _and
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        # Test auto_add
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', auto_add=True)('jerry')
+        self.assertEqual(rtn, (3, None))
         db.person.drop()
         db.category.drop()
 
     def test_IS_NOT_IN_DB(self):
         from gluon.dal import DAL, Field
         db = DAL('sqlite:memory')
-        db.define_table('person', Field('name'))
+        db.define_table('person', Field('name'), Field('nickname'))
         db.person.insert(name='george')
+        db.person.insert(name='costanza', nickname='T Bone')
         rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops')('george')
         self.assertEqual(rtn, ('george', 'oops'))
         rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops', allowed_override=['george'])('george')
@@ -261,6 +293,17 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('jerry', None))
         rtn = IS_NOT_IN_DB(db, 'person.name')(u'jerry')
         self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db(db.person.id > 0), 'person.name')(u'jerry')
+        self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db, db.person, error_message='oops')(1)
+        self.assertEqual(rtn, ('1', 'oops'))
+        vldtr = IS_NOT_IN_DB(db, 'person.name', error_message='oops')
+        vldtr.set_self_id({'name': 'costanza', 'nickname': 'T Bone'})
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', 'oops'))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', None))
+
         db.person.drop()
 
     def test_IS_INT_IN_RANGE(self):
@@ -413,7 +456,7 @@ class TestValidators(unittest.TestCase):
     def test_IS_ALPHANUMERIC(self):
         rtn = IS_ALPHANUMERIC()('1')
         self.assertEqual(rtn, ('1', None))
-        rtn =  IS_ALPHANUMERIC()('')
+        rtn = IS_ALPHANUMERIC()('')
         self.assertEqual(rtn, ('', None))
         rtn = IS_ALPHANUMERIC()('A_a')
         self.assertEqual(rtn, ('A_a', None))
@@ -421,62 +464,62 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('!', 'Enter only letters, numbers, and underscore'))
 
     def test_IS_EMAIL(self):
-        rtn =  IS_EMAIL()('a@b.com')
+        rtn = IS_EMAIL()('a@b.com')
         self.assertEqual(rtn, ('a@b.com', None))
-        rtn =  IS_EMAIL()('abc@def.com')
+        rtn = IS_EMAIL()('abc@def.com')
         self.assertEqual(rtn, ('abc@def.com', None))
-        rtn =  IS_EMAIL()('abc@3def.com')
+        rtn = IS_EMAIL()('abc@3def.com')
         self.assertEqual(rtn, ('abc@3def.com', None))
-        rtn =  IS_EMAIL()('abc@def.us')
+        rtn = IS_EMAIL()('abc@def.us')
         self.assertEqual(rtn, ('abc@def.us', None))
-        rtn =  IS_EMAIL()('abc@d_-f.us')
+        rtn = IS_EMAIL()('abc@d_-f.us')
         self.assertEqual(rtn, ('abc@d_-f.us', None))
-        rtn =  IS_EMAIL()('@def.com')           # missing name
+        rtn = IS_EMAIL()('@def.com')           # missing name
         self.assertEqual(rtn, ('@def.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('"abc@def".com')      # quoted name
+        rtn = IS_EMAIL()('"abc@def".com')      # quoted name
         self.assertEqual(rtn, ('"abc@def".com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc+def.com')        # no @
+        rtn = IS_EMAIL()('abc+def.com')        # no @
         self.assertEqual(rtn, ('abc+def.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def.x')          # one-char TLD
+        rtn = IS_EMAIL()('abc@def.x')          # one-char TLD
         self.assertEqual(rtn, ('abc@def.x', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def.12')         # numeric TLD
+        rtn = IS_EMAIL()('abc@def.12')         # numeric TLD
         self.assertEqual(rtn, ('abc@def.12', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def..com')       # double-dot in domain
+        rtn = IS_EMAIL()('abc@def..com')       # double-dot in domain
         self.assertEqual(rtn, ('abc@def..com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@.def.com')       # dot starts domain
+        rtn = IS_EMAIL()('abc@.def.com')       # dot starts domain
         self.assertEqual(rtn, ('abc@.def.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def.c_m')        # underscore in TLD
+        rtn = IS_EMAIL()('abc@def.c_m')        # underscore in TLD
         self.assertEqual(rtn, ('abc@def.c_m', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('NotAnEmail')         # missing @
+        rtn = IS_EMAIL()('NotAnEmail')         # missing @
         self.assertEqual(rtn, ('NotAnEmail', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@NotAnEmail')     # missing TLD
+        rtn = IS_EMAIL()('abc@NotAnEmail')     # missing TLD
         self.assertEqual(rtn, ('abc@NotAnEmail', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('customer/department@example.com')
+        rtn = IS_EMAIL()('customer/department@example.com')
         self.assertEqual(rtn, ('customer/department@example.com', None))
-        rtn =  IS_EMAIL()('$A12345@example.com')
+        rtn = IS_EMAIL()('$A12345@example.com')
         self.assertEqual(rtn, ('$A12345@example.com', None))
-        rtn =  IS_EMAIL()('!def!xyz%abc@example.com')
+        rtn = IS_EMAIL()('!def!xyz%abc@example.com')
         self.assertEqual(rtn, ('!def!xyz%abc@example.com', None))
-        rtn =  IS_EMAIL()('_Yosemite.Sam@example.com')
+        rtn = IS_EMAIL()('_Yosemite.Sam@example.com')
         self.assertEqual(rtn, ('_Yosemite.Sam@example.com', None))
-        rtn =  IS_EMAIL()('~@example.com')
+        rtn = IS_EMAIL()('~@example.com')
         self.assertEqual(rtn, ('~@example.com', None))
-        rtn =  IS_EMAIL()('.wooly@example.com')       # dot starts name
+        rtn = IS_EMAIL()('.wooly@example.com')       # dot starts name
         self.assertEqual(rtn, ('.wooly@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('wo..oly@example.com')      # adjacent dots in name
+        rtn = IS_EMAIL()('wo..oly@example.com')      # adjacent dots in name
         self.assertEqual(rtn, ('wo..oly@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('pootietang.@example.com')  # dot ends name
+        rtn = IS_EMAIL()('pootietang.@example.com')  # dot ends name
         self.assertEqual(rtn, ('pootietang.@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('.@example.com')            # name is bare dot
+        rtn = IS_EMAIL()('.@example.com')            # name is bare dot
         self.assertEqual(rtn, ('.@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('Ima.Fool@example.com')
+        rtn = IS_EMAIL()('Ima.Fool@example.com')
         self.assertEqual(rtn, ('Ima.Fool@example.com', None))
-        rtn =  IS_EMAIL()('Ima Fool@example.com')     # space in name
+        rtn = IS_EMAIL()('Ima Fool@example.com')     # space in name
         self.assertEqual(rtn, ('Ima Fool@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('localguy@localhost')       # localhost as domain
+        rtn = IS_EMAIL()('localguy@localhost')       # localhost as domain
         self.assertEqual(rtn, ('localguy@localhost', None))
         # test for banned
-        rtn =  IS_EMAIL(banned='^.*\.com(|\..*)$')('localguy@localhost')       # localhost as domain
+        rtn = IS_EMAIL(banned='^.*\.com(|\..*)$')('localguy@localhost')       # localhost as domain
         self.assertEqual(rtn, ('localguy@localhost', None))
         rtn = IS_EMAIL(banned='^.*\.com(|\..*)$')('abc@example.com')
         self.assertEqual(rtn, ('abc@example.com', 'Enter a valid email address'))
@@ -485,6 +528,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('localguy@localhost', 'Enter a valid email address'))
         rtn = IS_EMAIL(forced='^.*\.edu(|\..*)$')('localguy@example.edu')
         self.assertEqual(rtn, ('localguy@example.edu', None))
+        # test for not a string at all
+        rtn = IS_EMAIL(error_message='oops')(42)
+        self.assertEqual(rtn, (42, 'oops'))
 
     def test_IS_LIST_OF_EMAILS(self):
         emails = ['localguy@localhost', '_Yosemite.Sam@example.com']
@@ -543,16 +589,16 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('', 'Enter time as hh:mm:ss (seconds, am, pm optional)'))
 
     def test_IS_DATE(self):
-        v = IS_DATE(format="%m/%d/%Y",error_message="oops")
+        v = IS_DATE(format="%m/%d/%Y", error_message="oops")
         rtn = v('03/03/2008')
         self.assertEqual(rtn, (datetime.date(2008, 3, 3), None))
         rtn = v('31/03/2008')
         self.assertEqual(rtn, ('31/03/2008', 'oops'))
-        rtn = IS_DATE(format="%m/%d/%Y",error_message="oops").formatter(datetime.date(1834, 12, 14))
+        rtn = IS_DATE(format="%m/%d/%Y", error_message="oops").formatter(datetime.date(1834, 12, 14))
         self.assertEqual(rtn, '12/14/1834')
 
     def test_IS_DATETIME(self):
-        v = IS_DATETIME(format="%m/%d/%Y %H:%M",error_message="oops")
+        v = IS_DATETIME(format="%m/%d/%Y %H:%M", error_message="oops")
         rtn = v('03/03/2008 12:40')
         self.assertEqual(rtn, (datetime.datetime(2008, 3, 3, 12, 40), None))
         rtn = v('31/03/2008 29:40')
@@ -560,16 +606,20 @@ class TestValidators(unittest.TestCase):
         # Test timezone is removed and value is properly converted
         #
         # https://github.com/web2py/web2py/issues/1094
+
         class DummyTimezone(datetime.tzinfo):
 
             ONE = datetime.timedelta(hours=1)
 
             def utcoffset(self, dt):
                 return DummyTimezone.ONE
+
             def tzname(self, dt):
                 return "UTC+1"
+
             def dst(self, dt):
                 return DummyTimezone.ONE
+
             def localize(self, dt, is_dst=False):
                 return dt.replace(tzinfo=self)
         v = IS_DATETIME(format="%Y-%m-%d %H:%M", error_message="oops", timezone=DummyTimezone())
@@ -577,65 +627,65 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (datetime.datetime(1982, 12, 14, 7, 0), None))
 
     def test_IS_DATE_IN_RANGE(self):
-        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008,1,1),
-                             maximum=datetime.date(2009,12,31),
-                             format="%m/%d/%Y",error_message="oops")
+        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008, 1, 1),
+                             maximum=datetime.date(2009, 12, 31),
+                             format="%m/%d/%Y", error_message="oops")
 
         rtn = v('03/03/2008')
         self.assertEqual(rtn, (datetime.date(2008, 3, 3), None))
         rtn = v('03/03/2010')
         self.assertEqual(rtn, ('03/03/2010', 'oops'))
-        rtn = v(datetime.date(2008,3,3))
+        rtn = v(datetime.date(2008, 3, 3))
         self.assertEqual(rtn, (datetime.date(2008, 3, 3), None))
-        rtn =  v(datetime.date(2010,3,3))
+        rtn = v(datetime.date(2010, 3, 3))
         self.assertEqual(rtn, (datetime.date(2010, 3, 3), 'oops'))
-        v = IS_DATE_IN_RANGE(maximum=datetime.date(2009,12,31),
-            format="%m/%d/%Y")
+        v = IS_DATE_IN_RANGE(maximum=datetime.date(2009, 12, 31),
+                             format="%m/%d/%Y")
         rtn = v('03/03/2010')
         self.assertEqual(rtn, ('03/03/2010', 'Enter date on or before 12/31/2009'))
-        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008,1,1),
-            format="%m/%d/%Y")
+        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008, 1, 1),
+                             format="%m/%d/%Y")
         rtn = v('03/03/2007')
         self.assertEqual(rtn, ('03/03/2007', 'Enter date on or after 01/01/2008'))
-        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008,1,1),
-            maximum=datetime.date(2009,12,31),
-            format="%m/%d/%Y")
+        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008, 1, 1),
+                             maximum=datetime.date(2009, 12, 31),
+                             format="%m/%d/%Y")
         rtn = v('03/03/2007')
         self.assertEqual(rtn, ('03/03/2007', 'Enter date in range 01/01/2008 12/31/2009'))
 
     def test_IS_DATETIME_IN_RANGE(self):
         v = IS_DATETIME_IN_RANGE(
-            minimum=datetime.datetime(2008,1,1,12,20),
-            maximum=datetime.datetime(2009,12,31,12,20),
-            format="%m/%d/%Y %H:%M",error_message="oops")
+            minimum=datetime.datetime(2008, 1, 1, 12, 20),
+            maximum=datetime.datetime(2009, 12, 31, 12, 20),
+            format="%m/%d/%Y %H:%M", error_message="oops")
         rtn = v('03/03/2008 12:40')
         self.assertEqual(rtn, (datetime.datetime(2008, 3, 3, 12, 40), None))
         rtn = v('03/03/2010 10:34')
         self.assertEqual(rtn, ('03/03/2010 10:34', 'oops'))
-        rtn =  v(datetime.datetime(2008,3,3,0,0))
+        rtn = v(datetime.datetime(2008, 3, 3, 0, 0))
         self.assertEqual(rtn, (datetime.datetime(2008, 3, 3, 0, 0), None))
-        rtn =  v(datetime.datetime(2010,3,3,0,0))
+        rtn = v(datetime.datetime(2010, 3, 3, 0, 0))
         self.assertEqual(rtn, (datetime.datetime(2010, 3, 3, 0, 0), 'oops'))
-        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009,12,31,12,20),
-            format='%m/%d/%Y %H:%M:%S')
+        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009, 12, 31, 12, 20),
+                                 format='%m/%d/%Y %H:%M:%S')
         rtn = v('03/03/2010 12:20:00')
         self.assertEqual(rtn, ('03/03/2010 12:20:00', 'Enter date and time on or before 12/31/2009 12:20:00'))
-        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008,1,1,12,20),
-            format='%m/%d/%Y %H:%M:%S')
+        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008, 1, 1, 12, 20),
+                                 format='%m/%d/%Y %H:%M:%S')
         rtn = v('03/03/2007 12:20:00')
         self.assertEqual(rtn, ('03/03/2007 12:20:00', 'Enter date and time on or after 01/01/2008 12:20:00'))
-        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008,1,1,12,20),
-            maximum=datetime.datetime(2009,12,31,12,20),
-            format='%m/%d/%Y %H:%M:%S')
+        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008, 1, 1, 12, 20),
+                                 maximum=datetime.datetime(2009, 12, 31, 12, 20),
+                                 format='%m/%d/%Y %H:%M:%S')
         rtn = v('03/03/2007 12:20:00')
         self.assertEqual(rtn, ('03/03/2007 12:20:00', 'Enter date and time in range 01/01/2008 12:20:00 12/31/2009 12:20:00'))
-        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009,12,31,12,20),
-            format='%Y-%m-%d %H:%M:%S', error_message='oops')
+        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009, 12, 31, 12, 20),
+                                 format='%Y-%m-%d %H:%M:%S', error_message='oops')
         rtn = v('clearly not a date')
         self.assertEqual(rtn, ('clearly not a date', 'oops'))
 
     def test_IS_LIST_OF(self):
-        values = [0,1,2,3,4]
+        values = [0, 1, 2, 3, 4]
         rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10))(values)
         self.assertEqual(rtn, (values, None))
         values.append(11)
@@ -643,9 +693,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (values, 'Enter an integer between 0 and 9'))
         rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10))(1)
         self.assertEqual(rtn, ([1], None))
-        rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10), minimum=10)([1,2])
+        rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10), minimum=10)([1, 2])
         self.assertEqual(rtn, ([1, 2], 'Enter between 10 and 100 values'))
-        rtn =  IS_LIST_OF(IS_INT_IN_RANGE(0, 10), maximum=2)([1,2,3])
+        rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10), maximum=2)([1, 2, 3])
         self.assertEqual(rtn, ([1, 2, 3], 'Enter between 0 and 2 values'))
         # regression test for issue 742
         rtn = IS_LIST_OF(minimum=1)('')
@@ -704,69 +754,69 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('a bc', 'Must be slug'))
 
     def test_ANY_OF(self):
-        rtn = ANY_OF([IS_EMAIL(),IS_ALPHANUMERIC()])('a@b.co')
+        rtn = ANY_OF([IS_EMAIL(), IS_ALPHANUMERIC()])('a@b.co')
         self.assertEqual(rtn, ('a@b.co', None))
-        rtn = ANY_OF([IS_EMAIL(),IS_ALPHANUMERIC()])('abco')
+        rtn = ANY_OF([IS_EMAIL(), IS_ALPHANUMERIC()])('abco')
         self.assertEqual(rtn, ('abco', None))
-        rtn = ANY_OF([IS_EMAIL(),IS_ALPHANUMERIC()])('@ab.co')
+        rtn = ANY_OF([IS_EMAIL(), IS_ALPHANUMERIC()])('@ab.co')
         self.assertEqual(rtn, ('@ab.co', 'Enter only letters, numbers, and underscore'))
-        rtn = ANY_OF([IS_ALPHANUMERIC(),IS_EMAIL()])('@ab.co')
+        rtn = ANY_OF([IS_ALPHANUMERIC(), IS_EMAIL()])('@ab.co')
         self.assertEqual(rtn, ('@ab.co', 'Enter a valid email address'))
-        rtn = ANY_OF([IS_DATE(),IS_EMAIL()])('a@b.co')
+        rtn = ANY_OF([IS_DATE(), IS_EMAIL()])('a@b.co')
         self.assertEqual(rtn, ('a@b.co', None))
-        rtn = ANY_OF([IS_DATE(),IS_EMAIL()])('1982-12-14')
+        rtn = ANY_OF([IS_DATE(), IS_EMAIL()])('1982-12-14')
         self.assertEqual(rtn, (datetime.date(1982, 12, 14), None))
-        rtn = ANY_OF([IS_DATE(format='%m/%d/%Y'),IS_EMAIL()]).formatter(datetime.date(1834, 12, 14))
+        rtn = ANY_OF([IS_DATE(format='%m/%d/%Y'), IS_EMAIL()]).formatter(datetime.date(1834, 12, 14))
         self.assertEqual(rtn, '12/14/1834')
 
     def test_IS_EMPTY_OR(self):
         rtn = IS_EMPTY_OR(IS_EMAIL())('abc@def.com')
         self.assertEqual(rtn, ('abc@def.com', None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL())('   ')
+        rtn = IS_EMPTY_OR(IS_EMAIL())('   ')
         self.assertEqual(rtn, (None, None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL(), null='abc')('   ')
+        rtn = IS_EMPTY_OR(IS_EMAIL(), null='abc')('   ')
         self.assertEqual(rtn, ('abc', None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL(), null='abc', empty_regex='def')('def')
+        rtn = IS_EMPTY_OR(IS_EMAIL(), null='abc', empty_regex='def')('def')
         self.assertEqual(rtn, ('abc', None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL())('abc')
+        rtn = IS_EMPTY_OR(IS_EMAIL())('abc')
         self.assertEqual(rtn, ('abc', 'Enter a valid email address'))
-        rtn =  IS_EMPTY_OR(IS_EMAIL())(' abc ')
+        rtn = IS_EMPTY_OR(IS_EMAIL())(' abc ')
         self.assertEqual(rtn, ('abc', 'Enter a valid email address'))
-        rtn = IS_EMPTY_OR(IS_IN_SET([('id1','first label'), ('id2','second label')], zero='zero')).options(zero=False)
-        self.assertEqual(rtn, [('', ''),('id1', 'first label'), ('id2', 'second label')])
-        rtn = IS_EMPTY_OR(IS_IN_SET([('id1','first label'), ('id2','second label')], zero='zero')).options()
-        self.assertEqual(rtn, [('', 'zero'),('id1', 'first label'), ('id2', 'second label')])
+        rtn = IS_EMPTY_OR(IS_IN_SET([('id1', 'first label'), ('id2', 'second label')], zero='zero')).options(zero=False)
+        self.assertEqual(rtn, [('', ''), ('id1', 'first label'), ('id2', 'second label')])
+        rtn = IS_EMPTY_OR(IS_IN_SET([('id1', 'first label'), ('id2', 'second label')], zero='zero')).options()
+        self.assertEqual(rtn, [('', 'zero'), ('id1', 'first label'), ('id2', 'second label')])
 
     def test_CLEANUP(self):
         rtn = CLEANUP()('helloò')
         self.assertEqual(rtn, ('hello', None))
 
     def test_CRYPT(self):
-        rtn = str(CRYPT(digest_alg='md5',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='md5', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^md5\$.{16}\$.{32}$')
-        rtn = str(CRYPT(digest_alg='sha1',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha1', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha1\$.{16}\$.{40}$')
-        rtn = str(CRYPT(digest_alg='sha256',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha256', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha256\$.{16}\$.{64}$')
-        rtn = str(CRYPT(digest_alg='sha384',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha384', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha384\$.{16}\$.{96}$')
-        rtn = str(CRYPT(digest_alg='sha512',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha512', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha512\$.{16}\$.{128}$')
         alg = 'pbkdf2(1000,20,sha512)'
-        rtn = str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg=alg, salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^pbkdf2\(1000,20,sha512\)\$.{16}\$.{40}$')
-        rtn = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='md5', key='mykey', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^md5\$.{16}\$.{32}$')
-        a = str(CRYPT(digest_alg='sha1',salt=False)('test')[0])
-        self.assertEqual(CRYPT(digest_alg='sha1',salt=False)('test')[0], a)
-        self.assertEqual(CRYPT(digest_alg='sha1',salt=False)('test')[0], a[6:])
-        self.assertEqual(CRYPT(digest_alg='md5',salt=False)('test')[0], a)
-        self.assertEqual(CRYPT(digest_alg='md5',salt=False)('test')[0], a[6:])
+        a = str(CRYPT(digest_alg='sha1', salt=False)('test')[0])
+        self.assertEqual(CRYPT(digest_alg='sha1', salt=False)('test')[0], a)
+        self.assertEqual(CRYPT(digest_alg='sha1', salt=False)('test')[0], a[6:])
+        self.assertEqual(CRYPT(digest_alg='md5', salt=False)('test')[0], a)
+        self.assertEqual(CRYPT(digest_alg='md5', salt=False)('test')[0], a[6:])
 
     def test_IS_STRONG(self):
         rtn = IS_STRONG(es=True)('Abcd1234')
         self.assertEqual(rtn, ('Abcd1234',
-         'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|'))
+                               'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|'))
         rtn = IS_STRONG(es=True)('Abcd1234!')
         self.assertEqual(rtn, ('Abcd1234!', None))
         rtn = IS_STRONG(es=True, entropy=1)('a')
@@ -787,33 +837,34 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('********', None))
         rtn = IS_STRONG(es=True, max=4)('abcde')
         self.assertEqual(rtn,
-            ('abcde',
-             '|'.join(['Minimum length is 8',
-                      'Maximum length is 4',
-                      'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
-                      'Must include at least 1 upper case',
-                      'Must include at least 1 number']))
-            )
+                         ('abcde',
+                          '|'.join(['Minimum length is 8',
+                                    'Maximum length is 4',
+                                    'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
+                                    'Must include at least 1 upper case',
+                                    'Must include at least 1 number']))
+                         )
         rtn = IS_STRONG(es=True)('abcde')
         self.assertEqual(rtn,
-            ('abcde',
-             '|'.join(['Minimum length is 8',
-                      'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
-                      'Must include at least 1 upper case',
-                      'Must include at least 1 number']))
-            )
+                         ('abcde',
+                          '|'.join(['Minimum length is 8',
+                                    'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
+                                    'Must include at least 1 upper case',
+                                    'Must include at least 1 number']))
+                         )
         rtn = IS_STRONG(upper=0, lower=0, number=0, es=True)('Abcde1')
         self.assertEqual(rtn,
-            ('Abcde1',
-             '|'.join(['Minimum length is 8',
-                      'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
-                      'May not include any upper case letters',
-                      'May not include any lower case letters',
-                      'May not include any numbers']))
-            )
+                         ('Abcde1',
+                          '|'.join(['Minimum length is 8',
+                                    'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
+                                    'May not include any upper case letters',
+                                    'May not include any lower case letters',
+                                    'May not include any numbers']))
+                         )
 
     def test_IS_IMAGE(self):
         class DummyImageFile(object):
+
             def __init__(self, filename, ext, width, height):
                 from StringIO import StringIO
                 import struct
@@ -821,7 +872,7 @@ class TestValidators(unittest.TestCase):
                 self.file = StringIO()
                 if ext == 'bmp':
                     self.file.write(b'BM')
-                    self.file.write(b' '*16)
+                    self.file.write(b' ' * 16)
                     self.file.write(struct.pack('<LL', width, height))
                 elif ext == 'gif':
                     self.file.write(b'GIF87a')
@@ -832,7 +883,7 @@ class TestValidators(unittest.TestCase):
                     self.file.write(struct.pack('!xHH', height, width))
                 elif ext == 'png':
                     self.file.write(b'\211PNG\r\n\032\n')
-                    self.file.write(b' '*4)
+                    self.file.write(b' ' * 4)
                     self.file.write(b'IHDR')
                     self.file.write(struct.pack('!LL', width, height))
                 self.file.seek(0)
@@ -873,10 +924,10 @@ class TestValidators(unittest.TestCase):
         rtn = IS_IMAGE(error_message='oops')(img)
         self.assertEqual(rtn, (img, 'oops'))
 
-
     def test_IS_UPLOAD_FILENAME(self):
         import cgi
         from StringIO import StringIO
+
         def gen_fake(filename):
             formdata_file_data = """
 ---123
@@ -984,7 +1035,7 @@ this is the content of the fake file
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPV6(subnets='fb00::/8')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'Enter valid IPv6 address'))
-        rtn = IS_IPV6(subnets=['fc00::/8','2001::/32'])('2001::8ffa:fe22:b3af')
+        rtn = IS_IPV6(subnets=['fc00::/8', '2001::/32'])('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPV6(subnets='invalidsubnet')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'invalid subnet provided'))
@@ -1059,7 +1110,7 @@ this is the content of the fake file
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPADDRESS(subnets='fb00::/8')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'Enter valid IP address'))
-        rtn = IS_IPADDRESS(subnets=['fc00::/8','2001::/32'])('2001::8ffa:fe22:b3af')
+        rtn = IS_IPADDRESS(subnets=['fc00::/8', '2001::/32'])('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPADDRESS(subnets='invalidsubnet')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'invalid subnet provided'))

gluon/tools.py

@@ -1217,7 +1217,7 @@ class AuthJWT(object):
         self.auth = auth
         self.algorithm = algorithm
         if self.algorithm not in ('HS256', 'HS384', 'HS512'):
-            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
+            raise NotImplementedError('Algorithm %s not allowed' % algorithm)
         self.verify_expiration = verify_expiration
         self.leeway = leeway
         self.expiration = expiration
@@ -1314,6 +1314,7 @@ class AuthJWT(object):
         We (mis)use the heavy default auth mechanism to avoid any further computation,
         while sticking to a somewhat-stable Auth API.
         """
+        # TODO: Check the following comment
         ## is the following safe or should we use
         ## calendar.timegm(datetime.datetime.utcnow().timetuple())
         ## result seem to be the same (seconds since epoch, in UTC)
@@ -1395,9 +1396,10 @@ class AuthJWT(object):
             self.alter_payload(payload)
             ret = {'token': self.generate_token(payload)}
         elif ret is None:
-            raise HTTP(
-                401, u'Not Authorized - need to be logged in, to pass a token for refresh or username and password for login',
-                **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
+            raise HTTP(401,
+                       u'Not Authorized - need to be logged in, to pass a token '
+                       u'for refresh or username and password for login',
+                       **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
         response.headers['Content-Type'] = 'application/json'
         return serializers.json(ret)
 
@@ -1450,9 +1452,9 @@ class Auth(object):
         everybody_group_id=None,
         manager_actions={},
         auth_manager_role=None,
-        two_factor_authentication_group = None,
-        auth_two_factor_enabled = False,
-        auth_two_factor_tries_left = 3,
+        two_factor_authentication_group=None,
+        auth_two_factor_enabled=False,
+        auth_two_factor_tries_left=3,
         login_captcha=None,
         register_captcha=None,
         pre_registration_div=None,
@@ -1469,7 +1471,7 @@ class Auth(object):
         on_failed_authentication=lambda x: redirect(x),
         formstyle=None,
         label_separator=None,
-        logging_enabled = True,
+        logging_enabled=True,
         allow_delete_accounts=False,
         password_field='password',
         table_user_name='auth_user',
@@ -1743,7 +1745,7 @@ class Auth(object):
                  csrf_prevention=True, propagate_extension=None,
                  url_index=None, jwt=None, host_names=None):
 
-        ## next two lines for backward compatibility
+        # next two lines for backward compatibility
         if not db and environment and isinstance(environment, DAL):
             db = environment
         self.db = db
@@ -1780,7 +1782,7 @@ class Auth(object):
 
         url_index = url_index or URL(controller, 'index')
         url_login = URL(controller, function, args='login',
-                        extension = propagate_extension)
+                        extension=propagate_extension)
         # ## what happens after registration?
 
         settings = self.settings = Settings()
@@ -1835,9 +1837,9 @@ class Auth(object):
             hmac_key=hmac_key,
             formstyle=current.response.formstyle,
             label_separator=current.response.form_label_separator,
-            two_factor_methods = [],
-            two_factor_onvalidation = [],
-            host = host,
+            two_factor_methods=[],
+            two_factor_onvalidation=[],
+            host=host,
         )
         settings.lock_keys = True
         # ## these are messages that can be customized
@@ -1932,7 +1934,7 @@ class Auth(object):
                        'reset_password', 'request_reset_password',
                        'change_password', 'profile', 'groups',
                        'impersonate', 'not_authorized', 'confirm_registration',
-                       'bulk_register','manage_tokens','jwt'):
+                       'bulk_register', 'manage_tokens', 'jwt'):
             if len(request.args) >= 2 and args[0] == 'impersonate':
                 return getattr(self, args[0])(request.args[1])
             else:
@@ -1974,10 +1976,8 @@ class Auth(object):
         else:
             next = '?_next=' + urllib.quote(URL(args=request.args,
                                                 vars=request.get_vars))
-        href = lambda function: '%s/%s%s' % (action, function, next
-                                             if referrer_actions is DEFAULT
-                                             or function in referrer_actions
-                                             else '')
+        href = lambda function: \
+            '%s/%s%s' % (action, function, next if referrer_actions is DEFAULT or function in referrer_actions else '')
         if isinstance(prefix, str):
             prefix = T(prefix)
         if prefix:
@@ -1990,9 +1990,7 @@ class Auth(object):
         if self.user_id:  # User is logged in
             logout_next = self.settings.logout_next
             items.append({'name': T('Log Out'),
-                          'href': '%s/logout?_next=%s' % (action,
-                                                          urllib.quote(
-                                                          logout_next)),
+                          'href': '%s/logout?_next=%s' % (action, urllib.quote(logout_next)),
                           'icon': 'icon-off'})
             if 'profile' not in self.settings.actions_disabled:
                 items.append({'name': T('Profile'), 'href': href('profile'),
@@ -2025,8 +2023,8 @@ class Auth(object):
             if (self.settings.use_username and not
                     'retrieve_username' in self.settings.actions_disabled):
                 items.append({'name': T('Forgot username?'),
-                             'href': href('retrieve_username'),
-                             'icon': 'icon-edit'})
+                              'href': href('retrieve_username'),
+                              'icon': 'icon-edit'})
 
         def menu():  # For inclusion in MENU
             self.bar = [(items[0]['name'], False, items[0]['href'], [])]
@@ -2146,11 +2144,11 @@ class Auth(object):
             if self.user_id:
                 self.bar = SPAN(prefix, user_identifier, s1,
                                 Anr(items[0]['name'],
-                                _href=items[0]['href']), s3,
+                                    _href=items[0]['href']), s3,
                                 _class='auth_navbar')
             else:
                 self.bar = SPAN(s1, Anr(items[0]['name'],
-                                _href=items[0]['href']), s3,
+                                        _href=items[0]['href']), s3,
                                 _class='auth_navbar')
             for item in items[1:]:
                 self.bar.insert(-1, s2)
@@ -2207,11 +2205,10 @@ class Auth(object):
             if ('id' in fieldnames and
                 'modified_on' in fieldnames and
                 not current_record in fieldnames):
-                table._enable_record_versioning(
-                    archive_db=archive_db,
-                    archive_name=archive_names,
-                    current_record=current_record,
-                    current_record_label=current_record_label)
+                table._enable_record_versioning(archive_db=archive_db,
+                                                archive_name=archive_names,
+                                                current_record=current_record,
+                                                current_record_label=current_record_label)
 
     def define_signature(self):
         db = self.db
@@ -2475,13 +2472,11 @@ class Auth(object):
                     settings.table_token_name,
                     Field('user_id', reference_table_user, default=None,
                           label=self.messages.label_user_id),
-                    Field('expires_on', 'datetime', default=datetime.datetime(2999,12,31)),
-                    Field('token',writable=False,default=web2py_uuid,unique=True),
+                    Field('expires_on', 'datetime', default=datetime.datetime(2999, 12, 31)),
+                    Field('token', writable=False, default=web2py_uuid, unique=True),
                     *extra_fields,
-                    **dict(
-                        migrate=self.__get_migrate(
-                            settings.table_token_name, migrate),
-                        fake_migrate=fake_migrate))
+                    **dict(migrate=self.__get_migrate(settings.table_token_name, migrate),
+                           fake_migrate=fake_migrate))
         if not db._lazy_tables:
             settings.table_user = db[settings.table_user_name]
             settings.table_group = db[settings.table_group_name]
@@ -2532,9 +2527,7 @@ class Auth(object):
         # log messages should not be translated
         if type(description).__name__ == 'lazyT':
             description = description.m
-        self.table_event().insert(
-            description=str(description % vars),
-            origin=origin, user_id=user_id)
+        self.table_event().insert(description=str(description % vars), origin=origin, user_id=user_id)
 
     def get_or_create_user(self, keys, update_fields=['email'],
                            login=True, get=True):
@@ -2583,8 +2576,7 @@ class Auth(object):
             user_id = table_user.insert(**vars)
             user = table_user[user_id]
             if self.settings.create_user_groups:
-                group_id = self.add_group(
-                    self.settings.create_user_groups % user)
+                group_id = self.add_group(self.settings.create_user_groups % user)
                 self.add_membership(group_id, user_id)
             if self.settings.everybody_group_id:
                 self.add_membership(self.settings.everybody_group_id, user_id)
@@ -2750,7 +2742,7 @@ class Auth(object):
             redirect(session._cas_service)
 
         def cas_onaccept(form, onaccept=onaccept):
-            if not onaccept is DEFAULT:
+            if onaccept is not DEFAULT:
                 onaccept(form)
             return allow_access(interactivelogin=True)
         return self.login(next, onvalidation, cas_onaccept, log)
@@ -2837,14 +2829,14 @@ class Auth(object):
         response = current.response
         session = current.session
 
-        ### use session for federated login
+        # use session for federated login
         snext = self.get_vars_next()
 
         if snext:
             session._auth_next = snext
         elif session._auth_next:
             snext = session._auth_next
-        ### pass
+        # pass
 
         if next is DEFAULT:
             # important for security
@@ -2950,7 +2942,7 @@ class Auth(object):
                                )
 
                 captcha = settings.login_captcha or \
-                    (settings.login_captcha != False and settings.captcha)
+                    (settings.login_captcha is not False and settings.captcha)
                 if captcha:
                     addrow(form, captcha.label, captcha, captcha.comment,
                            settings.formstyle, 'captcha__row')
@@ -2978,8 +2970,7 @@ class Auth(object):
                         elif temp_user.registration_key in ('disabled', 'blocked'):
                             response.flash = self.messages.login_disabled
                             return form
-                        elif (temp_user.registration_key is not None
-                              and temp_user.registration_key.strip()):
+                        elif (temp_user.registration_key is not None and temp_user.registration_key.strip()):
                             response.flash = \
                                 self.messages.registration_verifying
                             return form
@@ -3090,7 +3081,7 @@ class Auth(object):
                         subject=self.messages.retrieve_two_factor_code_subject,
                         message=self.messages.retrieve_two_factor_code.format(session.auth_two_factor))
                 else:
-                    #Check for all method. It is possible to have multiples
+                    # Check for all method. It is possible to have multiples
                     for two_factor_method in two_factor_methods:
                         try:
                             # By default we use session.auth_two_factor generated before.
@@ -3106,8 +3097,6 @@ class Auth(object):
                             hideerror=settings.hideerror):
                 accepted_form = True
 
-                accepted_form = True
-
                 '''
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
@@ -3312,7 +3301,7 @@ class Auth(object):
 
         passfield = self.settings.password_field
         formstyle = self.settings.formstyle
-        try: # Make sure we have our original minimum length as other auth forms change it
+        try:  # Make sure we have our original minimum length as other auth forms change it
             table_user[passfield].requires[-1].min_length = self.settings.password_min_length
         except:
             pass
@@ -3353,7 +3342,7 @@ class Auth(object):
 
         key = web2py_uuid()
         if self.settings.registration_requires_approval:
-            key = 'pending-'+key
+            key = 'pending-' + key
 
         table_user.registration_key.default = key
         if form.accepts(request, session if self.csrf_prevention else None,
@@ -3362,12 +3351,10 @@ class Auth(object):
                         hideerror=self.settings.hideerror):
             description = self.messages.group_description % form.vars
             if self.settings.create_user_groups:
-                group_id = self.add_group(
-                    self.settings.create_user_groups % form.vars, description)
+                group_id = self.add_group(self.settings.create_user_groups % form.vars, description)
                 self.add_membership(group_id, form.vars.id)
             if self.settings.everybody_group_id:
-                self.add_membership(
-                    self.settings.everybody_group_id, form.vars.id)
+                self.add_membership(self.settings.everybody_group_id, form.vars.id)
             if self.settings.registration_requires_verification:
                 link = self.url(
                     self.settings.function, args=('verify_email', key), scheme=True)
@@ -3385,8 +3372,7 @@ class Auth(object):
                not self.settings.registration_requires_verification:
                 table_user[form.vars.id] = dict(registration_key='pending')
                 session.flash = self.messages.registration_pending
-            elif (not self.settings.registration_requires_verification or
-                      self.settings.login_after_registration):
+            elif (not self.settings.registration_requires_verification or self.settings.login_after_registration):
                 if not self.settings.registration_requires_verification:
                     table_user[form.vars.id] = dict(registration_key='')
                 session.flash = self.messages.registration_successful
@@ -3464,7 +3450,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_username_captcha or \
-                  (self.settings.retrieve_username_captcha != False and self.settings.captcha)
+                  (self.settings.retrieve_username_captcha is not False and self.settings.captcha)
         if not self.settings.mailer:
             response.flash = self.messages.function_disabled
             return ''
@@ -3622,7 +3608,7 @@ class Auth(object):
 
         if self.settings.prevent_password_reset_attacks:
             key = request.vars.key
-            if not key and len(request.args)>1:
+            if not key and len(request.args) > 1:
                 key = request.args[-1]
             if key:
                 session._reset_password_key = key
@@ -3732,7 +3718,7 @@ class Auth(object):
     def manage_tokens(self):
         if not self.user:
             redirect(self.settings.login_url)
-        table_token =self.table_token()
+        table_token = self.table_token()
         table_token.user_id.writable = False
         table_token.user_id.default = self.user.id
         table_token.token.writable = False
@@ -3796,8 +3782,7 @@ class Auth(object):
                   requires=self.table_user()[passfield].requires),
             Field('new_password2', 'password',
                   label=self.messages.verify_password,
-                  requires=[IS_EXPR(
-                      'value==%s' % repr(request.vars.new_password),
+                  requires=[IS_EXPR('value==%s' % repr(request.vars.new_password),
                                     self.messages.mismatched_password)]),
             submit_button=self.messages.password_reset_button,
             hidden=dict(_next=next),
@@ -3831,7 +3816,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_password_captcha or \
-                (self.settings.retrieve_password_captcha != False and self.settings.captcha)
+                (self.settings.retrieve_password_captcha is not False and self.settings.captcha)
 
         if next is DEFAULT:
             next = self.get_vars_next() or self.settings.request_reset_password_next
@@ -3876,7 +3861,7 @@ class Auth(object):
                         formname='reset_password', dbio=False,
                         onvalidation=onvalidation,
                         hideerror=self.settings.hideerror):
-            user = table_user(**{userfield:form.vars.get(userfield)})
+            user = table_user(**{userfield: form.vars.get(userfield)})
             key = user.registration_key
             if not user:
                 session.flash = self.messages['invalid_%s' % userfield]
@@ -4242,10 +4227,8 @@ class Auth(object):
                         else:
                             next = self.here()
                             current.session.flash = current.response.flash
-                            return call_or_redirect(
-                                self.settings.on_failed_authentication,
-                                self.settings.login_url +
-                                    '?_next=' + urllib.quote(next))
+                            return call_or_redirect(self.settings.on_failed_authentication,
+                                                    self.settings.login_url + '?_next=' + urllib.quote(next))
 
                 if callable(condition):
                     flag = condition()
@@ -4326,11 +4309,8 @@ class Auth(object):
         """
         Creates a group associated to a role
         """
-
-        group_id = self.table_group().insert(
-            role=role, description=description)
-        self.log_event(self.messages['add_group_log'],
-                       dict(group_id=group_id, role=role))
+        group_id = self.table_group().insert(role=role, description=description)
+        self.log_event(self.messages['add_group_log'], dict(group_id=group_id, role=role))
         return group_id
 
     def del_group(self, group_id):
@@ -4340,7 +4320,8 @@ class Auth(object):
         self.db(self.table_group().id == group_id).delete()
         self.db(self.table_membership().group_id == group_id).delete()
         self.db(self.table_permission().group_id == group_id).delete()
-        if group_id in self.user_groups: del self.user_groups[group_id]
+        if group_id in self.user_groups:
+            del self.user_groups[group_id]
         self.log_event(self.messages.del_group_log, dict(group_id=group_id))
 
     def id_group(self, role):
@@ -4372,7 +4353,6 @@ class Auth(object):
         """
         Checks if user is member of group_id or role
         """
-
         group_id = group_id or self.id_group(role)
         try:
             group_id = int(group_id)
@@ -4381,8 +4361,8 @@ class Auth(object):
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
-        if group_id and user_id and self.db((membership.user_id == user_id)
-                    & (membership.group_id == group_id)).select():
+        if group_id and user_id and self.db((membership.user_id == user_id) &
+                                            (membership.group_id == group_id)).select():
             r = True
         else:
             r = False
@@ -4405,8 +4385,8 @@ class Auth(object):
             user_id = self.user.id
         membership = self.table_membership()
         db = membership._db
-        record = db((membership.user_id==user_id)&
-                    (membership.group_id==group_id),
+        record = db((membership.user_id == user_id) &
+                    (membership.group_id == group_id),
                     ignore_common_filters=True).select().first()
         if record:
             if hasattr(record, 'is_active') and not record.is_active:
@@ -4429,15 +4409,18 @@ class Auth(object):
         """
 
         group_id = group_id or self.id_group(role)
+        try:
+            group_id = int(group_id)
+        except:
+            group_id = self.id_group(group_id)  # interpret group_id as a role
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
         self.log_event(self.messages['del_membership_log'],
                        dict(user_id=user_id, group_id=group_id))
-        ret = self.db(membership.user_id
-                      == user_id)(membership.group_id
-                                  == group_id).delete()
-        if group_id in self.user_groups: del self.user_groups[group_id]
+        ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
+        if group_id in self.user_groups:
+            del self.user_groups[group_id]
         return ret
 
     def has_permission(self,
@@ -4454,34 +4437,32 @@ class Auth(object):
         """
 
         if not group_id and self.settings.everybody_group_id and \
-                self.has_permission(
-            name, table_name, record_id, user_id=None,
-            group_id=self.settings.everybody_group_id):
+                self.has_permission(name, table_name, record_id, user_id=None,
+                                    group_id=self.settings.everybody_group_id):
                 return True
 
         if not user_id and not group_id and self.user:
             user_id = self.user.id
         if user_id:
             membership = self.table_membership()
-            rows = self.db(membership.user_id
-                           == user_id).select(membership.group_id)
+            rows = self.db(membership.user_id == user_id).select(membership.group_id)
             groups = set([row.group_id for row in rows])
             if group_id and group_id not in groups:
                 return False
         else:
             groups = set([group_id])
         permission = self.table_permission()
-        rows = self.db(permission.name == name)(permission.table_name
-                 == str(table_name))(permission.record_id
-                 == record_id).select(permission.group_id)
+        rows = self.db(permission.name ==
+                       name)(permission.table_name ==
+                             str(table_name))(permission.record_id ==
+                                              record_id).select(permission.group_id)
         groups_required = set([row.group_id for row in rows])
         if record_id:
-            rows = self.db(permission.name
-                            == name)(permission.table_name
-                     == str(table_name))(permission.record_id
-                     == 0).select(permission.group_id)
-            groups_required = groups_required.union(set([row.group_id
-                    for row in rows]))
+            rows = self.db(permission.name ==
+                           name)(permission.table_name ==
+                                 str(table_name))(permission.record_id ==
+                                                  0).select(permission.group_id)
+            groups_required = groups_required.union(set([row.group_id for row in rows]))
         if groups.intersection(groups_required):
             r = True
         else:
@@ -4505,12 +4486,12 @@ class Auth(object):
         permission = self.table_permission()
         if group_id == 0:
             group_id = self.user_group()
-        record = self.db((permission.group_id == group_id)&
-                         (permission.name == name)&
-                         (permission.table_name == str(table_name))&
+        record = self.db((permission.group_id == group_id) &
+                         (permission.name == name) &
+                         (permission.table_name == str(table_name)) &
                          (permission.record_id == long(record_id)),
-                          ignore_common_filters=True).select(
-            limitby=(0, 1), orderby_on_limitby=False).first()
+                         ignore_common_filters=True
+                         ).select(limitby=(0, 1), orderby_on_limitby=False).first()
         if record:
             if hasattr(record, 'is_active') and not record.is_active:
                 record.update_record(is_active=True)
@@ -4539,10 +4520,11 @@ class Auth(object):
         self.log_event(self.messages['del_permission_log'],
                        dict(group_id=group_id, name=name,
                             table_name=table_name, record_id=record_id))
-        return self.db(permission.group_id == group_id)(permission.name
-                 == name)(permission.table_name
-                           == str(table_name))(permission.record_id
-                 == long(record_id)).delete()
+        return self.db(permission.group_id ==
+                       group_id)(permission.name ==
+                                 name)(permission.table_name ==
+                                       str(table_name))(permission.record_id ==
+                                                        long(record_id)).delete()
 
     def accessible_query(self, name, table, user_id=None):
         """
@@ -4569,10 +4551,9 @@ class Auth(object):
                 cquery = table
             tablenames = db._adapter.tables(cquery)
             for tablename in tablenames:
-                cquery &= self.accessible_query(name, tablename,
-                                                user_id=user_id)
+                cquery &= self.accessible_query(name, tablename, user_id=user_id)
             return cquery
-        if not isinstance(table, str) and\
+        if not isinstance(table, str) and \
                 self.has_permission(name, table, 0, user_id):
             return table.id > 0
         membership = self.table_membership()
@@ -4601,11 +4582,11 @@ class Auth(object):
         If you have a table (db.mytable) that needs full revision history you
         can just do::
 
-            form=crud.update(db.mytable,myrecord,onaccept=auth.archive)
+            form = crud.update(db.mytable, myrecord, onaccept=auth.archive)
 
         or::
 
-            form=SQLFORM(db.mytable,myrecord).process(onaccept=auth.archive)
+            form = SQLFORM(db.mytable, myrecord).process(onaccept=auth.archive)
 
         crud.archive will define a new table "mytable_archive" and store
         a copy of the current record (if archive_current=True)
@@ -4619,18 +4600,18 @@ class Auth(object):
         in a model::
 
             db.define_table('mytable_archive',
-                Field('current_record',db.mytable),
-                db.mytable)
+                            Field('current_record', db.mytable),
+                            db.mytable)
 
         Notice such table includes all fields of db.mytable plus one: current_record.
         crud.archive does not timestamp the stored record unless your original table
         has a fields like::
 
             db.define_table(...,
-                Field('saved_on','datetime',
-                     default=request.now,update=request.now,writable=False),
-                Field('saved_by',auth.user,
-                     default=auth.user_id,update=auth.user_id,writable=False),
+                Field('saved_on', 'datetime',
+                      default=request.now, update=request.now, writable=False),
+                Field('saved_by', auth.user,
+                      default=auth.user_id, update=auth.user_id, writable=False),
 
         there is nothing special about these fields since they are filled before
         the record is archived.
@@ -4639,15 +4620,14 @@ class Auth(object):
         you can do, for example::
 
             db.define_table('myhistory',
-                Field('parent_record',db.mytable),
-                db.mytable)
+                Field('parent_record', db.mytable), db.mytable)
 
         and use it as::
 
-            form=crud.update(db.mytable,myrecord,
-                             onaccept=lambda form:crud.archive(form,
-                             archive_table=db.myhistory,
-                             current_record='parent_record'))
+            form = crud.update(db.mytable, myrecord,
+                               onaccept=lambda form:crud.archive(form,
+                                                                 archive_table=db.myhistory,
+                                                                 current_record='parent_record'))
 
         """
         if not archive_current and not form.record:
@@ -4655,7 +4635,7 @@ class Auth(object):
         table = form.table
         if not archive_table:
             archive_table_name = '%s_archive' % table
-            if not archive_table_name in table._db:
+            if archive_table_name not in table._db:
                 table._db.define_table(
                     archive_table_name,
                     Field(current_record, table),
@@ -4910,18 +4890,16 @@ class Crud(object):
         self.deleted = False
         captcha = self.settings.update_captcha or self.settings.captcha
         if record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
-                         self.settings.formstyle, 'captcha__row')
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
         captcha = self.settings.create_captcha or self.settings.captcha
         if not record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
-                         self.settings.formstyle, 'captcha__row')
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
         if request.extension not in ('html', 'load'):
             (_session, _formname) = (None, None)
         else:
             (_session, _formname) = (
                 session, '%s/%s' % (table._tablename, form.record_id))
-        if not formname is DEFAULT:
+        if formname is not DEFAULT:
             _formname = formname
         keepvalues = self.settings.keepvalues
         if request.vars.delete_this_record:
@@ -5210,7 +5188,7 @@ class Crud(object):
                     elif validate:
                         value, error = field.validate(txtval)
                         if not error:
-                            ### TODO deal with 'starts with', 'ends with', 'contains' on GAE
+                            # TODO deal with 'starts with', 'ends with', 'contains' on GAE
                             query &= self.get_query(field, opval, value)
                         else:
                             row[3].append(DIV(error, _class='error'))
@@ -5223,7 +5201,7 @@ class Crud(object):
                 results = db(query).select(*selected, **attributes)
                 for r in refsearch:
                     results = results.find(r)
-            except:  # hmmm, we should do better here
+            except:  # TODO: hmmm, we should do better here
                 results = None
         return form, results
 
@@ -5235,7 +5213,7 @@ def fetch(url, data=None, headers=None,
           cookie=Cookie.SimpleCookie(),
           user_agent='Mozilla/5.0'):
     headers = headers or {}
-    if not data is None:
+    if data is not None:
         data = urllib.urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
@@ -5698,12 +5676,12 @@ class Service(object):
         methods = self.jsonrpc_procedures
         data = json_parser.loads(request.body.read())
         jsonrpc_2 = data.get('jsonrpc')
-        if jsonrpc_2: #hand over to version 2 of the protocol
+        if jsonrpc_2:  # hand over to version 2 of the protocol
             return self.serve_jsonrpc2(data)
         id, method, params = data.get('id'), data.get('method'), data.get('params', [])
         if id is None:
             return return_error(0, 100, 'missing id')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, 100, 'method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -5727,8 +5705,7 @@ class Service(object):
         def return_response(id, result):
             if not must_respond:
                 return None
-            return serializers.json({'jsonrpc': '2.0',
-                'id': id, 'result': result})
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'result': result})
 
         def return_error(id, code, message=None, data=None):
             error = {'code': code}
@@ -5739,9 +5716,7 @@ class Service(object):
                 error['message'] = message
             if data is not None:
                 error['data'] = data
-            return serializers.json({'jsonrpc': '2.0',
-                                     'id': id,
-                                     'error': error})
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'error': error})
 
         def validate(data):
             """
@@ -5801,7 +5776,7 @@ class Service(object):
             return return_error(None, e.code, e.info)
 
         id, method, params = data.get('id'), data['method'], data.get('params', '')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, -32601, data='Method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -5907,7 +5882,7 @@ class Service(object):
                     UL(LI("Location: %s" % dispatcher.location),
                        LI("Namespace: %s" % dispatcher.namespace),
                        LI("SoapAction: %s" % dispatcher.action),
-                    ),
+                       ),
                     H3("Sample SOAP XML Request Message:"),
                     CODE(sample_req_xml, language="xml"),
                     H3("Sample SOAP XML Response Message:"),
@@ -6041,7 +6016,7 @@ def prettydate(d, T=lambda x: x, utc=False):
         return T('1 year' + suffix)
     elif dt.days >= 60:
         return T('%d months' + suffix) % int(dt.days / 30)
-    elif dt.days > 21:
+    elif dt.days >= 27:  # 4 weeks ugly
         return T('1 month' + suffix)
     elif dt.days >= 14:
         return T('%d weeks' + suffix) % int(dt.days / 7)
@@ -6447,10 +6422,9 @@ class Wiki(object):
                 args += value['args']
                 db.define_table(key, *args, **value['vars'])
 
-        if self.settings.templates is None and not \
-           self.settings.manage_permissions:
-            self.settings.templates = db.wiki_page.tags.contains('template') & \
-                db.wiki_page.can_read.contains('everybody')
+        if self.settings.templates is None and not self.settings.manage_permissions:
+            self.settings.templates = \
+                db.wiki_page.tags.contains('template') & db.wiki_page.can_read.contains('everybody')
 
         def update_tags_insert(page, id, db=db):
             for tag in page.tags or []:
@@ -6473,8 +6447,10 @@ class Wiki(object):
             'wiki_editor' not in auth.user_groups.values() and
                 self.settings.groups == auth.user_groups.values()):
             group = db.auth_group(role='wiki_editor')
-            gid = group.id if group else db.auth_group.insert(
-                role='wiki_editor')
+            if group:
+                gid = group.id
+            else:
+                db.auth_group.insert(role='wiki_editor')
             auth.add_membership(gid)
 
         settings.lock_keys = True
@@ -6501,9 +6477,8 @@ class Wiki(object):
         groups = self.settings.groups
         return ('wiki_editor' in groups or
                 (page is None and 'wiki_author' in groups) or
-                not page is None and (
-                set(groups).intersection(set(page.can_edit)) or
-                page.created_by == self.auth.user.id))
+                page is not None and (set(groups).intersection(set(page.can_edit)) or
+                                      page.created_by == self.auth.user.id))
 
     def can_manage(self):
         if not self.auth.user:
@@ -6524,7 +6499,7 @@ class Wiki(object):
                     return True
         return False
 
-    ### END POLICY
+    # END POLICY
 
     def automenu(self):
         """adds the menu if not present"""
@@ -6742,20 +6717,15 @@ class Wiki(object):
         if self.settings.templates:
             fields.append(
                 Field("from_template", "reference wiki_page",
-                      requires=IS_EMPTY_OR(
-                                   IS_IN_DB(db(self.settings.templates),
-                                            db.wiki_page._id,
-                                            '%(slug)s')),
-                      comment=current.T(
-                        "Choose Template or empty for new Page")))
+                      requires=IS_EMPTY_OR(IS_IN_DB(db(self.settings.templates), db.wiki_page._id, '%(slug)s')),
+                      comment=current.T("Choose Template or empty for new Page")))
         form = SQLFORM.factory(*fields, **dict(_class="well"))
         form.element("[type=submit]").attributes["_value"] = \
             current.T("Create Page from Slug")
 
         if form.process().accepted:
-             form.vars.from_template = 0 if not form.vars.from_template \
-                 else form.vars.from_template
-             redirect(URL(args=('_edit', form.vars.slug, form.vars.from_template or 0)))  # added param
+            form.vars.from_template = 0 if not form.vars.from_template else form.vars.from_template
+            redirect(URL(args=('_edit', form.vars.slug, form.vars.from_template or 0)))  # added param
         return dict(content=form)
 
     def pages(self):
@@ -6846,25 +6816,25 @@ class Wiki(object):
                     mode = 0
                 if mode in (2, 3):
                     submenu.append((current.T('View Page'), None,
-                    URL(controller, function, args=slug)))
+                                    URL(controller, function, args=slug)))
                 if mode in (1, 3):
                     submenu.append((current.T('Edit Page'), None,
-                    URL(controller, function, args=('_edit', slug))))
+                                    URL(controller, function, args=('_edit', slug))))
                 if mode in (1, 2):
                     submenu.append((current.T('Edit Page Media'), None,
-                    URL(controller, function, args=('_editmedia', slug))))
+                                    URL(controller, function, args=('_editmedia', slug))))
 
             submenu.append((current.T('Create New Page'), None,
                             URL(controller, function, args=('_create'))))
             # Moved next if to inside self.auth.user check
             if self.can_manage():
                 submenu.append((current.T('Manage Pages'), None,
-                            URL(controller, function, args=('_pages'))))
+                                URL(controller, function, args=('_pages'))))
                 submenu.append((current.T('Edit Menu'), None,
-                            URL(controller, function, args=('_edit', 'wiki-menu'))))
+                                URL(controller, function, args=('_edit', 'wiki-menu'))))
             # Also moved inside self.auth.user check
             submenu.append((current.T('Search Pages'), None,
-                        URL(controller, function, args=('_search'))))
+                            URL(controller, function, args=('_search'))))
         return menu
 
     def search(self, tags=None, query=None, cloud=True, preview=True,
@@ -6882,7 +6852,7 @@ class Wiki(object):
             if request.vars.q:
                 tags = [v.strip() for v in request.vars.q.split(',')]
                 tags = [v.lower() for v in tags if v]
-        if tags or not query is None:
+        if tags or query is not None:
             db = self.auth.db
             count = db.wiki_tag.wiki_page.count()
             fields = [db.wiki_page.id, db.wiki_page.slug,

gluon/utils.py

@@ -83,11 +83,9 @@ def compare(a, b):
     """ Compares two strings and not vulnerable to timing attacks """
     if HAVE_COMPARE_DIGEST:
         return hmac.compare_digest(a, b)
-    if len(a) != len(b):
-        return False
-    result = 0
-    for x, y in zip(a, b):
-        result |= ord(x) ^ ord(y)
+    result = len(a) ^ len(b)
+    for i in xrange(len(b)):
+		result |= ord(a[i%len(a)]) ^ ord(b[i])
     return result == 0
 
 

gluon/validators.py

@@ -645,11 +645,10 @@ class IS_IN_DB(Validator):
                 if not [v for v in values if v not in self.theset]:
                     return (values, None)
             else:
-                from pydal.adapters import GoogleDatastoreAdapter
-
                 def count(values, s=self.dbset, f=field):
                     return s(f.belongs(map(int, values))).count()
-                if GoogleDatastoreAdapter is not None and isinstance(self.dbset.db._adapter, GoogleDatastoreAdapter):
+
+                if self.dbset.db._adapter.dbengine == "google:datastore":
                     range_ids = range(0, len(values), 30)
                     total = sum(count(values[i:i + 30]) for i in range_ids)
                     if total == len(values):

scripts/lang_update_from_langfile.py

@@ -30,8 +30,15 @@ if __name__ == '__main__':
         dest="source",
         help="Specify language file (ro) where seek for translations"
     )
+    parser.add_argument(
+        '-f', '--force-update',
+        dest="force_update",
+        action="store_true",
+        default=False,
+        help="without it: add new + translate untranslated, if used: in addition update items if translation differs"
+    )
     args = parser.parse_args()
 
-    update_from_langfile(args.target, args.source)
+    update_from_langfile(args.target, args.source, force_update=args.force_update)
 
     print '%s was updated.' % args.target