R-2.14.6

simplified oneall dname handling

.travis.yml

@@ -12,8 +12,22 @@ python:
   - 'pypy'
 
 install:
+  - |
+      if [ "$TRAVIS_PYTHON_VERSION" = "pypy" ]; then
+        export PYENV_ROOT="$HOME/.pyenv"
+        if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
+          pushd "$PYENV_ROOT" && git pull && popd
+        else
+          rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
+        fi
+        export PYPY_VERSION="5.0.1"
+        "$PYENV_ROOT/bin/pyenv" install --skip-existing "pypy-$PYPY_VERSION"
+        virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION"
+        source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate"
+      fi
   - pip install -e .
 
+
 before_script:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install --download-cache $HOME/.pip-cache unittest2; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache coverage; fi;

CHANGELOG

@@ -1,5 +1,12 @@
-## 2.14.1
+## 2.14.6
 
+- Increased test coverage (thanks Richard)
+- Fixed some newly discovered security issues in admin:
+  CSRF vulnerability in admin that allows disabling apps
+  Brute force password attack vulnerability in admin
+  (thanks Narendra and Leonel)  
+
+## 2.14.1-5
 
 - fixed two major security issues that caused the examples app to leak information
 - new Auth(…,host_names=[…]) to prevent host header injection
@@ -45,6 +52,7 @@
     sessiondb = RedisSession(redis_conn=rconn, session_expiry=False)
     session.connect(request, response, db = sessiondb)
 
+Many thanks to Richard and Simone for their work and dedication.
 
 ## 2.13.*
 

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

VERSION

@@ -1 +1 @@
-Version 2.14.2-stable+timestamp.2016.03.24.17.44.22
+Version 2.14.6-stable+timestamp.2016.05.09.19.18.48

applications/admin/controllers/default.py

@@ -32,15 +32,15 @@ from gluon.languages import (read_possible_languages, read_dict, write_dict,
 
 
 if DEMO_MODE and request.function in ['change_password', 'pack',
-'pack_custom','pack_plugin', 'upgrade_web2py', 'uninstall',
+                                      'pack_custom', 'pack_plugin', 'upgrade_web2py', 'uninstall',
-'cleanup', 'compile_app', 'remove_compiled_app', 'delete',
+                                      'cleanup', 'compile_app', 'remove_compiled_app', 'delete',
-'delete_plugin', 'create_file', 'upload_file', 'update_languages',
+                                      'delete_plugin', 'create_file', 'upload_file', 'update_languages',
-'reload_routes', 'git_push', 'git_pull', 'install_plugin']:
+                                      'reload_routes', 'git_push', 'git_pull', 'install_plugin']:
     session.flash = T('disabled in demo mode')
     redirect(URL('site'))
 
 if is_gae and request.function in ('edit', 'edit_language',
-'edit_plurals', 'update_languages', 'create_file', 'install_plugin'):
+                                   'edit_plurals', 'update_languages', 'create_file', 'install_plugin'):
     session.flash = T('disabled in GAE mode')
     redirect(URL('site'))
 
@@ -74,8 +74,10 @@ def log_progress(app, mode='EDIT', filename=None, progress=0):
 def safe_open(a, b):
     if (DEMO_MODE or is_gae) and ('w' in b or 'a' in b):
         class tmp:
+
             def write(self, data):
                 pass
+
             def close(self):
                 pass
         return tmp()
@@ -119,6 +121,9 @@ def index():
         send = URL('site')
     if session.authorized:
         redirect(send)
+    elif failed_login_count() >= allowed_number_of_attempts:
+        time.sleep(2 ** allowed_number_of_attempts)
+        raise HTTP(403)
     elif request.vars.password:
         if verify_password(request.vars.password[:1024]):
             session.authorized = True
@@ -208,6 +213,7 @@ def site():
     file_or_appurl = 'file' in request.vars or 'appurl' in request.vars
 
     class IS_VALID_APPNAME(object):
+
         def __call__(self, value):
             if not re.compile('^\w+$').match(value):
                 return (value, T('Invalid application name'))
@@ -325,7 +331,7 @@ def report_progress(app):
         if not m:
             continue
         days = -(request.now - datetime.datetime.strptime(m[0],
-                 '%Y-%m-%d %H:%M:%S')).days
+                                                          '%Y-%m-%d %H:%M:%S')).days
         counter += int(m[1])
         events.append([days, counter])
     return events
@@ -353,6 +359,7 @@ def pack():
         session.flash = T('internal error: %s', e)
         redirect(URL('site'))
 
+
 def pack_plugin():
     app = get_app()
     if len(request.args) == 2:
@@ -368,7 +375,6 @@ def pack_plugin():
         redirect(URL('plugin', args=request.args))
 
 
-
 def pack_exe(app, base, filenames=None):
     import urllib
     import zipfile
@@ -397,10 +403,20 @@ def pack_exe(app, base, filenames=None):
 def pack_custom():
     app = get_app()
     base = apath(app, r=request)
-    if request.post_vars.file:
 
+    def ignore(fs):
+        return [f for f in fs if not (
+                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
+    files = {}
+    for (r, d, f) in os.walk(base):
+        files[r] = {'folders': ignore(d), 'files': ignore(f)}
+
+    if request.post_vars.file:
+        valid_set = set(os.path.relpath(os.path.join(r, f), base) for r in files for f in files[r]['files'])
         files = request.post_vars.file
-        files = [files] if not isinstance(files,list) else files
+        files = [files] if not isinstance(files, list) else files
+        files = [file for file in files if file in valid_set]
+
         if request.post_vars.doexe is None:
             fname = 'web2py.app.%s.w2p' % app
             try:
@@ -417,12 +433,7 @@ def pack_custom():
                 redirect(URL(args=request.args))
         else:
             return pack_exe(app, base, files)
-    def ignore(fs):
+
-        return [f for f in fs if not (
-                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
-    files = {}
-    for (r,d,f) in os.walk(base):
-        files[r] = {'folders':ignore(d),'files':ignore(f)}
     return locals()
 
 
@@ -485,14 +496,14 @@ def cleanup():
 def compile_app():
     app = get_app()
     c = app_compile(app, request,
-        skip_failed_views = (request.args(1) == 'skip_failed_views'))
+                    skip_failed_views=(request.args(1) == 'skip_failed_views'))
     if not c:
         session.flash = T('application compiled')
     elif isinstance(c, list):
         session.flash = DIV(*[T('application compiled'), BR(), BR(),
                               T('WARNING: The following views could not be compiled:'), BR()] +
-                             [CAT(BR(), view) for view in c] +
+                            [CAT(BR(), view) for view in c] +
-                             [BR(), BR(), T('DO NOT use the "Pack compiled" feature.')])
+                            [BR(), BR(), T('DO NOT use the "Pack compiled" feature.')])
     else:
         session.flash = DIV(T('Cannot compile: there are errors in your app:'),
                             CODE(c))
@@ -533,8 +544,8 @@ def delete():
         redirect(URL(sender, anchor=request.vars.id2))
     return dict(dialog=dialog, filename=filename)
 
-
 def enable():
+    if not URL.verify(request, hmac_key=session.hmac_key): raise HTTP(401)
     app = get_app()
     filename = os.path.join(apath(app, r=request), 'DISABLED')
     if is_gae:
@@ -546,6 +557,7 @@ def enable():
         safe_open(filename, 'wb').write('disabled: True\ntime-disabled: %s' % request.now)
         return SPAN(T('Enable'), _style='color:red')
 
+
 def peek():
     """ Visualize object code """
     app = get_app(request.vars.app)
@@ -609,7 +621,7 @@ def edit():
     # Load json only if it is ajax edited...
     app = get_app(request.vars.app)
     app_path = apath(app, r=request)
-    preferences={'theme':'web2py', 'editor': 'default', 'closetag': 'true', 'codefolding': 'false', 'tabwidth':'4', 'indentwithtabs':'false', 'linenumbers':'true', 'highlightline':'true'}
+    preferences = {'theme': 'web2py', 'editor': 'default', 'closetag': 'true', 'codefolding': 'false', 'tabwidth': '4', 'indentwithtabs': 'false', 'linenumbers': 'true', 'highlightline': 'true'}
     config = Config(os.path.join(request.folder, 'settings.cfg'),
                     section='editor', default_values={})
     preferences.update(config.read())
@@ -617,14 +629,14 @@ def edit():
     if not(request.ajax) and not(is_mobile):
         # return the scaffolding, the rest will be through ajax requests
         response.title = T('Editing %s') % app
-        return response.render ('default/edit.html', dict(app=app, editor_settings=preferences))
+        return response.render('default/edit.html', dict(app=app, editor_settings=preferences))
 
     # show settings tab and save prefernces
     if 'settings' in request.vars:
-        if request.post_vars:        #save new preferences
+        if request.post_vars:  # save new preferences
             post_vars = request.post_vars.items()
             # Since unchecked checkbox are not serialized, we must set them as false by hand to store the correct preference in the settings
-            post_vars+= [(opt, 'false') for opt in preferences if opt not in request.post_vars ]
+            post_vars += [(opt, 'false') for opt in preferences if opt not in request.post_vars]
             if config.save(post_vars):
                 response.headers["web2py-component-flash"] = T('Preferences saved correctly')
             else:
@@ -632,8 +644,8 @@ def edit():
             response.headers["web2py-component-command"] = "update_editor(%s);$('a[href=#editor_settings] button.close').click();" % response.json(config.read())
             return
         else:
-            details = {'realfilename':'settings', 'filename':'settings', 'id':'editor_settings', 'force': False}
+            details = {'realfilename': 'settings', 'filename': 'settings', 'id': 'editor_settings', 'force': False}
-            details['plain_html'] = response.render('default/editor_settings.html', {'editor_settings':preferences})
+            details['plain_html'] = response.render('default/editor_settings.html', {'editor_settings': preferences})
             return response.json(details)
 
     """ File edit handler """
@@ -764,8 +776,8 @@ def edit():
             view = request.args[3].replace('.html', '')
             view_link = URL(request.args[0], request.args[2], view)
     elif filetype == 'python' and request.args[1] == 'controllers':
-        ## it's a controller file.
+        # it's a controller file.
-        ## Create links to all of the associated view files.
+        # Create links to all of the associated view files.
         app = get_app()
         viewname = os.path.splitext(request.args[2])[0]
         viewpath = os.path.join(app, 'views', viewname)
@@ -796,22 +808,22 @@ def edit():
         return response.json({'file_hash': file_hash, 'saved_on': saved_on, 'functions': functions, 'controller': controller, 'application': request.args[0], 'highlight': highlight})
     else:
         file_details = dict(app=request.args[0],
-                    lineno=request.vars.lineno or 1,
+                            lineno=request.vars.lineno or 1,
-                    editor_settings=preferences,
+                            editor_settings=preferences,
-                    filename=filename,
+                            filename=filename,
-                    realfilename=realfilename,
+                            realfilename=realfilename,
-                    filetype=filetype,
+                            filetype=filetype,
-                    data=data,
+                            data=data,
-                    edit_controller=edit_controller,
+                            edit_controller=edit_controller,
-                    file_hash=file_hash,
+                            file_hash=file_hash,
-                    saved_on=saved_on,
+                            saved_on=saved_on,
-                    controller=controller,
+                            controller=controller,
-                    functions=functions,
+                            functions=functions,
-                    view_link=view_link,
+                            view_link=view_link,
-                    editviewlinks=editviewlinks,
+                            editviewlinks=editviewlinks,
-                    id=IS_SLUG()(filename)[0],
+                            id=IS_SLUG()(filename)[0],
-                    force= True if (request.vars.restore or
+                            force=True if (request.vars.restore or
-                                    request.vars.revert) else False)
+                                           request.vars.revert) else False)
         plain_html = response.render('default/edit_js.html', file_details)
         file_details['plain_html'] = plain_html
         if is_mobile:
@@ -820,14 +832,16 @@ def edit():
         else:
             return response.json(file_details)
 
+
 def todolist():
     """ Returns all TODO of the requested app
     """
     app = request.vars.app or ''
-    app_path = apath('%(app)s' % {'app':app}, r=request)
+    app_path = apath('%(app)s' % {'app': app}, r=request)
-    dirs=['models', 'controllers', 'modules', 'private' ]
+    dirs = ['models', 'controllers', 'modules', 'private']
+
     def listfiles(app, dir, regexp='.*\.py$'):
-        files = sorted( listdir(apath('%(app)s/%(dir)s/' % {'app':app, 'dir':dir}, r=request), regexp))
+        files = sorted(listdir(apath('%(app)s/%(dir)s/' % {'app': app, 'dir': dir}, r=request), regexp))
         files = [x.replace(os.path.sep, '/') for x in files if not x.endswith('.bak')]
         return files
 
@@ -838,17 +852,18 @@ def todolist():
     for d in dirs:
         for f in listfiles(app, d):
             matches = []
-            filename= apath(os.path.join(app, d, f), r=request)
+            filename = apath(os.path.join(app, d, f), r=request)
             with open(filename, 'r') as f_s:
                 src = f_s.read()
                 for m in regex.finditer(src):
                     start = m.start()
                     lineno = src.count('\n', 0, start) + 1
-                    matches.append({'text':m.group(0), 'lineno':lineno})
+                    matches.append({'text': m.group(0), 'lineno': lineno})
             if len(matches) != 0:
-                output.append({'filename':f,'matches':matches, 'dir':d})
+                output.append({'filename': f, 'matches': matches, 'dir': d})
+
+    return {'todo': output, 'app': app}
 
-    return {'todo':output, 'app': app}
 
 def editor_sessions():
     config = Config(os.path.join(request.folder, 'settings.cfg'),
@@ -858,13 +873,14 @@ def editor_sessions():
     if request.vars.session_name and request.vars.files:
         session_name = request.vars.session_name
         files = request.vars.files
-        preferences.update({session_name:','.join(files)})
+        preferences.update({session_name: ','.join(files)})
         if config.save(preferences.items()):
             response.headers["web2py-component-flash"] = T('Session saved correctly')
         else:
             response.headers["web2py-component-flash"] = T('Session saved on session only')
 
-    return response.render('default/editor_sessions.html', {'editor_sessions':preferences})
+    return response.render('default/editor_sessions.html', {'editor_sessions': preferences})
+
 
 def resolve():
     """
@@ -901,7 +917,7 @@ def resolve():
 
     def getclass(item):
         """ Determine item class """
-        operators = {' ':'normal', '+':'plus', '-':'minus'}
+        operators = {' ': 'normal', '+': 'plus', '-': 'minus'}
 
         return operators[item[0]]
 
@@ -921,7 +937,7 @@ def resolve():
         diff = TABLE(*[TR(TD(gen_data(i, item)),
                           TD(item[0]),
                           TD(leading(item[2:]),
-                          TT(item[2:].rstrip())),
+                             TT(item[2:].rstrip())),
                           _class=getclass(item))
                        for (i, item) in enumerate(d) if item[0] != '?'])
 
@@ -968,11 +984,11 @@ def edit_language():
 
         new_row = DIV(LABEL(prefix, k, _style="font-weight:normal;"),
                       CAT(elem, '\n', TAG.BUTTON(
-                    T('delete'),
+                          T('delete'),
-                    _onclick='return delkey("%s")' % name,
+                          _onclick='return delkey("%s")' % name,
-                    _class='btn')), _id=name, _class='span6 well well-small')
+                          _class='btn')), _id=name, _class='span6 well well-small')
 
-        rows.append(DIV(new_row,_class="row-fluid"))
+        rows.append(DIV(new_row, _class="row-fluid"))
     rows.append(DIV(INPUT(_type='submit', _value=T('update'), _class="btn btn-primary"), _class='controls'))
     form = FORM(*rows)
     if form.accepts(request.vars, keepvalues=True):
@@ -1128,18 +1144,18 @@ def design():
 
     # Get all static files
     statics = listdir(apath('%s/static/' % app, r=request), '[^\.#].*',
-                      maxnum = MAXNFILES)
+                      maxnum=MAXNFILES)
     statics = [x.replace(os.path.sep, '/') for x in statics]
     statics.sort()
 
     # Get all languages
-    langpath = os.path.join(apath(app, r=request),'languages')
+    langpath = os.path.join(apath(app, r=request), 'languages')
     languages = dict([(lang, info) for lang, info
                       in read_possible_languages(langpath).iteritems()
                       if info[2] != 0])  # info[2] is langfile_mtime:
-                                         # get only existed files
+    # get only existed files
 
-    #Get crontab
+    # Get crontab
     cronfolder = apath('%s/cron' % app, r=request)
     crontab = apath('%s/cron/crontab' % app, r=request)
     if not is_gae:
@@ -1265,7 +1281,7 @@ def plugin():
 
     # Get all static files
     statics = listdir(apath('%s/static/' % app, r=request), '[^\.#].*',
-                      maxnum = MAXNFILES)
+                      maxnum=MAXNFILES)
     statics = [x.replace(os.path.sep, '/') for x in statics]
     statics.sort()
 
@@ -1273,9 +1289,9 @@ def plugin():
     languages = sorted([lang + '.py' for lang, info in
                         T.get_possible_languages_info().iteritems()
                         if info[2] != 0])  # info[2] is langfile_mtime:
-                                    # get only existed files
+    # get only existed files
 
-    #Get crontab
+    # Get crontab
     crontab = apath('%s/cron/crontab' % app, r=request)
     if not os.path.exists(crontab):
         safe_write(crontab, '#crontab')
@@ -1298,6 +1314,7 @@ def plugin():
                 languages=languages,
                 crontab=crontab)
 
+
 def create_file():
     """ Create files handler """
     if request.vars and not request.vars.token == session.token:
@@ -1309,7 +1326,7 @@ def create_file():
             path = abspath(request.vars.location)
         else:
             if request.vars.dir:
-                    request.vars.location += request.vars.dir + '/'
+                request.vars.location += request.vars.dir + '/'
             app = get_app(name=request.vars.location.split('/')[0])
             path = apath(request.vars.location, r=request)
         filename = re.sub('[^\w./-]+', '_', request.vars.filename)
@@ -1419,7 +1436,7 @@ def create_file():
 
         elif (path[-8:] == '/static/') or (path[-9:] == '/private/'):
             if (request.vars.plugin and
-                not filename.startswith('plugin_%s/' % request.vars.plugin)):
+                    not filename.startswith('plugin_%s/' % request.vars.plugin)):
                 filename = 'plugin_%s/%s' % (request.vars.plugin, filename)
             text = ''
 
@@ -1439,17 +1456,17 @@ def create_file():
         log_progress(app, 'CREATE', filename)
         if request.vars.dir:
             result = T('file "%(filename)s" created',
-                          dict(filename=full_filename[len(path):]))
+                       dict(filename=full_filename[len(path):]))
         else:
             session.flash = T('file "%(filename)s" created',
-                          dict(filename=full_filename[len(path):]))
+                              dict(filename=full_filename[len(path):]))
         vars = {}
         if request.vars.id:
             vars['id'] = request.vars.id
         if request.vars.app:
             vars['app'] = request.vars.app
         redirect(URL('edit',
-                 args=[os.path.join(request.vars.location, filename)], vars=vars))
+                     args=[os.path.join(request.vars.location, filename)], vars=vars))
 
     except Exception, e:
         if not isinstance(e, HTTP):
@@ -1460,7 +1477,7 @@ def create_file():
         response.headers['web2py-component-content'] = 'append'
         response.headers['web2py-component-command'] = "%s %s %s" % (
             "$.web2py.invalidate('#files_menu');",
-            "load_file('%s');" % URL('edit', args=[app,request.vars.dir,filename]),
+            "load_file('%s');" % URL('edit', args=[app, request.vars.dir, filename]),
             "$.web2py.enableElement($('#form form').find($.web2py.formInputClickSelector));")
         return ''
     else:
@@ -1468,32 +1485,35 @@ def create_file():
 
 
 def listfiles(app, dir, regexp='.*\.py$'):
-        files = sorted(
+    files = sorted(
-         listdir(apath('%(app)s/%(dir)s/' % {'app':app, 'dir':dir}, r=request), regexp))
+        listdir(apath('%(app)s/%(dir)s/' % {'app': app, 'dir': dir}, r=request), regexp))
-        files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')]
+    files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')]
-        return files
+    return files
+
+
+def editfile(path, file, vars={}, app=None):
+    args = (path, file) if 'app' in vars else (app, path, file)
+    url = URL('edit', args=args, vars=vars)
+    return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;')
 
-def editfile(path,file,vars={}, app = None):
-        args=(path,file) if 'app' in vars else (app,path,file)
-        url = URL('edit', args=args, vars=vars)
-        return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;')
 
 def files_menu():
-        app = request.vars.app or 'welcome'
+    app = request.vars.app or 'welcome'
-        dirs=[{'name':'models', 'reg':'.*\.py$'},
+    dirs = [{'name': 'models', 'reg': '.*\.py$'},
-              {'name':'controllers', 'reg':'.*\.py$'},
+            {'name': 'controllers', 'reg': '.*\.py$'},
-              {'name':'views', 'reg':'[\w/\-]+(\.\w+)+$'},
+            {'name': 'views', 'reg': '[\w/\-]+(\.\w+)+$'},
-              {'name':'modules', 'reg':'.*\.py$'},
+            {'name': 'modules', 'reg': '.*\.py$'},
-              {'name':'static', 'reg': '[^\.#].*'},
+            {'name': 'static', 'reg': '[^\.#].*'},
-              {'name':'private', 'reg':'.*\.py$'}]
+            {'name': 'private', 'reg': '.*\.py$'}]
-        result_files = []
+    result_files = []
-        for dir in dirs:
+    for dir in dirs:
-                result_files.append(TAG[''](LI(dir['name'], _class="nav-header component", _onclick="collapse('" + dir['name'] + "_files');"),
+        result_files.append(TAG[''](LI(dir['name'], _class="nav-header component", _onclick="collapse('" + dir['name'] + "_files');"),
-                                      LI(UL(*[LI(editfile(dir['name'], f, dict(id=dir['name'] + f.replace('.','__')), app), _style="overflow:hidden", _id=dir['name']+"__"+f.replace('.','__'))
+                                    LI(UL(*[LI(editfile(dir['name'], f, dict(id=dir['name'] + f.replace('.', '__')), app), _style="overflow:hidden", _id=dir['name'] + "__" + f.replace('.', '__'))
-                                                      for f in listfiles(app, dir['name'], regexp=dir['reg'])],
+                                            for f in listfiles(app, dir['name'], regexp=dir['reg'])],
-                                                      _class="nav nav-list small-font"),
+                                          _class="nav nav-list small-font"),
-                                               _id=dir['name'] + '_files', _style="display: none;")))
+                                       _id=dir['name'] + '_files', _style="display: none;")))
-        return dict(result_files = result_files)
+    return dict(result_files=result_files)
+
 
 def upload_file():
     """ File uploading handler """
@@ -1556,7 +1576,7 @@ def errors():
     app = get_app()
     if is_gae:
         method = 'dbold' if ('old' in
-                     (request.args(1) or '')) else 'dbnew'
+                             (request.args(1) or '')) else 'dbnew'
     else:
         method = request.args(1) or 'new'
     db_ready = {}
@@ -1599,7 +1619,7 @@ def errors():
                     hash2error[hash]['count'] += 1
                 except KeyError:
                     error_lines = error['traceback'].split("\n")
-                    last_line = error_lines[-2] if len(error_lines)>1 else 'unknown'
+                    last_line = error_lines[-2] if len(error_lines) > 1 else 'unknown'
                     error_causer = os.path.split(error['layer'])[1]
                     hash2error[hash] = dict(count=1, pickel=error,
                                             causer=error_causer,
@@ -1638,9 +1658,9 @@ def errors():
                         last_line = error_lines[-2]
                         error_causer = os.path.split(error['layer'])[1]
                         hash2error[hash] = dict(count=1,
-                            pickel=error, causer=error_causer,
+                                                pickel=error, causer=error_causer,
-                            last_line=last_line, hash=hash,
+                                                last_line=last_line, hash=hash,
-                            ticket=fn.ticket_id)
+                                                ticket=fn.ticket_id)
             except AttributeError, e:
                 tk_db(tk_table.id == fn.id).delete()
                 tk_db.commit()
@@ -1657,11 +1677,11 @@ def errors():
                 tk_db(tk_table.ticket_id == item[7:]).delete()
                 tk_db.commit()
         tickets_ = tk_db(tk_table.id > 0).select(tk_table.ticket_id,
-            tk_table.created_datetime,
+                                                 tk_table.created_datetime,
-            orderby=~tk_table.created_datetime)
+                                                 orderby=~tk_table.created_datetime)
         tickets = [row.ticket_id for row in tickets_]
         times = dict([(row.ticket_id, row.created_datetime) for
-            row in tickets_])
+                      row in tickets_])
         return dict(app=app, tickets=tickets, method=method,
                     times=times, db_ready=db_ready)
 
@@ -1721,7 +1741,7 @@ def make_link(path):
             if ext.lower() == editable[key] and check_extension:
                 return A('"' + tryFile + '"',
                          _href=URL(r=request,
-                         f='edit/%s/%s/%s' % (app, key, filename))).xml()
+                                   f='edit/%s/%s/%s' % (app, key, filename))).xml()
     return ''
 
 
@@ -1867,7 +1887,7 @@ def bulk_register():
         redirect(URL('site'))
     return locals()
 
-### Begin experimental stuff need fixes:
+# Begin experimental stuff need fixes:
 # 1) should run in its own process - cannot os.chdir
 # 2) should not prompt user at console
 # 3) should give option to force commit and not reuqire manual merge
@@ -1934,6 +1954,7 @@ def git_push():
             redirect(URL('site'))
     return dict(app=app, form=form)
 
+
 def plugins():
     app = request.args(0)
     from serializers import loads_json
@@ -1948,12 +1969,16 @@ def plugins():
             session.plugins = []
     return dict(plugins=session.plugins["results"], app=request.args(0))
 
+
 def install_plugin():
     app = request.args(0)
     source = request.vars.source
     plugin = request.vars.plugin
     if not (source and app):
         raise HTTP(500, T("Invalid request"))
+    # make sure no XSS attacks in source
+    if not source.lower().split('://')[0] in ('http','https'):
+        raise HTTP(500, T("Invalid request"))
     form = SQLFORM.factory()
     result = None
     if form.process().accepted:
@@ -1969,5 +1994,5 @@ def install_plugin():
         else:
             session.flash = \
                 T('unable to install plugin "%s"', filename)
-        redirect(URL(f="plugins", args=[app,]))
+        redirect(URL(f="plugins", args=[app, ]))
     return dict(form=form, app=app, plugin=plugin, source=source)

applications/admin/models/access.py

@@ -4,6 +4,7 @@ import time
 from gluon import portalocker
 from gluon.admin import apath
 from gluon.fileutils import read_file
+from gluon.utils import web2py_uuid
 # ###########################################################
 # ## make sure administrator is on localhost or https
 # ###########################################################
@@ -49,15 +50,18 @@ except IOError:
 def verify_password(password):
     session.pam_user = None
     if DEMO_MODE:
-        return True
+        ret = True
     elif not _config.get('password'):
-        return False
+        ret - False
     elif _config['password'].startswith('pam_user:'):
         session.pam_user = _config['password'][9:].strip()
         import gluon.contrib.pam
-        return gluon.contrib.pam.authenticate(session.pam_user, password)
+        ret = gluon.contrib.pam.authenticate(session.pam_user, password)
     else:
-        return _config['password'] == CRYPT()(password)[0]
+        ret = _config['password'] == CRYPT()(password)[0]
+    if ret:
+        session.hmac_key = web2py_uuid()
+    return ret
 
 
 # ###########################################################
@@ -100,13 +104,12 @@ def write_hosts_deny(denied_hosts):
     portalocker.unlock(f)
     f.close()
 
-
 def login_record(success=True):
     denied_hosts = read_hosts_deny()
     val = (0, 0)
     if success and request.client in denied_hosts:
         del denied_hosts[request.client]
-    elif not success and not request.is_local:
+    elif not success:
         val = denied_hosts.get(request.client, (0, 0))
         if time.time() - val[1] < expiration_failed_logins \
             and val[0] >= allowed_number_of_attempts:
@@ -117,6 +120,11 @@ def login_record(success=True):
     write_hosts_deny(denied_hosts)
     return val[0]
 
+def failed_login_count():
+    denied_hosts = read_hosts_deny()
+    val = denied_hosts.get(request.client, (0, 0))
+    return val[0]
+
 
 # ###########################################################
 # ## session expiration

applications/admin/views/default/site.html

@@ -56,7 +56,7 @@
         {{pass}}
           </ul>
         </div>
-        {{=button_enable(URL('enable',args=a), a) if a!='admin' else ''}}
+        {{=button_enable(URL('enable',args=a, hmac_key=session.hmac_key), a) if a!='admin' else ''}}
       </td>
     </tr>
     {{pass}}

applications/examples/controllers/default.py

@@ -10,7 +10,7 @@ session.forget()
 cache_expire = not request.is_local and 300 or 0
 
 
-#@cache.action(time_expire=300, cache_model=cache.ram, quick='P')
+# @cache.action(time_expire=300, cache_model=cache.ram, quick='P')
 def index():
     return response.render()
 
@@ -19,14 +19,13 @@ def index():
 def what():
     import urllib
     try:
-        images = XML(urllib.urlopen(
+        images = XML(urllib.urlopen('http://www.web2py.com/poweredby/default/images').read())
-            'http://www.web2py.com/poweredby/default/images').read())
     except:
         images = []
     return response.render(images=images)
 
 
-#@cache.action(time_expire=300, cache_model=cache.ram, quick='P')
+# @cache.action(time_expire=300, cache_model=cache.ram, quick='P')
 def download():
     return response.render()
 
@@ -74,14 +73,15 @@ def license():
     filename = os.path.join(request.env.gluon_parent, 'LICENSE')
     return response.render(dict(license=MARKMIN(read_file(filename))))
 
+
 def version():
-    if request.args(0)=='raw':
+    if request.args(0) == 'raw':
         return request.env.web2py_version
     from gluon.fileutils import parse_version
     (a, b, c, pre_release, build) = parse_version(request.env.web2py_version)
-    return 'Version %i.%i.%i (%.4i-%.2i-%.2i %.2i:%.2i:%.2i) %s' % (
+    return 'Version %i.%i.%i (%.4i-%.2i-%.2i %.2i:%.2i:%.2i) %s' % \
-        a,b,c,build.year,build.month,build.day,
+           (a, b, c, build.year, build.month, build.day, build.hour, build.minute, build.second, pre_release)
-        build.hour,build.minute,build.second,pre_release)
+
 
 @cache.action(time_expire=300, cache_model=cache.ram, quick='P')
 def examples():

applications/examples/models/menu.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 
 response.menu = [
-    (T('Home'), False, URL('default', 'index')),
+    (T('Home'), request.controller == 'default' and request.function == 'index', URL('default', 'index')),
-    (T('About'), False, URL('default', 'what')),
+    (T('About'), request.controller == 'default' and request.function == 'what', URL('default', 'what')),
-    (T('Download'), False, URL('default', 'download')),
+    (T('Download'), request.controller == 'default' and request.function == 'download', URL('default', 'download')),
-    (T('Docs & Resources'), False, URL('default', 'documentation')),
+    (T('Docs & Resources'), request.controller == 'default' and request.function == 'documentation', URL('default', 'documentation')),
-    (T('Support'), False, URL('default', 'support')),
+    (T('Support'), request.controller == 'default' and request.function == 'support', URL('default', 'support')),
-    (T('Contributors'), False, URL('default', 'who'))]
+    (T('Contributors'), request.controller == 'default' and request.function == 'who', URL('default', 'who'))]
 
 #########################################################################
 ## Changes the menu active item

applications/examples/static/css/examples.css

@@ -0,0 +1,69 @@
+/* Gray the black as suggested by Anthony */
+h1,h2,h3,h4,h5,h6 {color: rgb(35, 35, 35); text-transform:none}
+.black {
+    color: rgb(35, 35, 35);
+    background-color: rgb(35, 35, 35);
+}
+
+/* Spacing between thead and tbody */
+/* Ref: http://stackoverflow.com/questions/9258754/spacing-between-thead-and-tbody */
+tbody:before {
+    content: "-";
+    display: block;
+    line-height: 1em;
+    color: transparent;
+}
+
+/* Improve buttons in download page */
+th, td {padding: 0}
+
+
+tbody tr:hover {background-color:transparent}
+tbody tr {border-bottom: none}
+p {text-align: left}
+p, li { line-height: 1.6em}
+
+/* Improve CODE() display though padding has no effect as some PRE are hardcoded somewhere can't find it */
+/* padding of 10px should make it... */
+pre {background-color: rgb(35, 35, 35)!important; border-radius:5px; color:white; padding: 10px}
+
+/* Improve buttons in download page */
+a.btn.btn180 {padding:10px; font-size:1.2em; width:200px}
+
+.menu .web2py-menu-active a {
+    color: #26a69a;
+}
+
+.spaced-vertical {
+    margin: 0 0.5em 0.5em 0;
+}
+
+.btn:hover,
+a.noeffect img:hover {
+    transition: scale .5s;
+    transform: scale(1.05);
+}
+
+.btn,
+a.noeffect img {
+    transition: all .2s ease-in-out;
+}
+
+/* Lower saturation of color #26a69a - 20 points lower */
+/* The below change to color #26a69a should come before other color change or they override all buttons background-color */
+/* The color should maybe change at stupid.css level as it herited from there also in stupid.css it would be better
+   to define this color at one place actually color is defined all over the place */
+a {color:#47a69d}
+.btn, button, [type=button], [type=submit] {background-color:#47a69d}
+.progress .determinate {background-color:#47a69d}
+.progress .indeterminate {background-color:#47a69d}
+a:not(.btn):not(.noeffect):hover {color:#47a69d}
+a:not(.btn):not(.noeffect):after {background-color:#47a69d}
+.tags > span {background-color:#47a69d}
+.tags.dismissible > span.off:hover {background-color:#47a69d}
+.aquamarine{background-color:#47a69d}
+
+/* Lower the saturation of 20 points */
+.green {background-color: #58cc65}
+.yellow {background-color: #ffe333}
+.red {background-color: #cc4229}

applications/examples/static/css/stupid.css

@@ -5,8 +5,9 @@
 ************/
 
 /*** basic styles ***/
-*, *:after, *:before {border:0; margin:0; padding:0; -webkit-box-sizing:border-box; -moz-box-sizing:border-box; box-sizing:border-box}
+html {box-sizing:border-box;}
-html, body {max-width: 100vw !important; overflow-x: hidden !important}
+*, *:after, *:before {border:0; margin:0; padding:0; box-sizing:inherit;}
+html, body {max-width: 100vw; overflow-x: hidden}
 body {font-family:"HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif}
 p, li {margin-bottom:0.5em}
 p {text-align:justify}
@@ -14,7 +15,7 @@ label, strong {font-weight:bold}
 ul {list-style-type:none; padding-left:20px}
 a {text-decoration:none; color:#26a69a; white-space:nowrap}
 a:hover {cursor:pointer}
-h1,h2,h3,h4,h5,h6{font-weight:strong; text-transform:uppercase}
+h1,h2,h3,h4,h5,h6{font-weight:bold; text-transform:uppercase}
 h1{font-size:4em; margin:1.0em 0 0.25em 0}
 h2{font-size:2.4em; margin:0.9em 0 0.25em 0}
 h3{font-size:1.8em; margin:0.8em 0 0.25em 0}
@@ -23,15 +24,13 @@ h5{font-size:1.4em; margin:0.6em 0 0.25em 0}
 h6{font-size:1.2em; margin:0.5em 0 0.25em 0}
 table {border-collapse:collapse}
 tbody tr:hover {background-color:#fbf6d9}
-td, th {padding:5px; vertical-align:top; text-align:left; border:0}
 thead tr {background-color:#f1f1f1}
 tbody tr {border-bottom:2px solid #f1f1f1}
-th {font-weight:string; padding:5px; vertical-align:bottom; text-align:left}
+td, th {padding: 5px; text-align: left; vertical-align:top}
 thead th {vertical-align:bottom}
-tbody th {vertical-align:top}
+header, main, footer {display:block; with:100%} /* IE fix */
-header, footer {with:100%}
 
-@media (max-width:599px) {
+@media all and (max-width:599px) {
   h1{font-size:2em}
   h2{font-size:1.8em}
   h3{font-size:1.6em}
@@ -41,20 +40,20 @@ header, footer {with:100%}
 }
 
 /*** buttons ***/
-.btn, button, [type=button], [type=submit] {padding:0.5em 1em !important; margin:0 0.5em 0.5em 0; line-height:2.4em; background-color:#26a69a; color:white}
+.btn, button, [type=button], [type=submit] {padding:0.5em 1em; margin:0 0.5em 0.5em 0; display:inline-block; background-color:#26a69a; color:white}
 .btn:hover, button:hover, [type=button]:hover, [type=submit]:hover {box-shadow:0 0 10px #666; text-decoration:none; cursor:pointer}
-.btn.small, table .btn {padding:0.25em 0.5em !important; font-size:0.8em; line-height:1.5em}
+.btn.small, table .btn {padding:0.25em 0.5em; font-size:0.8em}
-.btn.large {padding:1em 2em !important; font-size:1.2em; line-height:4em}
+.btn.large {padding:1em 2em; font-size:1.2em}
 .btn.oval {border-radius:50%}
 
 /*** helpers ***/
 .rounded {-moz-border-radius:5px; border-radius:5px}
-.padded {padding:10px 20px !important}
+.padded {padding:10px 20px}
-.center {text-align:center !important; margin-left:auto; margin-right:auto}
+.center {text-align:center; margin-left:auto; margin-right:auto}
 .center>div {text-align:left}
 .right {right:0; text-align:right}
-.middle div {vertical-align:middle !important}
+.middle div {vertical-align:middle}
-.bottom div {vertical-align:bottom !important}
+.bottom div {vertical-align:bottom}
 .xscroll {overflow-x:scroll}
 .yscroll {overflow-y:scroll}
 .nowrap {white-space:nowrap; overflow-x:hidden}
@@ -69,16 +68,17 @@ header, footer {with:100%}
 input:not([type]), input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]), [type=file]:before {outline:none; padding:0.5em 1em; margin:0.5px; border-bottom:1px solid #ddd; width:100%}
 textarea {width:100%; border:1px solid #ddd; padding:4px 8px; outline:none; outline:none}
 select {-webkit-appearance:none; outline:none; padding:0.5em 1em; border-radius:0; margin:0.5px; border-bottom:1px solid #ddd; width:100%;background-color:transparent}
-input, textarea, select, button {font-size:12px}
+input, textarea, select, button, .btn {font-size:12px}
 input:not([type]):hover, input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]):hover, select:hover, textarea:hover {background-color:#fbf6d9; transition:background-color 1s ease}
-input:invalid, input.error {background:#cc1f00!important;color:white}
+input:invalid, input.error {background:#cc1f00;color:white}
 
 /*** grid ***/
 .container {margin-right:-20px}
-.container>.quarter, .container>.half, .container>.third, .container>.twothirds, .container>.threequarters, .container>.fill{display:inline-block; padding:5px 20px 5px 0; vertical-align:top}
+.container>.quarter, .container>.half, .container>.third, .container>.twothirds, .container>.threequarters {display:inline-block; padding: 0 20px 0 0; vertical-align:top}
-.container>.fill {width:100%; margin-right:-20px}
+.container>.fill{display: inline-block}
 .container img, .container video {max-width:100%}
-@media (min-width:800px) {
+
+@media all and (min-width:800px) {
   .max900 {max-width:900px; margin-left:auto; margin-right:auto}
   .quarter {width:25%; margin-right:-5px}
   .half {width:50%; margin-right:-10px}
@@ -86,7 +86,7 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
   .twothirds {width:66.66%; margin-right:-13.33px}
   .threequarters {width:75%; margin-right:-15px}
 }
-@media (min-width:600px) and (max-width:799px) {
+@media all and (min-width:600px) and (max-width:799px) {
   .quarter.compressible {width:25%; margin-right:-5px}
   .half.compressible {width:50%; margin-right:-10px}
   .threequarters.compressible {width:75%; margin-right:-15px}
@@ -95,7 +95,7 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
   .twothirds {width:66.66%; margin-right:-13.33px}
   label.quarter:not(.compressible).right, label.half:not(.compressible).right, label.threequarters:not(.compressible).right   {float:left; text-align:left}
 }
-@media (max-width:599px) {  
+@media all and (max-width:599px) {  
   .quarter:not(.compressible), .half:not(.compressible), .third:not(.compressible), .twothirds:not(.compressible), .threequarters:not(.compressible) {width:100%;}
   label.quarter:not(.compressible).right, label.half:not(.compressible).right, label.threequarters:not(.compressible).right,
   label.third:not(.compressible).right, label.twothirds:not(.compressible).right   {float:left; text-align:left}
@@ -115,7 +115,7 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
   display:block;
   width:120%;
   background-color:#acece6;
-  border-radius:0 !important;
+  border-radius:0;
   background-clip:padding-box;
   overflow:hidden;
 }
@@ -193,17 +193,17 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
 .menu ul ul {top:0; left:80%; z-index:1100}
 .menu li:hover > ul {visibility:visible; opacity:1}
 .menu>li>ul>li:first-child:before{content:''; position:absolute; width:1px; height:1px; border:10px solid transparent; left:50px; top:-20px; margin-left:-10px; border-bottom-color:white}
-.menu.dark ul {background:black; border:1px solid black}
+.menu.dark ul {background:#111111; border:1px solid #111111}
 .menu.dark ul a {color:white}
-.menu.dark>li>ul>li:first-child:before{border-bottom-color:black}
+.menu.dark>li>ul>li:first-child:before{border-bottom-color:#111111}
 
-@media (max-width:599px) {
+@media all and (max-width:599px) {
   header .menu li, header .menu ul {width: 100%}  
   header .menu.right {float:left; text-align:left}
   header .menu ul ul {top:2.5em; left:-1px}
 }
 
-@media (min-width:600px) {
+@media all and (min-width:600px) {
   .ham {display:none!important}
   .burger.accordion * {max-height:1000px; overflow:visible}
 }
@@ -268,8 +268,8 @@ a:not(.btn):not(.noeffect):after {
 [data-tooltip]:before, [data-tooltip]:after {display:none; position:absolute; top:0}
 [data-tooltip]:hover:after,[data-tooltip]:hover:before {display:block}
 [data-tooltip]:hover:before {
-  border-bottom:.6em solid black;
+  border-bottom:.6em solid #111111;
-  border-bottom:.6em solid black;
+  border-bottom:.6em solid #111111;
   border-left:7px solid transparent;
   border-right:7px solid transparent;
   content:"";
@@ -288,7 +288,7 @@ a:not(.btn):not(.noeffect):after {
   content:attr(data-tooltip);
   left:0;
   top:2px;
-  margin-left:-20;
+  margin-left:-20px;
   margin-top:1.5em;
   padding:5px 15px;
   white-space:pre-wrap;
@@ -339,9 +339,6 @@ a:not(.btn):not(.noeffect):after {
   transform: rotateY( 180deg );
 }
 
-/*** colors from http://clrs.cc/ ***/
-.navy{background-color:#001f3f!important;color:white}.blue{background-color:#0074d9!important;color:white}.aqua{background-color:#7fdbff!important;color:black}.teal{background-color:#39cccc!important;color:white}.olive{background-color:#3d9970!important;color:white}.green{background-color:#2ecc40!important;color:white}.aquamarine{background-color:#26a69a!important;color:white}.lime{background-color:#01ff70!important;color:black}.yellow{background-color:#ffdc00!important;color:black}.orange{background-color:#ff851b!important;color:white}.red{background-color:#cc1f00!important;color:white}.fuchsia{background-color:#f012be!important;color:white}.pink{background-color:#ee6e73!important;color:white}.purple{background-color:#b10dc9!important;color:white}.maroon{background-color:#85144b!important;color:white}.white{background-color:#fff!important;color:black;-webkit-box-shadow:inset 0px 0px 0px 1px #ddd;-moz-box-shadow:inset 0px 0px 0px 1px #ddd;box-shadow:inset 0px 0px 0px 1px #ddd}.gray{background-color:#aaa!important;color:white}.silver{background-color:#f1f1f1!important;color:black}.black{background-color:#000!important;color:white}.glass{background:rgba(255,255,255,0.5)!important;color:black}
-
 /**** tags ****/
 .tags > span {
   padding: 4px 9px;
@@ -350,10 +347,13 @@ a:not(.btn):not(.noeffect):after {
   background-color: #26a69a;
   border-radius: 5px;
   font-size:12px;
-  margin: 5px 5px 5px 0 !important;
+  margin: 2px 5px 2px 0;
-  line-height: 32px;
+  display: inline-block;
 }
 .tags.dismissible > span:hover {opacity: 0.5}
 .tags.dismissible > span:not(.off):after {content:" ✕"}
 .tags > span.off {background-color: #ccc}
 .tags.dismissible > span.off:hover {background-color:#26a69a}
+
+/*** colors from http://clrs.cc/ ***/
+.navy{background-color:#001f3f;color:white}.blue{background-color:#0074d9;color:white}.aqua{background-color:#7fdbff;color:#111111}.teal{background-color:#39cccc;color:white}.olive{background-color:#3d9970;color:white}.green{background-color:#2ecc40;color:white}.aquamarine{background-color:#26a69a;color:white}.lime{background-color:#01ff70;color:#111111}.yellow{background-color:#ffdc00;color:#111111}.orange{background-color:#ff851b;color:white}.red{background-color:#cc1f00;color:white}.fuchsia{background-color:#f012be;color:white}.pink{background-color:#ee6e73;color:white}.purple{background-color:#b10dc9;color:white}.maroon{background-color:#85144b;color:white}.white{background-color:#fff;color:#111111;-webkit-box-shadow:inset 0px 0px 0px 1px #ddd;-moz-box-shadow:inset 0px 0px 0px 1px #ddd;box-shadow:inset 0px 0px 0px 1px #ddd}.gray{background-color:#aaa;color:white}.silver{background-color:#f1f1f1;color:#111111}.black{background-color:#111111;color:white}.glass{background:rgba(255,255,255,0.5);color:#111111}

applications/examples/views/default/download.html

@@ -17,7 +17,7 @@
     <tbody>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="http://www.web2py.com/examples/static/web2py_win.zip">For Windows</a>
+          <a class="btn btn180 rounded green" href="http://www.web2py.com/examples/static/web2py_win.zip">For Windows</a>
         </td>
         <td>
           <a class="btn btn180 rounded yellow" href="http://www.web2py.com/examples/static/nightly/web2py_win.zip">For Windows</a>
@@ -28,7 +28,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="http://www.web2py.com/examples/static/web2py_osx.zip">For Mac</a>
+          <a class="btn btn180 rounded green" href="http://www.web2py.com/examples/static/web2py_osx.zip">For Mac</a>
         </td>
         <td>
           <a class="btn btn180 rounded yellow" href="http://www.web2py.com/examples/static/nightly/web2py_osx.zip">For Mac</a>
@@ -37,7 +37,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="http://www.web2py.com/examples/static/web2py_src.zip">Source Code</a>
+          <a class="btn btn180 rounded green" href="http://www.web2py.com/examples/static/web2py_src.zip">Source Code</a>
         </td>
         <td>
           <a class="btn btn180 rounded yellow" href="http://www.web2py.com/examples/static/nightly/web2py_src.zip">Source Code</a>
@@ -48,7 +48,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
+          <a class="btn btn180 rounded green" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
         </td>
         <td>
           <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>
@@ -69,9 +69,9 @@
 <h3>Instructions</h3>
 <p>After download, unzip it and click on web2py.exe (windows) or web2py.app (osx).
   To run from source, type:</p>
-{{=CODE("python2.7 web2py.py",language=None,counter='>',_class='boxCode')}}
+{{=CODE("python2.7 web2py.py", language=None, counter='>', _class='boxCode')}}
 <p>or for more info type:</p>
-{{=CODE("python2.7 web2py.py -h",language=None,counter='>',_class='boxCode')}}
+{{=CODE("python2.7 web2py.py -h", language=None, counter='>', _class='boxCode')}}
 
 
 <h3>Caveats</h3>
@@ -84,7 +84,7 @@
 <p>Applications built with web2py can be released under any license the author wishes as long they do not contain web2py code. They can link unmodified web2py libraries and they can be distributed with official web2py binaries. In particular web2py applications can be distributed in closed source. The admin interface provides a button to byte-code compile.</p>
 <p>It is fine to distribute web2py (source or compiled) with your applications as long as you make it clear in the license where your application ends and web2py starts.</p>
 <p>web2py is copyrighted by Massimo Di Pierro. The web2py trademark is owned by Massimo Di Pierro.</p>
-<a class="btn btn-small" href="{{=URL('license')}}">read more</a>
+<a class="btn btn-small rounded" href="{{=URL('license')}}">read more</a>
 
 <h3>Artwork</h3>
 <center>

applications/examples/views/default/index.html

@@ -3,22 +3,22 @@
 <div class="container">
   <div class="twothirds">
     <div class="padded">
-      <h3>web2py<sup>TM</sup> Web Framework</h3>
+      <h3><img src="{{=URL('static/images', 'web2py_logo.png')}}"> Web Framework</h3>
       <p>Free open source full-stack framework for rapid development of fast, scalable, <a href="http://www.web2py.com/book/default/chapter/01#Security" target="_blank">secure</a> and portable database-driven web-based applications. Written and programmable in <a href="http://www.python.org" target="_blank">Python</a>.</p>
       <table width="100%">
 	<tr>
 	  <td>
-	    <a href="http://web2py.com/book">
+	    <a class="noeffect" href="http://web2py.com/book">
 	      <img src="{{=URL('static','images/book-5th.png')}}" />
 	    </a>
 	  </td>
 	  <td>
-	    <a href="https://vimeo.com/album/3016728">
+	    <a class="noeffect" href="https://vimeo.com/album/3016728">
 	      <img src="{{=URL('static','images/videos.png')}}" />
 	    </a>
 	  </td>
 	  <td>
-	    <a href="http://link.packtpub.com/SUlnrN">
+	    <a class="noeffect" href="http://link.packtpub.com/SUlnrN">
 	      <img src="{{=URL('static','images/book-recipes.png')}}" />
 	    </a>
 	  </td>
@@ -29,8 +29,8 @@
   </div>
   <div class="third">
     <div class="padded center">
-      <a href="http://www.infoworld.com/slideshow/24605/infoworlds-2012-technology-of-the-year-award-winners-183313#slide23">
+      <a class="noeffect" href="http://www.infoworld.com/slideshow/24605/infoworlds-2012-technology-of-the-year-award-winners-183313#slide23">
-        <img src="{{=URL('static','images/infoworld2012.jpeg')}}">
+        <img class="spaced-vertical" src="{{=URL('static','images/infoworld2012.jpeg')}}">
       </a>
       <a class="btn rounded red fill" href="{{=URL('download')}}">
         Download Now

applications/examples/views/default/support.html

@@ -32,7 +32,6 @@
       <li><a target="_blank" href="http://www.albendas.com">Albendas</a> (Spain)</li>
       <li><a target="_blank" href="http://www.appliedobjects.com">Applied Objects</a> (New Zealand)</li>
       <li><a target="_blank" href="http://www.sistemasagiles.com.ar/">Sistemas Ágiles</a> ("Agile Systems") (Argentina)</li>
-      <li><a target="_blank" href="http://www.definescope.com/en/services/consulting/">DefineScope</a> (Portugal)</li>
       <li><a target="_blank" href="http://www.tasko.it/">Tasko</a> (Italy)</li>
       <li><a target="_blank" href="http://www.geekondemand.it/"> GeekOnDemand</a> (Italy)</li>
       <li><a target="_blank" href="http://stifix.com"> Stifix</a> (Indonesia)</li>

applications/examples/views/default/who.html

@@ -82,6 +82,7 @@
 </li><li>Keith Yang (openid)
 </li><li><a href="http://dev.s-cubism.com/web2py_plugins">Kenji Hosoda</a> (plugins)
 </li><li>Kyle Smith (javascript)
+</li><li><a href="https://github.com/leonelcamara">Leonel Câmara</a>
 </li><li><a href="http://blog.donews.com/limodou/">Limodou</a> (winservice)
 </li><li><a href="https://github.com/lucasdavila">Lucas D'Ávila</a>
 </li><li>Marc Abramowitz (tests and travis continuous integration) 
@@ -99,6 +100,7 @@
 </li><li>Michael Willis (shell)
 </li><li>Michele Comitini (facebook)
 </li><li>Michael Toomim (scheduler)
+</li><li>Narendra Bhati (security)
 </li><li>Nathan Freeze (admin design, IS_STRONG, DAL features, <a href="http://web2pyslices.com">web2pyslices.com</a>)
 </li><li>Niall Sweeny (MSSQL support)
 </li><li>Niccolo Polo (epydoc)

applications/examples/views/layout.html

@@ -1,22 +1,17 @@
+<!DOCTYPE html>
 <html>
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="apple-mobile-web-app-capable" content="yes" />
-    <link href="{{=URL('static','css/stupid.css')}}" rel="stylesheet" type="text/css"/>
     <link href="{{=URL('static','css/calendar.css')}}" rel="stylesheet" type="text/css"/>
     <link href="{{=URL('static','css/web2py.css')}}" rel="stylesheet" type="text/css"/>
+    <link href="{{=URL('static','css/stupid.css')}}" rel="stylesheet" type="text/css"/>
+    <link href="{{=URL('static','css/examples.css')}}" rel="stylesheet" type="text/css"/>
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css">
-    <style>
+    <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
-      th, td {color: black!important}
+    <link rel="apple-touch-icon" href="{{=URL('static','images/favicon.png')}}">
-      tbody tr:hover {background-color:transparent}
-      tbody tr {border-bottom: none}
-      p {text-align: left}
-      p, li { line-height: 1.6em}
-      pre {background-color: black!important;border-radius:5px; color:white; padding:10px}
-      a.btn.btn180 {padding:20px; font-size:1.2em; width:200px!important}      
-    </style>
     {{
     left_sidebar_enabled = globals().get('left_sidebar_enabled', False)
     right_sidebar_enabled = globals().get('right_sidebar_enabled', False)
@@ -29,7 +24,7 @@
     <header class="black padded">      
       <div class="container middle max900">
         <div class="fill middle">                    
-          <label class="ham padded fa fa-bars" for="menu"></label>
+          <label class="ham" for="menu"><i class="fa fa-bars padded"></i></label>
           <div class="burger accordion">
             <input type="checkbox" id="menu"/>
             {{=MENU(response.menu,_class='menu')}}

applications/welcome/controllers/default.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 # this file is released under public domain and you can use without limitations
 
-#########################################################################
+# -------------------------------------------------------------------------
-## This is a sample controller
+# This is a sample controller
-## - index is the default action of any application
+# - index is the default action of any application
-## - user is required for authentication and authorization
+# - user is required for authentication and authorization
-## - download is for downloading files uploaded in the db (does streaming)
+# - download is for downloading files uploaded in the db (does streaming)
-#########################################################################
+# -------------------------------------------------------------------------
+
 
 def index():
     """

applications/welcome/models/db.py

@@ -1,60 +1,84 @@
 # -*- coding: utf-8 -*-
 
-#########################################################################
+# -------------------------------------------------------------------------
-## This scaffolding model makes your app work on Google App Engine too
+# This scaffolding model makes your app work on Google App Engine too
-## File is released under public domain and you can use without limitations
+# File is released under public domain and you can use without limitations
-#########################################################################
+# -------------------------------------------------------------------------
 
 if request.global_settings.web2py_version < "2.14.1":
     raise HTTP(500, "Requires web2py 2.13.3 or newer")
 
-## if SSL/HTTPS is properly configured and you want all HTTP requests to
+# -------------------------------------------------------------------------
-## be redirected to HTTPS, uncomment the line below:
+# if SSL/HTTPS is properly configured and you want all HTTP requests to
+# be redirected to HTTPS, uncomment the line below:
+# -------------------------------------------------------------------------
 # request.requires_https()
 
-## app configuration made easy. Look inside private/appconfig.ini
+# -------------------------------------------------------------------------
+# app configuration made easy. Look inside private/appconfig.ini
+# -------------------------------------------------------------------------
 from gluon.contrib.appconfig import AppConfig
-## once in production, remove reload=True to gain full speed
+
+# -------------------------------------------------------------------------
+# once in production, remove reload=True to gain full speed
+# -------------------------------------------------------------------------
 myconf = AppConfig(reload=True)
 
 if not request.env.web2py_runtime_gae:
-    ## if NOT running on Google App Engine use SQLite or other DB
+    # ---------------------------------------------------------------------
+    # if NOT running on Google App Engine use SQLite or other DB
+    # ---------------------------------------------------------------------
     db = DAL(myconf.get('db.uri'),
-             pool_size = myconf.get('db.pool_size'),
+             pool_size=myconf.get('db.pool_size'),
-             migrate_enabled = myconf.get('db.migrate'),
+             migrate_enabled=myconf.get('db.migrate'),
-             check_reserved = ['all'])
+             check_reserved=['all'])
 else:
-    ## connect to Google BigTable (optional 'google:datastore://namespace')
+    # ---------------------------------------------------------------------
+    # connect to Google BigTable (optional 'google:datastore://namespace')
+    # ---------------------------------------------------------------------
     db = DAL('google:datastore+ndb')
-    ## store sessions and tickets there
+    # ---------------------------------------------------------------------
+    # store sessions and tickets there
+    # ---------------------------------------------------------------------
     session.connect(request, response, db=db)
-    ## or store session in Memcache, Redis, etc.
+    # ---------------------------------------------------------------------
-    ## from gluon.contrib.memdb import MEMDB
+    # or store session in Memcache, Redis, etc.
-    ## from google.appengine.api.memcache import Client
+    # from gluon.contrib.memdb import MEMDB
-    ## session.connect(request, response, db = MEMDB(Client()))
+    # from google.appengine.api.memcache import Client
+    # session.connect(request, response, db = MEMDB(Client()))
+    # ---------------------------------------------------------------------
 
-## by default give a view/generic.extension to all actions from localhost
+# -------------------------------------------------------------------------
-## none otherwise. a pattern can be 'controller/function.extension'
+# by default give a view/generic.extension to all actions from localhost
+# none otherwise. a pattern can be 'controller/function.extension'
+# -------------------------------------------------------------------------
 response.generic_patterns = ['*'] if request.is_local else []
-## choose a style for forms
+# -------------------------------------------------------------------------
+# choose a style for forms
+# -------------------------------------------------------------------------
 response.formstyle = myconf.get('forms.formstyle')  # or 'bootstrap3_stacked' or 'bootstrap2' or other
 response.form_label_separator = myconf.get('forms.separator') or ''
 
-
+# -------------------------------------------------------------------------
-## (optional) optimize handling of static files
+# (optional) optimize handling of static files
+# -------------------------------------------------------------------------
 # response.optimize_css = 'concat,minify,inline'
 # response.optimize_js = 'concat,minify,inline'
-## (optional) static assets folder versioning
+
+# -------------------------------------------------------------------------
+# (optional) static assets folder versioning
+# -------------------------------------------------------------------------
 # response.static_version = '0.0.0'
-#########################################################################
+
-## Here is sample code if you need for
+# -------------------------------------------------------------------------
-## - email capabilities
+# Here is sample code if you need for
-## - authentication (registration, login, logout, ... )
+# - email capabilities
-## - authorization (role based authorization)
+# - authentication (registration, login, logout, ... )
-## - services (xml, csv, json, xmlrpc, jsonrpc, amf, rss)
+# - authorization (role based authorization)
-## - old style crud actions
+# - services (xml, csv, json, xmlrpc, jsonrpc, amf, rss)
-## (more options discussed in gluon/tools.py)
+# - old style crud actions
-#########################################################################
+# (more options discussed in gluon/tools.py)
+# -------------------------------------------------------------------------
 
 from gluon.tools import Auth, Service, PluginManager
 
@@ -63,10 +87,14 @@ auth = Auth(db, host_names=myconf.get('host.names'))
 service = Service()
 plugins = PluginManager()
 
-## create all tables needed by auth if not custom tables
+# -------------------------------------------------------------------------
+# create all tables needed by auth if not custom tables
+# -------------------------------------------------------------------------
 auth.define_tables(username=False, signature=False)
 
-## configure email
+# -------------------------------------------------------------------------
+# configure email
+# -------------------------------------------------------------------------
 mail = auth.settings.mailer
 mail.settings.server = 'logging' if request.is_local else myconf.get('smtp.server')
 mail.settings.sender = myconf.get('smtp.sender')
@@ -74,27 +102,31 @@ mail.settings.login = myconf.get('smtp.login')
 mail.settings.tls = myconf.get('smtp.tls') or False
 mail.settings.ssl = myconf.get('smtp.ssl') or False
 
-## configure auth policy
+# -------------------------------------------------------------------------
+# configure auth policy
+# -------------------------------------------------------------------------
 auth.settings.registration_requires_verification = False
 auth.settings.registration_requires_approval = False
 auth.settings.reset_password_requires_verification = True
 
-#########################################################################
+# -------------------------------------------------------------------------
-## Define your tables below (or better in another model file) for example
+# Define your tables below (or better in another model file) for example
-##
+#
-## >>> db.define_table('mytable',Field('myfield','string'))
+# >>> db.define_table('mytable', Field('myfield', 'string'))
-##
+#
-## Fields can be 'string','text','password','integer','double','boolean'
+# Fields can be 'string','text','password','integer','double','boolean'
-##       'date','time','datetime','blob','upload', 'reference TABLENAME'
+#       'date','time','datetime','blob','upload', 'reference TABLENAME'
-## There is an implicit 'id integer autoincrement' field
+# There is an implicit 'id integer autoincrement' field
-## Consult manual for more options, validators, etc.
+# Consult manual for more options, validators, etc.
-##
+#
-## More API examples for controllers:
+# More API examples for controllers:
-##
+#
-## >>> db.mytable.insert(myfield='value')
+# >>> db.mytable.insert(myfield='value')
-## >>> rows=db(db.mytable.myfield=='value').select(db.mytable.ALL)
+# >>> rows = db(db.mytable.myfield == 'value').select(db.mytable.ALL)
-## >>> for row in rows: print row.id, row.myfield
+# >>> for row in rows: print row.id, row.myfield
-#########################################################################
+# -------------------------------------------------------------------------
 
-## after defining tables, uncomment below to enable auditing
+# -------------------------------------------------------------------------
+# after defining tables, uncomment below to enable auditing
+# -------------------------------------------------------------------------
 # auth.enable_record_versioning(db)

applications/welcome/models/menu.py

@@ -1,28 +1,32 @@
 # -*- coding: utf-8 -*-
 # this file is released under public domain and you can use without limitations
 
-#########################################################################
+# ----------------------------------------------------------------------------------------------------------------------
-## Customize your APP title, subtitle and menus here
+# Customize your APP title, subtitle and menus here
-#########################################################################
+# ----------------------------------------------------------------------------------------------------------------------
 
-response.logo = A(B('web',SPAN(2),'py'),XML('™ '),
+response.logo = A(B('web', SPAN(2), 'py'), XML('™ '),
-                  _class="navbar-brand",_href="http://www.web2py.com/",
+                  _class="navbar-brand", _href="http://www.web2py.com/",
                   _id="web2py-logo")
-response.title = request.application.replace('_',' ').title()
+response.title = request.application.replace('_', ' ').title()
 response.subtitle = ''
 
-## read more at http://dev.w3.org/html5/markup/meta.name.html
+# ----------------------------------------------------------------------------------------------------------------------
+# read more at http://dev.w3.org/html5/markup/meta.name.html
+# ----------------------------------------------------------------------------------------------------------------------
 response.meta.author = myconf.get('app.author')
 response.meta.description = myconf.get('app.description')
 response.meta.keywords = myconf.get('app.keywords')
 response.meta.generator = myconf.get('app.generator')
 
-## your http://google.com/analytics id
+# ----------------------------------------------------------------------------------------------------------------------
+# your http://google.com/analytics id
+# ----------------------------------------------------------------------------------------------------------------------
 response.google_analytics_id = None
 
-#########################################################################
+# ----------------------------------------------------------------------------------------------------------------------
-## this is the main application menu add/remove items as required
+# this is the main application menu add/remove items as required
-#########################################################################
+# ----------------------------------------------------------------------------------------------------------------------
 
 response.menu = [
     (T('Home'), False, URL('default', 'index'), [])
@@ -30,109 +34,118 @@ response.menu = [
 
 DEVELOPMENT_MENU = True
 
-#########################################################################
+
-## provide shortcuts for development. remove in production
+# ----------------------------------------------------------------------------------------------------------------------
-#########################################################################
+# provide shortcuts for development. remove in production
+# ----------------------------------------------------------------------------------------------------------------------
 
 def _():
+    # ------------------------------------------------------------------------------------------------------------------
     # shortcuts
+    # ------------------------------------------------------------------------------------------------------------------
     app = request.application
     ctr = request.controller
+    # ------------------------------------------------------------------------------------------------------------------
     # useful links to internal and external resources
+    # ------------------------------------------------------------------------------------------------------------------
     response.menu += [
         (T('My Sites'), False, URL('admin', 'default', 'site')),
-          (T('This App'), False, '#', [
+        (T('This App'), False, '#', [
-              (T('Design'), False, URL('admin', 'default', 'design/%s' % app)),
+            (T('Design'), False, URL('admin', 'default', 'design/%s' % app)),
-              LI(_class="divider"),
+            LI(_class="divider"),
-              (T('Controller'), False,
+            (T('Controller'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/controllers/%s.py' % (app, ctr))),
+                 'admin', 'default', 'edit/%s/controllers/%s.py' % (app, ctr))),
-              (T('View'), False,
+            (T('View'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/views/%s' % (app, response.view))),
+                 'admin', 'default', 'edit/%s/views/%s' % (app, response.view))),
-              (T('DB Model'), False,
+            (T('DB Model'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/models/db.py' % app)),
+                 'admin', 'default', 'edit/%s/models/db.py' % app)),
-              (T('Menu Model'), False,
+            (T('Menu Model'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/models/menu.py' % app)),
+                 'admin', 'default', 'edit/%s/models/menu.py' % app)),
-              (T('Config.ini'), False,
+            (T('Config.ini'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/private/appconfig.ini' % app)),
+                 'admin', 'default', 'edit/%s/private/appconfig.ini' % app)),
-              (T('Layout'), False,
+            (T('Layout'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/views/layout.html' % app)),
+                 'admin', 'default', 'edit/%s/views/layout.html' % app)),
-              (T('Stylesheet'), False,
+            (T('Stylesheet'), False,
-               URL(
+             URL(
-               'admin', 'default', 'edit/%s/static/css/web2py-bootstrap3.css' % app)),
+                 'admin', 'default', 'edit/%s/static/css/web2py-bootstrap3.css' % app)),
-              (T('Database'), False, URL(app, 'appadmin', 'index')),
+            (T('Database'), False, URL(app, 'appadmin', 'index')),
-              (T('Errors'), False, URL(
+            (T('Errors'), False, URL(
-               'admin', 'default', 'errors/' + app)),
+                'admin', 'default', 'errors/' + app)),
-              (T('About'), False, URL(
+            (T('About'), False, URL(
-               'admin', 'default', 'about/' + app)),
+                'admin', 'default', 'about/' + app)),
-              ]),
+        ]),
-          ('web2py.com', False, '#', [
+        ('web2py.com', False, '#', [
-             (T('Download'), False,
+            (T('Download'), False,
-              'http://www.web2py.com/examples/default/download'),
+             'http://www.web2py.com/examples/default/download'),
-             (T('Support'), False,
+            (T('Support'), False,
-              'http://www.web2py.com/examples/default/support'),
+             'http://www.web2py.com/examples/default/support'),
-             (T('Demo'), False, 'http://web2py.com/demo_admin'),
+            (T('Demo'), False, 'http://web2py.com/demo_admin'),
-             (T('Quick Examples'), False,
+            (T('Quick Examples'), False,
-              'http://web2py.com/examples/default/examples'),
+             'http://web2py.com/examples/default/examples'),
-             (T('FAQ'), False, 'http://web2py.com/AlterEgo'),
+            (T('FAQ'), False, 'http://web2py.com/AlterEgo'),
-             (T('Videos'), False,
+            (T('Videos'), False,
-              'http://www.web2py.com/examples/default/videos/'),
+             'http://www.web2py.com/examples/default/videos/'),
-             (T('Free Applications'),
+            (T('Free Applications'),
-              False, 'http://web2py.com/appliances'),
+             False, 'http://web2py.com/appliances'),
-             (T('Plugins'), False, 'http://web2py.com/plugins'),
+            (T('Plugins'), False, 'http://web2py.com/plugins'),
-             (T('Recipes'), False, 'http://web2pyslices.com/'),
+            (T('Recipes'), False, 'http://web2pyslices.com/'),
-             ]),
+        ]),
-          (T('Documentation'), False, '#', [
+        (T('Documentation'), False, '#', [
-             (T('Online book'), False, 'http://www.web2py.com/book'),
+            (T('Online book'), False, 'http://www.web2py.com/book'),
-             LI(_class="divider"),
+            LI(_class="divider"),
-             (T('Preface'), False,
+            (T('Preface'), False,
-              'http://www.web2py.com/book/default/chapter/00'),
+             'http://www.web2py.com/book/default/chapter/00'),
-             (T('Introduction'), False,
+            (T('Introduction'), False,
-              'http://www.web2py.com/book/default/chapter/01'),
+             'http://www.web2py.com/book/default/chapter/01'),
-             (T('Python'), False,
+            (T('Python'), False,
-              'http://www.web2py.com/book/default/chapter/02'),
+             'http://www.web2py.com/book/default/chapter/02'),
-             (T('Overview'), False,
+            (T('Overview'), False,
-              'http://www.web2py.com/book/default/chapter/03'),
+             'http://www.web2py.com/book/default/chapter/03'),
-             (T('The Core'), False,
+            (T('The Core'), False,
-              'http://www.web2py.com/book/default/chapter/04'),
+             'http://www.web2py.com/book/default/chapter/04'),
-             (T('The Views'), False,
+            (T('The Views'), False,
-              'http://www.web2py.com/book/default/chapter/05'),
+             'http://www.web2py.com/book/default/chapter/05'),
-             (T('Database'), False,
+            (T('Database'), False,
-              'http://www.web2py.com/book/default/chapter/06'),
+             'http://www.web2py.com/book/default/chapter/06'),
-             (T('Forms and Validators'), False,
+            (T('Forms and Validators'), False,
-              'http://www.web2py.com/book/default/chapter/07'),
+             'http://www.web2py.com/book/default/chapter/07'),
-             (T('Email and SMS'), False,
+            (T('Email and SMS'), False,
-              'http://www.web2py.com/book/default/chapter/08'),
+             'http://www.web2py.com/book/default/chapter/08'),
-             (T('Access Control'), False,
+            (T('Access Control'), False,
-              'http://www.web2py.com/book/default/chapter/09'),
+             'http://www.web2py.com/book/default/chapter/09'),
-             (T('Services'), False,
+            (T('Services'), False,
-              'http://www.web2py.com/book/default/chapter/10'),
+             'http://www.web2py.com/book/default/chapter/10'),
-             (T('Ajax Recipes'), False,
+            (T('Ajax Recipes'), False,
-              'http://www.web2py.com/book/default/chapter/11'),
+             'http://www.web2py.com/book/default/chapter/11'),
-             (T('Components and Plugins'), False,
+            (T('Components and Plugins'), False,
-              'http://www.web2py.com/book/default/chapter/12'),
+             'http://www.web2py.com/book/default/chapter/12'),
-             (T('Deployment Recipes'), False,
+            (T('Deployment Recipes'), False,
-              'http://www.web2py.com/book/default/chapter/13'),
+             'http://www.web2py.com/book/default/chapter/13'),
-             (T('Other Recipes'), False,
+            (T('Other Recipes'), False,
-              'http://www.web2py.com/book/default/chapter/14'),
+             'http://www.web2py.com/book/default/chapter/14'),
-             (T('Helping web2py'), False,
+            (T('Helping web2py'), False,
-              'http://www.web2py.com/book/default/chapter/15'),
+             'http://www.web2py.com/book/default/chapter/15'),
-             (T("Buy web2py's book"), False,
+            (T("Buy web2py's book"), False,
-              'http://stores.lulu.com/web2py'),
+             'http://stores.lulu.com/web2py'),
-             ]),
+        ]),
-          (T('Community'), False, None, [
+        (T('Community'), False, None, [
-             (T('Groups'), False,
+            (T('Groups'), False,
-              'http://www.web2py.com/examples/default/usergroups'),
+             'http://www.web2py.com/examples/default/usergroups'),
-              (T('Twitter'), False, 'http://twitter.com/web2py'),
+            (T('Twitter'), False, 'http://twitter.com/web2py'),
-              (T('Live Chat'), False,
+            (T('Live Chat'), False,
-               'http://webchat.freenode.net/?channels=web2py'),
+             'http://webchat.freenode.net/?channels=web2py'),
-              ]),
+        ]),
-        ]
+    ]
-if DEVELOPMENT_MENU: _()
 
-if "auth" in locals(): auth.wikimenu() 
+
+if DEVELOPMENT_MENU:
+    _()
+
+if "auth" in locals():
+    auth.wikimenu()

applications/welcome/routes.example.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+# ----------------------------------------------------------------------------------------------------------------------
 #  This is an app-specific example router
 #
 #  This simple router is used for setting languages from app/languages directory
@@ -8,31 +9,33 @@
 #  a default_language
 #
 # See <web2py-root-dir>/examples/routes.parametric.example.py for parameter's detail
-#-------------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------------------------------------------------
 # To enable this route file you must do the steps:
-#
 # 1. rename <web2py-root-dir>/examples/routes.parametric.example.py to routes.py
-# 2. rename this APP/routes.example.py to APP/routes.py
+# 2. rename this APP/routes.example.py to APP/routes.py (where APP - is your application directory)
-#    (where APP - is your application directory)
+# 3. restart web2py (or reload routes in web2py admin interface)
-# 3. restart web2py (or reload routes in web2py admin interfase)
 #
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
+# ----------------------------------------------------------------------------------------------------------------------
 
 from fileutils import abspath
 from languages import read_possible_languages
 
 possible_languages = read_possible_languages(abspath('applications', app))
-#NOTE! app - is an application based router's parameter with name of an
+# ----------------------------------------------------------------------------------------------------------------------
-#            application. E.g.'welcome'
+# NOTE! app - is an application based router's parameter with name of an application. E.g.'welcome'
+# ----------------------------------------------------------------------------------------------------------------------
 
 routers = {
     app: dict(
-        default_language = possible_languages['default'][0],
+        default_language=possible_languages['default'][0],
-        languages = [lang for lang in possible_languages
+        languages=[lang for lang in possible_languages if lang != 'default']
-                           if lang != 'default']
     )
 }
 
-#NOTE! To change language in your application using these rules add this line
+# ----------------------------------------------------------------------------------------------------------------------
-#in one of your models files:
+# NOTE! To change language in your application using these rules add this line in one of your models files:
+# ----------------------------------------------------------------------------------------------------------------------
 #   if request.uri_language: T.force(request.uri_language)

fabfile.py

@@ -115,8 +115,9 @@ def deploy(appname=None, all=False):
     """fab -H username@host deploy:appname,all"""
     appname = appname or os.path.split(os.getcwd())[-1]
     appfolder = applications+'/'+appname
-    if os.path.exists('_update.zip'):
+    zipfile = os.path.join(appfolder, '_update.zip')
-        os.unlink('_update.zip')
+    if os.path.exists(zipfile):
+        os.unlink(zipfile)
 
     backup = mkdir_or_backup(appname)
             

gluon/admin.py

@@ -148,7 +148,7 @@ def app_compile(app, request, skip_failed_views=False):
         failed_views = compile_application(folder, skip_failed_views)
         return failed_views
     except (Exception, RestrictedError):
-        tb = traceback.format_exc(sys.exc_info)
+        tb = traceback.format_exc()
         remove_compiled_application(folder)
         return tb
 
@@ -167,7 +167,7 @@ def app_create(app, request, force=False, key=None, info=False):
             os.mkdir(path)
         except:
             if info:
-                return False, traceback.format_exc(sys.exc_info)
+                return False, traceback.format_exc()
             else:
                 return False
     elif not force:
@@ -197,7 +197,7 @@ def app_create(app, request, force=False, key=None, info=False):
     except:
         rmtree(path)
         if info:
-            return False, traceback.format_exc(sys.exc_info)
+            return False, traceback.format_exc()
         else:
             return False
 

gluon/cache.py

@@ -44,9 +44,9 @@ except ImportError:
     have_settings = False
 
 try:
-   import cPickle as pickle
+    import cPickle as pickle
 except:
-   import pickle
+    import pickle
 
 try:
     import psutil
@@ -54,6 +54,7 @@ try:
 except ImportError:
     HAVE_PSUTIL = False
 
+
 def remove_oldest_entries(storage, percentage=90):
     # compute current memory usage (%)
     old_mem = psutil.virtual_memory().percent
@@ -66,7 +67,8 @@ def remove_oldest_entries(storage, percentage=90):
         # comute used memory again
         new_mem = psutil.virtual_memory().percent
         # if the used memory did not decrease stop
-        if new_mem >= old_mem: break
+        if new_mem >= old_mem:
+            break
         # net new measurement for memory usage and loop
         old_mem = new_mem
 
@@ -78,6 +80,7 @@ __all__ = ['Cache', 'lazy_cache']
 
 DEFAULT_TIME_EXPIRE = 300
 
+
 class CacheAbstract(object):
     """
     Abstract class for cache implementations.
@@ -99,7 +102,7 @@ class CacheAbstract(object):
     """
 
     cache_stats_name = 'web2py_cache_statistics'
-    max_ram_utilization = None # percent
+    max_ram_utilization = None  # percent
 
     def __init__(self, request=None):
         """Initializes the object
@@ -182,13 +185,14 @@ class CacheInRam(CacheAbstract):
         self.request = request
         self.storage = OrderedDict() if HAVE_PSUTIL else {}
         self.app = request.application if request else ''
+
     def initialize(self):
         if self.initialized:
             return
         else:
             self.initialized = True
         self.locker.acquire()
-        if not self.app in self.meta_storage:
+        if self.app not in self.meta_storage:
             self.storage = self.meta_storage[self.app] = \
                 OrderedDict() if HAVE_PSUTIL else {}
             self.stats[self.app] = {'hit_total': 0, 'misses': 0}
@@ -205,7 +209,7 @@ class CacheInRam(CacheAbstract):
         else:
             self._clear(storage, regex)
 
-        if not self.app in self.stats:
+        if self.app not in self.stats:
             self.stats[self.app] = {'hit_total': 0, 'misses': 0}
 
         self.locker.release()
@@ -251,8 +255,8 @@ class CacheInRam(CacheAbstract):
         self.locker.acquire()
         self.storage[key] = (now, value)
         self.stats[self.app]['misses'] += 1
-        if HAVE_PSUTIL and self.max_ram_utilization!=None and random.random()<0.10:
+        if HAVE_PSUTIL and self.max_ram_utilization is not None and random.random() < 0.10:
-            remove_oldest_entries(self.storage, percentage = self.max_ram_utilization)
+            remove_oldest_entries(self.storage, percentage=self.max_ram_utilization)
         self.locker.release()
         return value
 
@@ -292,14 +296,15 @@ class CacheOnDisk(CacheAbstract):
             self.folder = folder
             self.key_filter_in = lambda key: key
             self.key_filter_out = lambda key: key
-            self.file_lock_time_wait = file_lock_time_wait # How long we should wait before retrying to lock a file held by another process
+            self.file_lock_time_wait = file_lock_time_wait
+            # How long we should wait before retrying to lock a file held by another process
             # We still need a mutex for each file as portalocker only blocks other processes
             self.file_locks = defaultdict(thread.allocate_lock)
 
-
             # Make sure we use valid filenames.
             if sys.platform == "win32":
                 import base64
+
                 def key_filter_in_windows(key):
                     """
                     Windows doesn't allow \ / : * ? "< > | in filenames.
@@ -316,7 +321,6 @@ class CacheOnDisk(CacheAbstract):
                 self.key_filter_in = key_filter_in_windows
                 self.key_filter_out = key_filter_out_windows
 
-
         def wait_portalock(self, val_file):
             """
             Wait for the process file lock.
@@ -328,15 +332,12 @@ class CacheOnDisk(CacheAbstract):
                 except:
                     time.sleep(self.file_lock_time_wait)
 
-
         def acquire(self, key):
             self.file_locks[key].acquire()
 
-
         def release(self, key):
             self.file_locks[key].release()
 
-
         def __setitem__(self, key, value):
             key = self.key_filter_in(key)
             val_file = recfile.open(key, mode='wb', path=self.folder)
@@ -344,7 +345,6 @@ class CacheOnDisk(CacheAbstract):
             pickle.dump(value, val_file, pickle.HIGHEST_PROTOCOL)
             val_file.close()
 
-
         def __getitem__(self, key):
             key = self.key_filter_in(key)
             try:
@@ -357,12 +357,10 @@ class CacheOnDisk(CacheAbstract):
             val_file.close()
             return value
 
-
         def __contains__(self, key):
             key = self.key_filter_in(key)
             return (key in self.file_locks) or recfile.exists(key, path=self.folder)
 
-
         def __delitem__(self, key):
             key = self.key_filter_in(key)
             try:
@@ -370,13 +368,11 @@ class CacheOnDisk(CacheAbstract):
             except IOError:
                 raise KeyError
 
-
         def __iter__(self):
             for dirpath, dirnames, filenames in os.walk(self.folder):
                 for filename in filenames:
                     yield self.key_filter_out(filename)
 
-
         def safe_apply(self, key, function, default_value=None):
             """
             Safely apply a function to the value of a key in storage and set
@@ -403,25 +399,21 @@ class CacheOnDisk(CacheAbstract):
             val_file.close()
             return new_value
 
-
         def keys(self):
             return list(self.__iter__())
 
-
         def get(self, key, default=None):
             try:
                 return self[key]
             except KeyError:
                 return default
 
-
     def __init__(self, request=None, folder=None):
         self.initialized = False
         self.request = request
         self.folder = folder
         self.storage = None
 
-
     def initialize(self):
         if self.initialized:
             return
@@ -440,7 +432,6 @@ class CacheOnDisk(CacheAbstract):
 
         self.storage = CacheOnDisk.PersistentStorage(folder)
 
-
     def __call__(self, key, f,
                  time_expire=DEFAULT_TIME_EXPIRE):
         self.initialize()
@@ -487,7 +478,6 @@ class CacheOnDisk(CacheAbstract):
         self.storage.release(key)
         return value
 
-
     def clear(self, regex=None):
         self.initialize()
         storage = self.storage
@@ -504,7 +494,6 @@ class CacheOnDisk(CacheAbstract):
                 pass
             storage.release(key)
 
-
     def increment(self, key, value=1):
         self.initialize()
         self.storage.acquire(key)
@@ -513,7 +502,6 @@ class CacheOnDisk(CacheAbstract):
         return value
 
 
-
 class CacheAction(object):
     def __init__(self, func, key, time_expire, cache, cache_model):
         self.__name__ = func.__name__
@@ -572,9 +560,9 @@ class Cache(object):
                 logger.warning('no cache.disk (AttributeError)')
 
     def action(self, time_expire=DEFAULT_TIME_EXPIRE, cache_model=None,
-             prefix=None, session=False, vars=True, lang=True,
+               prefix=None, session=False, vars=True, lang=True,
-             user_agent=False, public=True, valid_statuses=None,
+               user_agent=False, public=True, valid_statuses=None,
-             quick=None):
+               quick=None):
         """Better fit for caching an action
 
         Warning:
@@ -602,6 +590,7 @@ class Cache(object):
         """
         from gluon import current
         from gluon.http import HTTP
+
         def wrap(func):
             def wrapped_f():
                 if current.request.env.request_method != 'GET':
@@ -621,13 +610,14 @@ class Cache(object):
                     cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire)
                     if not session_ and public_:
                         cache_control += ', public'
-                        expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)).strftime('%a, %d %b %Y %H:%M:%S GMT')
+                        expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)
+                                   ).strftime('%a, %d %b %Y %H:%M:%S GMT')
                     else:
                         cache_control += ', private'
                         expires = 'Fri, 01 Jan 1990 00:00:00 GMT'
 
                 if cache_model:
-                    #figure out the correct cache key
+                    # figure out the correct cache key
                     cache_key = [current.request.env.path_info, current.response.view]
                     if session_:
                         cache_key.append(current.response.session_id)
@@ -644,28 +634,28 @@ class Cache(object):
                     if prefix:
                         cache_key = prefix + cache_key
                     try:
-                        #action returns something
+                        # action returns something
-                        rtn = cache_model(cache_key, lambda : func(), time_expire=time_expire)
+                        rtn = cache_model(cache_key, lambda: func(), time_expire=time_expire)
                         http, status = None, current.response.status
                     except HTTP, e:
-                        #action raises HTTP (can still be valid)
+                        # action raises HTTP (can still be valid)
-                        rtn = cache_model(cache_key, lambda : e.body, time_expire=time_expire)
+                        rtn = cache_model(cache_key, lambda: e.body, time_expire=time_expire)
                         http, status = HTTP(e.status, rtn, **e.headers), e.status
                     else:
-                        #action raised a generic exception
+                        # action raised a generic exception
                         http = None
                 else:
-                    #no server-cache side involved
+                    # no server-cache side involved
                     try:
-                        #action returns something
+                        # action returns something
                         rtn = func()
                         http, status = None, current.response.status
                     except HTTP, e:
-                        #action raises HTTP (can still be valid)
+                        # action raises HTTP (can still be valid)
                         status = e.status
                         http = HTTP(e.status, e.body, **e.headers)
                     else:
-                        #action raised a generic exception
+                        # action raised a generic exception
                         http = None
                 send_headers = False
                 if http and isinstance(valid_statuses, list):
@@ -675,15 +665,13 @@ class Cache(object):
                     if str(status)[0] in '123':
                         send_headers = True
                 if send_headers:
-                    headers = {
+                    headers = {'Pragma': None,
-                        'Pragma' : None,
+                               'Expires': expires,
-                        'Expires' : expires,
+                               'Cache-Control': cache_control}
-                        'Cache-Control' : cache_control
-                        }
                     current.response.headers.update(headers)
                 if cache_model and not send_headers:
-                    #we cached already the value, but the status is not valid
+                    # we cached already the value, but the status is not valid
-                    #so we need to delete the cached value
+                    # so we need to delete the cached value
                     cache_model(cache_key, None)
                 if http:
                     if send_headers:
@@ -740,8 +728,7 @@ class Cache(object):
         allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix')
         it will add prefix to all the cache keys used.
         """
-        return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\
+        return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix: cache_model(prefix + key, f, time_expire)
-            cache_model(prefix + key, f, time_expire)
 
 
 def lazy_cache(key=None, time_expire=None, cache_model='ram'):

gluon/compileapp.py

@@ -676,8 +676,8 @@ def run_view_in(environment):
     else:
         filename = pjoin(folder, 'views', view)
         if os.path.exists(path): # compiled views
-            x = view.replace('/', '.')
+            x = view.replace('/', '_')
-            files = ['views.%s.pyc' % x]
+            files = ['views_%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(path, files[0]))
             # Don't use a generic view if the non-compiled view exists.
             if is_compiled or (not is_compiled and not os.path.exists(filename)):

gluon/contrib/imageutils.py

@@ -27,7 +27,7 @@ from gluon import current
 
 class RESIZE(object):
 
-    def __init__(self, nx=160, ny=80, quality=100, padding = False
+    def __init__(self, nx=160, ny=80, quality=100, padding = False,
                  error_message=' image resize'):
         (self.nx, self.ny, self.quality, self.error_message, self.padding) = (
             nx, ny, quality, error_message, padding)

gluon/contrib/login_methods/oneall_account.py

@@ -51,7 +51,7 @@ class OneallAccount(object):
             reg_id=profile.get('identity_token','')
             username=profile.get('preferredUsername',email)
             first_name=name.get('givenName', dname.split(' ')[0])
-            last_name=profile.get('familyName', dname.split(' ')[1] if(len(dname.split(' ')) > 1) else None)
+            last_name=profile.get('familyName', dname.split(' ')[1] if(dname.count(' ') > 0) else None)
             return dict(registration_id=reg_id,username=username,email=email,
                         first_name=first_name,last_name=last_name)
         self.mappings.default = defaultmapping

gluon/contrib/markdown/markdown2.py

@@ -53,8 +53,9 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * header-ids: Adds "id" attributes to headers. The id value is a slug of
   the header text.
 * html-classes: Takes a dict mapping html tag names (lowercase) to a
-  string to use for a "class" tag attribute. Currently only supports
+  string to use for a "class" tag attribute. Currently only supports "img",
-  "pre" and "code" tags. Add an issue if you require this for other tags.
+  "table", "pre" and "code" tags. Add an issue if you require this for other
+  tags.
 * markdown-in-html: Allow the use of `markdown="1"` in a block HTML tag to
   have markdown processing be done on its contents. Similar to
   <http://michelf.com/projects/php-markdown/extra/#markdown-attr> but with
@@ -70,9 +71,14 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * smarty-pants: Replaces ' and " with curly quotation marks or curly
   apostrophes.  Replaces --, ---, ..., and . . . with en dashes, em dashes,
   and ellipses.
+* spoiler: A special kind of blockquote commonly hidden behind a
+  click on SO. Syntax per <http://meta.stackexchange.com/a/72878>.
 * toc: The returned HTML string gets a new "toc_html" attribute which is
   a Table of Contents for the document. (experimental)
 * xml: Passes one-liner processing instructions and namespaced XML tags.
+* tables: Tables using the same format as GFM
+  <https://help.github.com/articles/github-flavored-markdown#tables> and
+  PHP-Markdown Extra <https://michelf.ca/projects/php-markdown/extra/#table>.
 * wiki-tables: Google Code Wiki-style tables. See
   <http://code.google.com/p/support/wiki/WikiSyntax#Tables>.
 """
@@ -82,13 +88,11 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 #   not yet sure if there implications with this. Compare 'pydoc sre'
 #   and 'perldoc perlre'.
 
-__version_info__ = (2, 2, 4)
+__version_info__ = (2, 3, 1)
 __version__ = '.'.join(map(str, __version_info__))
 __author__ = "Trent Mick"
 
-import os
 import sys
-from pprint import pprint
 import re
 import logging
 try:
@@ -102,13 +106,7 @@ import codecs
 
 #---- Python version compat
 
-try:
-    from urllib.parse import quote # python3
-except ImportError:
-    from urllib import quote # python2
-
 if sys.version_info[:2] < (2,4):
-    from sets import Set as set
     def reversed(sequence):
         for i in sequence[::-1]:
             yield i
@@ -804,6 +802,8 @@ class Markdown(object):
             text = self._prepare_pyshell_blocks(text)
         if "wiki-tables" in self.extras:
             text = self._do_wiki_tables(text)
+        if "tables" in self.extras:
+            text = self._do_tables(text)
 
         text = self._do_code_blocks(text)
 
@@ -844,6 +844,79 @@ class Markdown(object):
 
         return _pyshell_block_re.sub(self._pyshell_block_sub, text)
 
+    def _table_sub(self, match):
+        trim_space_re = '^[ \t\n]+|[ \t\n]+$'
+        trim_bar_re = '^\||\|$'
+
+        head, underline, body = match.groups()
+
+        # Determine aligns for columns.
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", underline)).split('|')]
+        align_from_col_idx = {}
+        for col_idx, col in enumerate(cols):
+            if col[0] == ':' and col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="center"'
+            elif col[0] == ':':
+                align_from_col_idx[col_idx] = ' align="left"'
+            elif col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="right"'
+
+        # thead
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<thead>', '<tr>']
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", head)).split('|')]
+        for col_idx, col in enumerate(cols):
+            hlines.append('  <th%s>%s</th>' % (
+                align_from_col_idx.get(col_idx, ''),
+                self._run_span_gamut(col)
+            ))
+        hlines.append('</tr>')
+        hlines.append('</thead>')
+
+        # tbody
+        hlines.append('<tbody>')
+        for line in body.strip('\n').split('\n'):
+            hlines.append('<tr>')
+            cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", line)).split('|')]
+            for col_idx, col in enumerate(cols):
+                hlines.append('  <td%s>%s</td>' % (
+                    align_from_col_idx.get(col_idx, ''),
+                    self._run_span_gamut(col)
+                ))
+            hlines.append('</tr>')
+        hlines.append('</tbody>')
+        hlines.append('</table>')
+
+        return '\n'.join(hlines) + '\n'
+
+    def _do_tables(self, text):
+        """Copying PHP-Markdown and GFM table syntax. Some regex borrowed from
+        https://github.com/michelf/php-markdown/blob/lib/Michelf/Markdown.php#L2538
+        """
+        less_than_tab = self.tab_width - 1
+        table_re = re.compile(r'''
+                (?:(?<=\n\n)|\A\n?)             # leading blank line
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (.*[|].*)  \n                   # $1: header row (at least one pipe)
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (                               # $2: underline row
+                    # underline row with leading bar
+                    (?:  \|\ *:?-+:?\ *  )+  \|?  \n
+                    |
+                    # or, underline row without leading bar
+                    (?:  \ *:?-+:?\ *\|  )+  (?:  \ *:?-+:?\ *  )?  \n
+                )
+
+                (                               # $3: data rows
+                    (?:
+                        ^[ ]{0,%d}(?!\ )         # ensure line begins with 0 to less_than_tab spaces
+                        .*\|.*  \n
+                    )+
+                )
+            ''' % (less_than_tab, less_than_tab, less_than_tab), re.M | re.X)
+        return table_re.sub(self._table_sub, text)
+
     def _wiki_table_sub(self, match):
         ttext = match.group(0).strip()
         #print 'wiki table: %r' % match.group(0)
@@ -853,7 +926,7 @@ class Markdown(object):
             row = [c.strip() for c in re.split(r'(?<!\\)\|\|', line)]
             rows.append(row)
         #pprint(rows)
-        hlines = ['<table>', '<tbody>']
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<tbody>']
         for row in rows:
             hrow = ['<tr>']
             for cell in row:
@@ -899,6 +972,9 @@ class Markdown(object):
 
         text = self._encode_amps_and_angles(text)
 
+        if "strike" in self.extras:
+            text = self._do_strike(text)
+
         text = self._do_italics_and_bold(text)
 
         if "smarty-pants" in self.extras:
@@ -1206,7 +1282,6 @@ class Markdown(object):
                                  .replace('_', self._escape_table['_'])
                         title = self.titles.get(link_id)
                         if title:
-                            before = title
                             title = _xml_escape_attr(title) \
                                 .replace('*', self._escape_table['*']) \
                                 .replace('_', self._escape_table['_'])
@@ -1418,7 +1493,6 @@ class Markdown(object):
     def _list_item_sub(self, match):
         item = match.group(4)
         leading_line = match.group(1)
-        leading_space = match.group(2)
         if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
             item = self._run_block_gamut(self._outdent(item))
         else:
@@ -1654,6 +1728,11 @@ class Markdown(object):
         self._escape_table[text] = hashed
         return hashed
 
+    _strike_re = re.compile(r"~~(?=\S)(.+?)(?<=\S)~~", re.S)
+    def _do_strike(self, text):
+        text = self._strike_re.sub(r"<strike>\1</strike>", text)
+        return text
+
     _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
     _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
     _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
@@ -1714,38 +1793,53 @@ class Markdown(object):
         text = text.replace(". . .", "&#8230;")
         return text
 
-    _block_quote_re = re.compile(r'''
+    _block_quote_base = r'''
         (                           # Wrap whole match in \1
           (
-            ^[ \t]*>[ \t]?          # '>' at the start of a line
+            ^[ \t]*>%s[ \t]?        # '>' at the start of a line
               .+\n                  # rest of the first line
             (.+\n)*                 # subsequent consecutive lines
             \n*                     # blanks
           )+
         )
-        ''', re.M | re.X)
+    '''
+    _block_quote_re = re.compile(_block_quote_base % '', re.M | re.X)
+    _block_quote_re_spoiler = re.compile(_block_quote_base % '[ \t]*?!?', re.M | re.X)
     _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M);
-
+    _bq_one_level_re_spoiler = re.compile('^[ \t]*>[ \t]*?![ \t]?', re.M);
+    _bq_all_lines_spoilers = re.compile(r'\A(?:^[ \t]*>[ \t]*?!.*[\n\r]*)+\Z', re.M)
     _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
     def _dedent_two_spaces_sub(self, match):
         return re.sub(r'(?m)^  ', '', match.group(1))
 
     def _block_quote_sub(self, match):
         bq = match.group(1)
-        bq = self._bq_one_level_re.sub('', bq)  # trim one level of quoting
+        is_spoiler = 'spoiler' in self.extras and self._bq_all_lines_spoilers.match(bq)
-        bq = self._ws_only_line_re.sub('', bq)  # trim whitespace-only lines
+        # trim one level of quoting
+        if is_spoiler:
+            bq = self._bq_one_level_re_spoiler.sub('', bq)
+        else:
+            bq = self._bq_one_level_re.sub('', bq)
+        # trim whitespace-only lines
+        bq = self._ws_only_line_re.sub('', bq)
         bq = self._run_block_gamut(bq)          # recurse
 
         bq = re.sub('(?m)^', '  ', bq)
         # These leading spaces screw with <pre> content, so we need to fix that:
         bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
 
-        return "<blockquote>\n%s\n</blockquote>\n\n" % bq
+        if is_spoiler:
+            return '<blockquote class="spoiler">\n%s\n</blockquote>\n\n' % bq
+        else:
+            return '<blockquote>\n%s\n</blockquote>\n\n' % bq
 
     def _do_block_quotes(self, text):
         if '>' not in text:
             return text
-        return self._block_quote_re.sub(self._block_quote_sub, text)
+        if 'spoiler' in self.extras:
+            return self._block_quote_re_spoiler.sub(self._block_quote_sub, text)
+        else:
+            return self._block_quote_re.sub(self._block_quote_sub, text)
 
     def _form_paragraphs(self, text):
         # Strip leading and trailing lines:
@@ -2053,7 +2147,6 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False):
     if DEBUG:
         print("dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
               % (tabsize, skip_first_line))
-    indents = []
     margin = None
     for i, line in enumerate(lines):
         if i == 0 and skip_first_line: continue

gluon/contrib/markmin/markmin2html.py

@@ -7,10 +7,11 @@ import re
 import urllib
 from cgi import escape
 from string import maketrans
+
 try:
-   from ast import parse as ast_parse
+    from ast import parse as ast_parse
-   import ast
+    import ast
-except ImportError: # python 2.5
+except ImportError:  # python 2.5
     from compiler import parse
     import compiler.ast as ast
 
@@ -530,40 +531,46 @@ As shown in Ref.!`!`mdipierro`!`!:cite
 ``<ul/>``, ``<ol/>``, ``<code/>``, ``<table/>``, ``<blockquote/>``, ``<h1/>``, ..., ``<h6/>`` do not have ``<p>...</p>`` around them.
 
 """
-html_colors=['aqua', 'black', 'blue', 'fuchsia', 'gray', 'green',
+html_colors = ['aqua', 'black', 'blue', 'fuchsia', 'gray', 'green',
-             'lime', 'maroon', 'navy', 'olive', 'purple', 'red',
+               'lime', 'maroon', 'navy', 'olive', 'purple', 'red',
-             'silver', 'teal', 'white', 'yellow']
+               'silver', 'teal', 'white', 'yellow']
 
 META = '\x06'
 LINK = '\x07'
 DISABLED_META = '\x08'
 LATEX = '<img src="http://chart.apis.google.com/chart?cht=tx&chl=%s" />'
-regex_URL=re.compile(r'@/(?P<a>\w*)/(?P<c>\w*)/(?P<f>\w*(\.\w+)?)(/(?P<args>[\w\.\-/]+))?')
+regex_URL = re.compile(r'@/(?P<a>\w*)/(?P<c>\w*)/(?P<f>\w*(\.\w+)?)(/(?P<args>[\w\.\-/]+))?')
-regex_env2=re.compile(r'@\{(?P<a>[\w\-\.]+?)(\:(?P<b>.*?))?\}')
+regex_env2 = re.compile(r'@\{(?P<a>[\w\-\.]+?)(\:(?P<b>.*?))?\}')
-regex_expand_meta = re.compile('('+META+'|'+DISABLED_META+'|````)')
+regex_expand_meta = re.compile('(' + META + '|' + DISABLED_META + '|````)')
-regex_dd=re.compile(r'\$\$(?P<latex>.*?)\$\$')
+regex_dd = re.compile(r'\$\$(?P<latex>.*?)\$\$')
-regex_code = re.compile('('+META+'|'+DISABLED_META+r'|````)|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',re.S)
+regex_code = re.compile(
-regex_strong=re.compile(r'\*\*(?P<t>[^\s*]+( +[^\s*]+)*)\*\*')
+    '(' + META + '|' + DISABLED_META + r'|````)|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',
-regex_del=re.compile(r'~~(?P<t>[^\s*]+( +[^\s*]+)*)~~')
+    re.S)
-regex_em=re.compile(r"''(?P<t>([^\s']| |'(?!'))+)''")
+regex_strong = re.compile(r'\*\*(?P<t>[^\s*]+( +[^\s*]+)*)\*\*')
-regex_num=re.compile(r"^\s*[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?\s*$")
+regex_del = re.compile(r'~~(?P<t>[^\s*]+( +[^\s*]+)*)~~')
-regex_list=re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s*)?(.*)$')
+regex_em = re.compile(r"''(?P<t>([^\s']| |'(?!'))+)''")
-regex_bq_headline=re.compile('^(?:(\.+|\++|\-+)(\.)?\s+)?(-{3}-*)$')
+regex_num = re.compile(r"^\s*[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?\s*$")
-regex_tq=re.compile('^(-{3}-*)(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[a-zA-Z][_a-zA-Z\-\d]*)\])?)?$')
+regex_list = re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s*)?(.*)$')
+regex_bq_headline = re.compile('^(?:(\.+|\++|\-+)(\.)?\s+)?(-{3}-*)$')
+regex_tq = re.compile('^(-{3}-*)(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[a-zA-Z][_a-zA-Z\-\d]*)\])?)?$')
 regex_proto = re.compile(r'(?<!["\w>/=])(?P<p>\w+):(?P<k>\w+://[\w\d\-+=?%&/:.]+)', re.M)
-regex_auto = re.compile(r'(?<!["\w>/=])(?P<k>\w+://[\w\d\-+_=?%&/:.,;#]+\w|[\w\-.]+@[\w\-.]+)',re.M)
+regex_auto = re.compile(r'(?<!["\w>/=])(?P<k>\w+://[\w\d\-+_=?%&/:.,;#]+\w|[\w\-.]+@[\w\-.]+)', re.M)
-regex_link=re.compile(r'('+LINK+r')|\[\[(?P<s>.+?)\]\]',re.S)
+regex_link = re.compile(r'(' + LINK + r')|\[\[(?P<s>.+?)\]\]', re.S)
-regex_link_level2=re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?(?:\s+(?P<p>popup))?\s*$',re.S)
+regex_link_level2 = re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?(?:\s+(?P<p>popup))?\s*$', re.S)
-regex_media_level2=re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?\s+(?P<p>img|IMG|left|right|center|video|audio|blockleft|blockright)(?:\s+(?P<w>\d+px))?\s*$',re.S)
+regex_media_level2 = re.compile(
+    r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?\s+(?P<p>img|IMG|left|right|center|video|audio|blockleft|blockright)(?:\s+(?P<w>\d+px))?\s*$',
+    re.S)
 
 regex_markmin_escape = re.compile(r"(\\*)(['`:*~\\[\]{}@\$+\-.#\n])")
 regex_backslash = re.compile(r"\\(['`:*~\\[\]{}@\$+\-.#\n])")
-ttab_in  = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05')
+ttab_in = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05')
-ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05',"'`:*~\\[]{}@$+-.#\n")
+ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')
 
+
 def make_dict(b):
-    return '{%s}' % regex_quote.sub("'\g<name>':",b)
+    return '{%s}' % regex_quote.sub("'\g<name>':", b)
+
 
 def safe_eval(node_or_string, env):
     """
@@ -578,6 +585,7 @@ def safe_eval(node_or_string, env):
         node_or_string = ast_parse(node_or_string, mode='eval')
     if isinstance(node_or_string, ast.Expression):
         node_or_string = node_or_string.body
+
     def _convert(node):
         if isinstance(node, ast.Str):
             return node.s
@@ -594,11 +602,11 @@ def safe_eval(node_or_string, env):
             if node.id in _safe_names:
                 return _safe_names[node.id]
         elif isinstance(node, ast.BinOp) and \
-             isinstance(node.op, (Add, Sub)) and \
+                isinstance(node.op, (Add, Sub)) and \
-             isinstance(node.right, Num) and \
+                isinstance(node.right, Num) and \
-             isinstance(node.right.n, complex) and \
+                isinstance(node.right.n, complex) and \
-             isinstance(node.left, Num) and \
+                isinstance(node.left, Num) and \
-             isinstance(node.left.n, (int, long, float)):
+                isinstance(node.left.n, (int, long, float)):
             left = node.left.n
             right = node.right.n
             if isinstance(node.op, Add):
@@ -606,57 +614,66 @@ def safe_eval(node_or_string, env):
             else:
                 return left - right
         raise ValueError('malformed string')
+
     return _convert(node_or_string)
 
+
 def markmin_escape(text):
     """ insert \\ before markmin control characters: '`:*~[]{}@$ """
     return regex_markmin_escape.sub(
-       lambda m: '\\'+m.group(0).replace('\\','\\\\'), text)
+        lambda m: '\\' + m.group(0).replace('\\', '\\\\'), text)
 
-def replace_autolinks(text,autolinks):
+
+def replace_autolinks(text, autolinks):
     return regex_auto.sub(lambda m: autolinks(m.group('k')), text)
 
-def replace_at_urls(text,url):
-    # this is experimental @{function/args}
-    def u1(match,url=url):
-        a,c,f,args = match.group('a','c','f','args')
-        return url(a=a or None,c=c or None,f = f or None,
-                   args=(args or '').split('/'), scheme=True, host=True)
-    return regex_URL.sub(u1,text)
 
-def replace_components(text,env):
+def replace_at_urls(text, url):
+    # this is experimental @{function/args}
+    def u1(match, url=url):
+        a, c, f, args = match.group('a', 'c', 'f', 'args')
+        return url(a=a or None, c=c or None, f=f or None,
+                   args=(args or '').split('/'), scheme=True, host=True)
+
+    return regex_URL.sub(u1, text)
+
+
+def replace_components(text, env):
     # not perfect but acceptable
     def u2(match, env=env):
         f = env.get(match.group('a'), match.group(0))
         if callable(f):
             b = match.group('b')
             try:
-                b = safe_eval(make_dict(b),env)
+                b = safe_eval(make_dict(b), env)
             except:
                 pass
             try:
-                f = f(**b) if isinstance(b,dict) else f(b)
+                f = f(**b) if isinstance(b, dict) else f(b)
             except Exception, e:
                 f = 'ERROR: %s' % e
             return str(f)
+
     text = regex_env2.sub(u2, text)
     return text
 
+
 def autolinks_simple(url):
     """
     it automatically converts the url to link,
     image, video or audio tag
     """
-    u_url=url.lower()
+    u_url = url.lower()
-    if '@' in url and not '://' in url:
+    if '@' in url and '://' not in url:
         return '<a href="mailto:%s">%s</a>' % (url, url)
-    elif u_url.endswith(('.jpg','.jpeg','.gif','.png')):
+    elif u_url.endswith(('.jpg', '.jpeg', '.gif', '.png')):
         return '<img src="%s" controls />' % url
-    elif u_url.endswith(('.mp4','.mpeg','.mov','.ogv')):
+    elif u_url.endswith(('.mp4', '.mpeg', '.mov', '.ogv')):
         return '<video src="%s" controls></video>' % url
-    elif u_url.endswith(('.mp3','.wav','.ogg')):
+    elif u_url.endswith(('.mp3', '.wav', '.ogg')):
         return '<audio src="%s" controls></audio>' % url
-    return '<a href="%s">%s</a>' % (url,url)
+    return '<a href="%s">%s</a>' % (url, url)
+
 
 def protolinks_simple(proto, url):
     """
@@ -667,16 +684,18 @@ def protolinks_simple(proto, url):
             proto="iframe"
             url="http://www.example.com/path"
     """
-    if proto in ('iframe','embed'): #== 'iframe':
+    if proto in ('iframe', 'embed'):  # == 'iframe':
-        return '<iframe src="%s" frameborder="0" allowfullscreen></iframe>'%url
+        return '<iframe src="%s" frameborder="0" allowfullscreen></iframe>' % url
-    #elif proto == 'embed':  # NOTE: embed is a synonym to iframe now
+    # elif proto == 'embed':  # NOTE: embed is a synonym to iframe now
     #    return '<a href="%s" class="%sembed">%s></a>'%(url,class_prefix,url)
     elif proto == 'qr':
-        return '<img style="width:100px" src="http://chart.apis.google.com/chart?cht=qr&chs=100x100&chl=%s&choe=UTF-8&chld=H" alt="QR Code" title="QR Code" />'%url
+        return '<img style="width:100px" src="http://chart.apis.google.com/chart?cht=qr&chs=100x100&chl=%s&choe=UTF-8&chld=H" alt="QR Code" title="QR Code" />' % url
-    return proto+':'+url
+    return proto + ':' + url
+
 
 def email_simple(email):
-   return '<a href="mailto:%s">%s</a>' % (email, email)
+    return '<a href="mailto:%s">%s</a>' % (email, email)
+
 
 def render(text,
            extra={},
@@ -925,17 +944,19 @@ def render(text,
     >>> render("anchor with name 'NEWLINE': [[NEWLINE [newline] ]]")
     '<p>anchor with name \\'NEWLINE\\': <span class="anchor" id="markmin_NEWLINE">newline</span></p>'
     """
-    if autolinks=="default": autolinks = autolinks_simple
+    if autolinks == "default":
-    if protolinks=="default": protolinks = protolinks_simple
+        autolinks = autolinks_simple
-    pp='\n' if pretty_print else ''
+    if protolinks == "default":
-    if isinstance(text,unicode):
+        protolinks = protolinks_simple
+    pp = '\n' if pretty_print else ''
+    if isinstance(text, unicode):
         text = text.encode('utf8')
     text = str(text or '')
     text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
-    text = text.replace('\x05','').replace('\r\n', '\n') # concatenate strings separeted by \\n
+    text = text.replace('\x05', '').replace('\r\n', '\n')  # concatenate strings separeted by \\n
 
     if URL is not None:
-        text = replace_at_urls(text,URL)
+        text = replace_at_urls(text, URL)
 
     if latex == 'google':
         text = regex_dd.sub('``\g<latex>``:latex ', text)
@@ -945,9 +966,10 @@ def render(text,
     # store them into segments they will be treated as code
     #############################################################
     segments = []
+
     def mark_code(m):
         g = m.group(0)
-        if g in (META, DISABLED_META ):
+        if g in (META, DISABLED_META):
             segments.append((None, None, None, g))
             return m.group()
         elif g == '````':
@@ -956,10 +978,12 @@ def render(text,
         else:
             c = m.group('c') or ''
             p = m.group('p') or ''
-            if 'code' in allowed and not c in allowed['code']: c = ''
+            if 'code' in allowed and c not in allowed['code']:
-            code = m.group('t').replace('!`!','`')
+                c = ''
+            code = m.group('t').replace('!`!', '`')
             segments.append((code, c, p, m.group(0)))
         return META
+
     text = regex_code.sub(mark_code, text)
 
     #############################################################
@@ -967,56 +991,58 @@ def render(text,
     # store them into links they will be treated as link
     #############################################################
     links = []
+
     def mark_link(m):
-        links.append( None if m.group() == LINK
+        links.append(None if m.group() == LINK
-                         else m.group('s') )
+                     else m.group('s'))
         return LINK
+
     text = regex_link.sub(mark_link, text)
     text = escape(text)
 
     if protolinks:
-        text = regex_proto.sub(lambda m: protolinks(*m.group('p','k')), text)
+        text = regex_proto.sub(lambda m: protolinks(*m.group('p', 'k')), text)
 
     if autolinks:
-        text = replace_autolinks(text,autolinks)
+        text = replace_autolinks(text, autolinks)
 
     #############################################################
     # normalize spaces
     #############################################################
-    strings=text.split('\n')
+    strings = text.split('\n')
 
-    def parse_title(t, s): #out, lev, etags, tag, s):
+    def parse_title(t, s):  # out, lev, etags, tag, s):
-        hlevel=str(len(t))
+        hlevel = str(len(t))
         out.extend(etags[::-1])
-        out.append("<h%s>%s"%(hlevel,s))
+        out.append("<h%s>%s" % (hlevel, s))
-        etags[:]=["</h%s>%s"%(hlevel,pp)]
+        etags[:] = ["</h%s>%s" % (hlevel, pp)]
-        lev=0
+        lev = 0
-        ltags[:]=[]
+        ltags[:] = []
-        tlev[:]=[]
+        tlev[:] = []
         return (lev, 'h')
 
     def parse_list(t, p, s, tag, lev, mtag, lineno):
-        lent=len(t)
+        lent = len(t)
-        if lent<lev: # current item level < previous item level
+        if lent < lev:  # current item level < previous item level
-            while ltags[-1]>lent:
+            while ltags[-1] > lent:
                 ltags.pop()
                 out.append(etags.pop())
-            lev=lent
+            lev = lent
-            tlev[lev:]=[]
+            tlev[lev:] = []
 
-        if lent>lev: # current item level > previous item level
+        if lent > lev:  # current item level > previous item level
-            if lev==0: # previous line is not a list (paragraph or title)
+            if lev == 0:  # previous line is not a list (paragraph or title)
                 out.extend(etags[::-1])
-                ltags[:]=[]
+                ltags[:] = []
-                tlev[:]=[]
+                tlev[:] = []
-                etags[:]=[]
+                etags[:] = []
-            if pend and mtag == '.': # paragraph in a list:
+            if pend and mtag == '.':  # paragraph in a list:
                 out.append(etags.pop())
                 ltags.pop()
-            for i in xrange(lent-lev):
+            for i in xrange(lent - lev):
-                out.append('<'+tag+'>'+pp)
+                out.append('<' + tag + '>' + pp)
-                etags.append('</'+tag+'>'+pp)
+                etags.append('</' + tag + '>' + pp)
-                lev+=1
+                lev += 1
                 ltags.append(lev)
                 tlev.append(tag)
         elif lent == lev:
@@ -1025,22 +1051,22 @@ def render(text,
                 for i in xrange(ltags.count(lent)):
                     ltags.pop()
                     out.append(etags.pop())
-                tlev[-1]=tag
+                tlev[-1] = tag
-                out.append('<'+tag+'>'+pp)
+                out.append('<' + tag + '>' + pp)
-                etags.append('</'+tag+'>'+pp)
+                etags.append('</' + tag + '>' + pp)
                 ltags.append(lev)
             else:
-                if ltags.count(lev)>1:
+                if ltags.count(lev) > 1:
                     out.append(etags.pop())
                     ltags.pop()
-        mtag='l'
+        mtag = 'l'
         out.append('<li>')
-        etags.append('</li>'+pp)
+        etags.append('</li>' + pp)
         ltags.append(lev)
         if s[:1] == '-':
             (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
-        if p and mtag=='l':
+        if p and mtag == 'l':
-            (lev,mtag,lineno)=parse_point(t, s, lev, '', lineno)
+            (lev, mtag, lineno) = parse_point(t, s, lev, '', lineno)
         else:
             out.append(s)
 
@@ -1048,28 +1074,28 @@ def render(text,
 
     def parse_point(t, s, lev, mtag, lineno):
         """ paragraphs in lists """
-        lent=len(t)
+        lent = len(t)
-        if lent>lev:
+        if lent > lev:
             return parse_list(t, '.', s, 'ul', lev, mtag, lineno)
-        elif lent<lev:
+        elif lent < lev:
-            while ltags[-1]>lent:
+            while ltags[-1] > lent:
                 ltags.pop()
                 out.append(etags.pop())
-            lev=lent
+            lev = lent
-            tlev[lev:]=[]
+            tlev[lev:] = []
-            mtag=''
+            mtag = ''
-        elif lent==lev:
+        elif lent == lev:
             if pend and mtag == '.':
                 out.append(etags.pop())
                 ltags.pop()
-        if br and mtag in ('l','.'):
+        if br and mtag in ('l', '.'):
             out.append(br)
         if s == META:
-           mtag = ''
+            mtag = ''
         else:
             mtag = '.'
             if s[:1] == '-':
-               (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
+                (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
             if mtag == '.':
                 out.append(pbeg)
                 if pend:
@@ -1083,19 +1109,19 @@ def render(text,
         # - is empty -> this is an <hr /> tag
         # - consists '|' -> table
         # - consists other characters -> blockquote
-        if (lineno+1 >= strings_len or
+        if (lineno + 1 >= strings_len or
-            not(s.count('-') == len(s) and len(s)>3)):
+                not (s.count('-') == len(s) and len(s) > 3)):
-           return (s, mtag, lineno)
+            return (s, mtag, lineno)
 
-        lineno+=1
+        lineno += 1
         s = strings[lineno].strip()
         if s:
             if '|' in s:
                 # table
-                tout=[]
+                tout = []
-                thead=[]
+                thead = []
-                tbody=[]
+                tbody = []
-                rownum=0
+                rownum = 0
                 t_id = ''
                 t_cls = ''
 
@@ -1104,14 +1130,14 @@ def render(text,
                     s = strings[lineno].strip()
                     if s[:1] == '=':
                         # header or footer
-                        if s.count('=')==len(s) and len(s)>3:  
+                        if s.count('=') == len(s) and len(s) > 3:
-                            if not thead: # if thead list is empty:
+                            if not thead:  # if thead list is empty:
                                 thead = tout
                             else:
                                 tbody.extend(tout)
                             tout = []
-                            rownum=0
+                            rownum = 0
-                            lineno+=1
+                            lineno += 1
                             continue
 
                     m = regex_tq.match(s)
@@ -1121,36 +1147,36 @@ def render(text,
                         break
 
                     if rownum % 2:
-                       tr = '<tr class="even">'
+                        tr = '<tr class="even">'
                     else:
-                       tr = '<tr class="first">' if rownum == 0 else '<tr>'
+                        tr = '<tr class="first">' if rownum == 0 else '<tr>'
                     tout.append(tr + ''.join(['<td%s>%s</td>' % (
-                                    ' class="num"' 
+                        ' class="num"'
-                                    if regex_num.match(f) else '',
+                        if regex_num.match(f) else '',
-                                    f.strip()
+                        f.strip()
-                                    ) for f in s.split('|')])+'</tr>'+pp)
+                    ) for f in s.split('|')]) + '</tr>' + pp)
-                    rownum+=1
+                    rownum += 1
-                    lineno+=1
+                    lineno += 1
 
-                t_cls = ' class="%s%s"'%(class_prefix, t_cls) \
+                t_cls = ' class="%s%s"' % (class_prefix, t_cls) \
                     if t_cls and t_cls != 'id' else ''
-                t_id  = ' id="%s%s"'%(id_prefix, t_id) if t_id else ''
+                t_id = ' id="%s%s"' % (id_prefix, t_id) if t_id else ''
                 s = ''
                 if thead:
-                    s += '<thead>'+pp+''.join([l for l in thead])+'</thead>'+pp
+                    s += '<thead>' + pp + ''.join([l for l in thead]) + '</thead>' + pp
-                if not tbody: # tbody strings are in tout list
+                if not tbody:  # tbody strings are in tout list
                     tbody = tout
                     tout = []
-                if tbody: # if tbody list is not empty:
+                if tbody:  # if tbody list is not empty:
-                    s += '<tbody>'+pp+''.join([l for l in tbody])+'</tbody>'+pp
+                    s += '<tbody>' + pp + ''.join([l for l in tbody]) + '</tbody>' + pp
-                if tout: # tfoot is not empty:
+                if tout:  # tfoot is not empty:
-                    s += '<tfoot>'+pp+''.join([l for l in tout])+'</tfoot>'+pp
+                    s += '<tfoot>' + pp + ''.join([l for l in tout]) + '</tfoot>' + pp
                 s = '<table%s%s>%s%s</table>%s' % (t_cls, t_id, pp, s, pp)
-                mtag='t'
+                mtag = 't'
             else:
                 # parse blockquote:
-                bq_begin=lineno
+                bq_begin = lineno
-                t_mode = False # embedded table
+                t_mode = False  # embedded table
                 t_cls = ''
                 t_id = ''
 
@@ -1160,57 +1186,57 @@ def render(text,
                     if not t_mode:
                         m = regex_tq.match(s)
                         if m:
-                            if (lineno+1 == strings_len or 
+                            if (lineno + 1 == strings_len or
-                                '|' not in strings[lineno+1]):
+                                        '|' not in strings[lineno + 1]):
-                               t_cls = m.group('c') or ''
+                                t_cls = m.group('c') or ''
-                               t_id = m.group('p') or ''
+                                t_id = m.group('p') or ''
-                               break
+                                break
 
                         if regex_bq_headline.match(s):
-                            if (lineno+1 < strings_len and 
+                            if (lineno + 1 < strings_len and
-                                strings[lineno+1].strip()):
+                                    strings[lineno + 1].strip()):
-                                    t_mode = True
+                                t_mode = True
-                            lineno+=1
+                            lineno += 1
                             continue
                     elif regex_tq.match(s):
-                        t_mode=False
+                        t_mode = False
-                        lineno+=1
+                        lineno += 1
                         continue
 
-                    lineno+=1
+                    lineno += 1
 
-                t_cls = ' class="%s%s"'%(class_prefix,t_cls) \
+                t_cls = ' class="%s%s"' % (class_prefix, t_cls) \
                     if t_cls and t_cls != 'id' else ''
-                t_id  = ' id="%s%s"'%(id_prefix,t_id) \
+                t_id = ' id="%s%s"' % (id_prefix, t_id) \
                     if t_id else ''
 
                 s = '<blockquote%s%s>%s</blockquote>%s' \
-                         % (t_cls,
+                    % (t_cls,
-                            t_id,
+                       t_id,
-                            '\n'.join(strings[bq_begin:lineno]),pp)
+                       '\n'.join(strings[bq_begin:lineno]), pp)
-                mtag='q'
+                mtag = 'q'
         else:
             s = '<hr />'
-            lineno-=1
+            lineno -= 1
-            mtag='q'
+            mtag = 'q'
         return (s, 'q', lineno)
 
     if sep == 'p':
-      pbeg = "<p>"
+        pbeg = "<p>"
-      pend = "</p>"+pp
+        pend = "</p>" + pp
-      br = ''
+        br = ''
     else:
-      pbeg = pend = ''
+        pbeg = pend = ''
-      br = "<br />"+pp if sep=='br' else ''
+        br = "<br />" + pp if sep == 'br' else ''
 
-    lev = 0    # nesting level of lists
+    lev = 0  # nesting level of lists
-    c0 = ''    # first character of current line
+    c0 = ''  # first character of current line
-    out = []   # list of processed lines
+    out = []  # list of processed lines
-    etags = [] # trailing tags
+    etags = []  # trailing tags
-    ltags = [] # level# correspondent to trailing tag
+    ltags = []  # level# correspondent to trailing tag
     tlev = []  # list of tags for each level ('ul' or 'ol')
     mtag = ''  # marked tag (~last tag) ('l','.','h','p','t'). Used to set <br/>
-               # and to avoid <p></p> around tables and blockquotes
+    # and to avoid <p></p> around tables and blockquotes
     lineno = 0
     strings_len = len(strings)
     while lineno < strings_len:
@@ -1222,65 +1248,67 @@ def render(text,
             ####  ++++  ----  .... -------  field | field | field  <-body
             ##### +++++ ----- .....         ---------------------:class[id]
         """
-        pc0=c0 # first character of previous line
+        pc0 = c0  # first character of previous line
-        c0=s[:1]
+        c0 = s[:1]
-        if c0: # for non empty strings
+        if c0:  # for non empty strings
-            if c0 in "#+-.": # first character is one of: # + - .
+            if c0 in "#+-.":  # first character is one of: # + - .
-                (t1,t2,p,ss) = regex_list.findall(s)[0]
+                (t1, t2, p, ss) = regex_list.findall(s)[0]
                 # t1 - tag ("###")
                 # t2 - tag ("+++", "---", "...")
                 # p - paragraph point ('.')->for "++." or "--."
                 # ss - other part of string
                 if t1 or t2:
                     # headers and lists:
-                    if c0 == '#': # headers
+                    if c0 == '#':  # headers
                         (lev, mtag) = parse_title(t1, ss)
-                        lineno+=1
+                        lineno += 1
                         continue
-                    elif c0 == '+': # ordered list
+                    elif c0 == '+':  # ordered list
-                        (lev, mtag, lineno)= parse_list(t2, p, ss, 'ol', lev, mtag, lineno)
+                        (lev, mtag, lineno) = parse_list(t2, p, ss, 'ol', lev, mtag, lineno)
-                        lineno+=1
+                        lineno += 1
                         continue
-                    elif c0 == '-': # unordered list, table or blockquote
+                    elif c0 == '-':  # unordered list, table or blockquote
                         if p or ss:
                             (lev, mtag, lineno) = parse_list(t2, p, ss, 'ul', lev, mtag, lineno)
-                            lineno+=1
+                            lineno += 1
                             continue
                         else:
                             (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
-                    elif lev>0: # and c0 == '.' # paragraph in lists
+                    elif lev > 0:  # and c0 == '.' # paragraph in lists
                         (lev, mtag, lineno) = parse_point(t2, ss, lev, mtag, lineno)
-                        lineno+=1
+                        lineno += 1
                         continue
 
             if lev == 0 and (mtag == 'q' or s == META):
                 # new paragraph
-                pc0=''
+                pc0 = ''
 
-            if pc0 == '' or (mtag != 'p' and s0 not in (' ','\t')):
+            if pc0 == '' or (mtag != 'p' and s0 not in (' ', '\t')):
                 # paragraph
                 out.extend(etags[::-1])
-                etags=[]
+                etags = []
-                ltags=[]
+                ltags = []
-                tlev=[]
+                tlev = []
-                lev=0
+                lev = 0
-                if br and mtag == 'p': out.append(br)
+                if br and mtag == 'p':
+                    out.append(br)
                 if mtag != 'q' and s != META:
-                   if pend: etags=[pend]
+                    if pend:
-                   out.append(pbeg)
+                        etags = [pend]
-                   mtag = 'p'
+                    out.append(pbeg)
+                    mtag = 'p'
                 else:
-                   mtag = ''
+                    mtag = ''
                 out.append(s)
             else:
-                if lev>0 and mtag=='.' and s == META:
+                if lev > 0 and mtag == '.' and s == META:
                     out.append(etags.pop())
                     ltags.pop()
                     out.append(s)
                     mtag = ''
                 else:
-                    out.append(' '+s)
+                    out.append(' ' + s)
-        lineno+=1
+        lineno += 1
     out.extend(etags[::-1])
     text = ''.join(out)
 
@@ -1295,7 +1323,7 @@ def render(text,
     # deal with images, videos, audios and links
     #############################################################
     def sub_media(m):
-        t,a,k,p,w = m.group('t','a','k','p','w')
+        t, a, k, p, w = m.group('t', 'a', 'k', 'p', 'w')
         if not k:
             return m.group(0)
         k = escape(k)
@@ -1305,40 +1333,40 @@ def render(text,
         p_begin = p_end = ''
         if p == 'center':
             p_begin = '<p style="text-align:center">'
-            p_end = '</p>'+pp
+            p_end = '</p>' + pp
         elif p == 'blockleft':
             p_begin = '<p style="text-align:left">'
-            p_end = '</p>'+pp
+            p_end = '</p>' + pp
         elif p == 'blockright':
             p_begin = '<p style="text-align:right">'
-            p_end = '</p>'+pp
+            p_end = '</p>' + pp
-        elif p in ('left','right'):
+        elif p in ('left', 'right'):
-            style = ('float:%s' % p)+(';%s' % style if style else '')
+            style = ('float:%s' % p) + (';%s' % style if style else '')
         if t and regex_auto.match(t):
             p_begin = p_begin + '<a href="%s">' % t
             p_end = '</a>' + p_end
             t = ''
         if style:
             style = ' style="%s"' % style
-        if p in ('video','audio'):
+        if p in ('video', 'audio'):
             t = render(t, {}, {}, 'br', URL, environment, latex,
                        autolinks, protolinks, class_prefix, id_prefix, pretty_print)
             return '<%(p)s controls="controls"%(title)s%(style)s><source src="%(k)s" />%(t)s</%(p)s>' \
-                    % dict(p=p, title=title, style=style, k=k, t=t)
+                   % dict(p=p, title=title, style=style, k=k, t=t)
-        alt = ' alt="%s"'%escape(t).replace(META, DISABLED_META) if t else ''
+        alt = ' alt="%s"' % escape(t).replace(META, DISABLED_META) if t else ''
         return '%(begin)s<img src="%(k)s"%(alt)s%(title)s%(style)s />%(end)s' \
-                % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)
+               % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)
 
     def sub_link(m):
-        t,a,k,p = m.group('t','a','k','p')
+        t, a, k, p = m.group('t', 'a', 'k', 'p')
         if not k and not t:
             return m.group(0)
         t = t or ''
         a = escape(a) if a else ''
         if k:
-            if '#' in k and not ':' in k.split('#')[0]: 
+            if '#' in k and ':' not in k.split('#')[0]:
                 # wikipage, not external url
-                k=k.replace('#','#'+id_prefix)
+                k = k.replace('#', '#' + id_prefix)
             k = escape(k)
             title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
             target = ' target="_blank"' if p == 'popup' else ''
@@ -1347,18 +1375,18 @@ def render(text,
             return '<a href="%(k)s"%(title)s%(target)s>%(t)s</a>' \
                    % dict(k=k, title=title, target=target, t=t)
         if t == 'NEWLINE' and not a:
-            return '<br />'+pp
+            return '<br />' + pp
         return '<span class="anchor" id="%s">%s</span>' % (
-            escape(id_prefix+t),
+            escape(id_prefix + t),
-            render(a, {},{},'br', URL,
+            render(a, {}, {}, 'br', URL,
                    environment, latex, autolinks,
                    protolinks, class_prefix,
                    id_prefix, pretty_print))
 
     parts = text.split(LINK)
     text = parts[0]
-    for i,s in enumerate(links):
+    for i, s in enumerate(links):
-        if s == None:
+        if s is None:
             html = LINK
         else:
             html = regex_media_level2.sub(sub_media, s)
@@ -1366,51 +1394,53 @@ def render(text,
                 html = regex_link_level2.sub(sub_link, html)
             if html == s:
                 # return unprocessed string as a signal of an error
-                html = '[[%s]]'%s
+                html = '[[%s]]' % s
-        text += html + parts[i+1]
+        text += html + parts[i + 1]
 
     #############################################################
     # process all code text
     #############################################################
     def expand_meta(m):
-        code,b,p,s = segments.pop(0)
+        code, b, p, s = segments.pop(0)
-        if code==None or m.group() == DISABLED_META:
+        if code is None or m.group() == DISABLED_META:
             return escape(s)
         if b in extra:
-            if code[:1]=='\n': code=code[1:]
+            if code[:1] == '\n':
-            if code[-1:]=='\n': code=code[:-1]
+                code = code[1:]
+            if code[-1:] == '\n':
+                code = code[:-1]
             if p:
-                return str(extra[b](code,p))
+                return str(extra[b](code, p))
             else:
                 return str(extra[b](code))
-        elif b=='cite':
+        elif b == 'cite':
-            return '['+','.join('<a href="#%s" class="%s">%s</a>' \
+            return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
-                                   % (id_prefix+d,b,d) \
+                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
-                                   for d in escape(code).split(','))+']'
+        elif b == 'latex':
-        elif b=='latex':
             return LATEX % urllib.quote(code)
         elif b in html_colors:
             return '<span style="color: %s">%s</span>' \
-                  % (b, render(code, {}, {}, 'br', URL, environment, latex,
+                   % (b, render(code, {}, {}, 'br', URL, environment, latex,
-                      autolinks, protolinks, class_prefix, id_prefix, pretty_print))
+                                autolinks, protolinks, class_prefix, id_prefix, pretty_print))
         elif b in ('c', 'color') and p:
-             c=p.split(':')
+            c = p.split(':')
-             fg='color: %s;' % c[0] if c[0] else ''
+            fg = 'color: %s;' % c[0] if c[0] else ''
-             bg='background-color: %s;' % c[1] if len(c)>1 and c[1] else ''
+            bg = 'background-color: %s;' % c[1] if len(c) > 1 and c[1] else ''
-             return '<span style="%s%s">%s</span>' \
+            return '<span style="%s%s">%s</span>' \
-                 % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
+                   % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
-                    autolinks, protolinks, class_prefix, id_prefix, pretty_print))
+                                     autolinks, protolinks, class_prefix, id_prefix, pretty_print))
-        cls = ' class="%s%s"'%(class_prefix,b) if b and b != 'id' else ''
+        cls = ' class="%s%s"' % (class_prefix, b) if b and b != 'id' else ''
-        id  = ' id="%s%s"'%(id_prefix,escape(p)) if p else ''
+        id = ' id="%s%s"' % (id_prefix, escape(p)) if p else ''
-        beg=(code[:1]=='\n')
+        beg = (code[:1] == '\n')
-        end=[None,-1][code[-1:]=='\n']
+        end = [None, -1][code[-1:] == '\n']
         if beg and end:
             return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
         return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+
     text = regex_expand_meta.sub(expand_meta, text)
 
     if environment:
-        text = replace_components(text,environment)
+        text = replace_components(text, environment)
 
     return text.translate(ttab_out)
 
@@ -1423,16 +1453,18 @@ def markmin2html(text, extra={}, allowed={}, sep='p',
                   class_prefix=class_prefix, id_prefix=id_prefix,
                   pretty_print=pretty_print)
 
+
 def run_doctests():
     import doctest
     doctest.testmod()
 
+
 if __name__ == '__main__':
     import sys
     import doctest
     from textwrap import dedent
 
-    html=dedent("""
+    html = dedent("""
          <!doctype html>
          <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
          <head>
@@ -1446,7 +1478,7 @@ if __name__ == '__main__':
          </html>""")[1:]
 
     if sys.argv[1:2] == ['-h']:
-        style=dedent("""
+        style = dedent("""
               <style>
                 blockquote { background-color: #FFFAAE; padding: 7px; }
                 table { border-collapse: collapse; }
@@ -1467,22 +1499,23 @@ if __name__ == '__main__':
                           body=markmin2html(__doc__, pretty_print=True))
     elif sys.argv[1:2] == ['-t']:
         from timeit import Timer
-        loops=1000
+
-        ts = Timer("markmin2html(__doc__)","from markmin2html import markmin2html")
+        loops = 1000
+        ts = Timer("markmin2html(__doc__)", "from markmin2html import markmin2html")
         print 'timeit "markmin2html(__doc__)":'
         t = min([ts.timeit(loops) for i in range(3)])
-        print "%s loops, best of 3: %.3f ms per loop" % (loops, t/1000*loops)
+        print "%s loops, best of 3: %.3f ms per loop" % (loops, t / 1000 * loops)
     elif len(sys.argv) > 1:
-        fargv = open(sys.argv[1],'r')
+        fargv = open(sys.argv[1], 'r')
         try:
-            markmin_text=fargv.read()
+            markmin_text = fargv.read()
 
             # embed css file from second parameter into html file
             if len(sys.argv) > 2:
                 if sys.argv[2].startswith('@'):
-                    markmin_style = '<link rel="stylesheet" href="'+sys.argv[2][1:]+'"/>'
+                    markmin_style = '<link rel="stylesheet" href="' + sys.argv[2][1:] + '"/>'
                 else:
-                    fargv2 = open(sys.argv[2],'r')
+                    fargv2 = open(sys.argv[2], 'r')
                     try:
                         markmin_style = "<style>\n" + fargv2.read() + "</style>"
                     finally:
@@ -1496,10 +1529,9 @@ if __name__ == '__main__':
             fargv.close()
 
     else:
-        print "Usage: "+sys.argv[0]+" -h | -t | file.markmin [file.css|@path_to/css]"
+        print "Usage: " + sys.argv[0] + " -h | -t | file.markmin [file.css|@path_to/css]"
         print "where: -h  - print __doc__"
         print "       -t  - timeit __doc__ (for testing purpuse only)"
         print "       file.markmin  [file.css] - process file.markmin + built in file.css (optional)"
         print "       file.markmin  [@path_to/css] - process file.markmin + link path_to/css (optional)"
         run_doctests()
-        

gluon/contrib/markmin/markmin2latex.py

@@ -7,53 +7,57 @@ import sys
 import doctest
 from optparse import OptionParser
 
-__all__ = ['render','markmin2latex']
+__all__ = ['render', 'markmin2latex']
 
 META = 'META'
 regex_newlines = re.compile('(\n\r)|(\r\n)')
-regex_dd=re.compile('\$\$(?P<latex>.*?)\$\$')
+regex_dd = re.compile('\$\$(?P<latex>.*?)\$\$')
-regex_code = re.compile('('+META+')|(``(?P<t>.*?)``(:(?P<c>\w+))?)',re.S)
+regex_code = re.compile('(' + META + ')|(``(?P<t>.*?)``(:(?P<c>\w+))?)', re.S)
-regex_title = re.compile('^#{1} (?P<t>[^\n]+)',re.M)
+regex_title = re.compile('^#{1} (?P<t>[^\n]+)', re.M)
 regex_maps = [
-    (re.compile('[ \t\r]+\n'),'\n'),
+    (re.compile('[ \t\r]+\n'), '\n'),
-    (re.compile('\*\*(?P<t>[^\s\*]+( +[^\s\*]+)*)\*\*'),'{\\\\bf \g<t>}'),
+    (re.compile('\*\*(?P<t>[^\s\*]+( +[^\s\*]+)*)\*\*'), '{\\\\bf \g<t>}'),
-    (re.compile("''(?P<t>[^\s']+( +[^\s']+)*)''"),'{\\it \g<t>}'),
+    (re.compile("''(?P<t>[^\s']+( +[^\s']+)*)''"), '{\\it \g<t>}'),
-    (re.compile('^#{5,6}\s*(?P<t>[^\n]+)',re.M),'\n\n{\\\\bf \g<t>}\n'),
+    (re.compile('^#{5,6}\s*(?P<t>[^\n]+)', re.M), '\n\n{\\\\bf \g<t>}\n'),
-    (re.compile('^#{4}\s*(?P<t>[^\n]+)',re.M),'\n\n\\\\goodbreak\\subsubsection{\g<t>}\n'),
+    (re.compile('^#{4}\s*(?P<t>[^\n]+)', re.M), '\n\n\\\\goodbreak\\subsubsection{\g<t>}\n'),
-    (re.compile('^#{3}\s*(?P<t>[^\n]+)',re.M),'\n\n\\\\goodbreak\\subsection{\g<t>}\n'),
+    (re.compile('^#{3}\s*(?P<t>[^\n]+)', re.M), '\n\n\\\\goodbreak\\subsection{\g<t>}\n'),
-    (re.compile('^#{2}\s*(?P<t>[^\n]+)',re.M),'\n\n\\\\goodbreak\\section{\g<t>}\n'),
+    (re.compile('^#{2}\s*(?P<t>[^\n]+)', re.M), '\n\n\\\\goodbreak\\section{\g<t>}\n'),
-    (re.compile('^#{1}\s*(?P<t>[^\n]+)',re.M),''),
+    (re.compile('^#{1}\s*(?P<t>[^\n]+)', re.M), ''),
-    (re.compile('^\- +(?P<t>.*)',re.M),'\\\\begin{itemize}\n\\item \g<t>\n\\end{itemize}'),
+    (re.compile('^\- +(?P<t>.*)', re.M), '\\\\begin{itemize}\n\\item \g<t>\n\\end{itemize}'),
-    (re.compile('^\+ +(?P<t>.*)',re.M),'\\\\begin{itemize}\n\\item \g<t>\n\\end{itemize}'),
+    (re.compile('^\+ +(?P<t>.*)', re.M), '\\\\begin{itemize}\n\\item \g<t>\n\\end{itemize}'),
-    (re.compile('\\\\end\{itemize\}\s+\\\\begin\{itemize\}'),'\n'),
+    (re.compile('\\\\end\{itemize\}\s+\\\\begin\{itemize\}'), '\n'),
-    (re.compile('\n\s+\n'),'\n\n')]
+    (re.compile('\n\s+\n'), '\n\n')]
-regex_table = re.compile('^\-{4,}\n(?P<t>.*?)\n\-{4,}(:(?P<c>\w+))?\n',re.M|re.S)
+regex_table = re.compile('^\-{4,}\n(?P<t>.*?)\n\-{4,}(:(?P<c>\w+))?\n', re.M | re.S)
 
 regex_anchor = re.compile('\[\[(?P<t>\S+)\]\]')
 regex_bibitem = re.compile('\-\s*\[\[(?P<t>\S+)\]\]')
 regex_image_width = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+) +(?P<p>left|right|center) +(?P<w>\d+px)\]\]')
 regex_image = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+) +(?P<p>left|right|center)\]\]')
-#regex_video = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+) +video\]\]')
+# regex_video = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+) +video\]\]')
-#regex_audio = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+) +audio\]\]')
+# regex_audio = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+) +audio\]\]')
 regex_link = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+)\]\]')
-regex_auto = re.compile('(?<!["\w])(?P<k>\w+://[\w\.\-\?&%\:]+)',re.M)
+regex_auto = re.compile('(?<!["\w])(?P<k>\w+://[\w\.\-\?&%\:]+)', re.M)
 regex_commas = re.compile('[ ]+(?P<t>[,;\.])')
 regex_noindent = re.compile('\n\n(?P<t>[a-z])')
-#regex_quote_left = re.compile('"(?=\w)')
-#regex_quote_right = re.compile('(?=\w\.)"')
 
-def latex_escape(text,pound=True):
+
-    text=text.replace('\\','{\\textbackslash}')
+# regex_quote_left = re.compile('"(?=\w)')
-    for c in '^_&$%{}': text=text.replace(c,'\\'+c)
+# regex_quote_right = re.compile('(?=\w\.)"')
-    text=text.replace('\\{\\textbackslash\\}','{\\textbackslash}')
+
-    if pound: text=text.replace('#','\\#')
+def latex_escape(text, pound=True):
+    text = text.replace('\\', '{\\textbackslash}')
+    for c in '^_&$%{}':
+        text = text.replace(c, '\\' + c)
+    text = text.replace('\\{\\textbackslash\\}', '{\\textbackslash}')
+    if pound: text = text.replace('#', '\\#')
     return text
 
+
 def render(text,
            extra={},
            allowed={},
            sep='p',
-           image_mapper=lambda x:x,
+           image_mapper=lambda x: x,
            chapters=False):
     #############################################################
     # replace all blocks marked with ``...``:class with META
@@ -61,62 +65,68 @@ def render(text,
     #############################################################
     text = str(text or '')
     segments, i = [], 0
-    text = regex_dd.sub('``\g<latex>``:latex ',text)
+    text = regex_dd.sub('``\g<latex>``:latex ', text)
-    text = regex_newlines.sub('\n',text)
+    text = regex_newlines.sub('\n', text)
     while True:
-        item = regex_code.search(text,i)
+        item = regex_code.search(text, i)
-        if not item: break
+        if not item:
-        if item.group()==META:
+            break
-            segments.append((None,None))
+        if item.group() == META:
-            text = text[:item.start()]+META+text[item.end():]
+            segments.append((None, None))
+            text = text[:item.start()] + META + text[item.end():]
         else:
             c = item.group('c') or ''
-            if 'code' in allowed and not c in allowed['code']: c = ''
+            if 'code' in allowed and c not in allowed['code']:
-            code = item.group('t').replace('!`!','`')
+                c = ''
-            segments.append((code,c))
+            code = item.group('t').replace('!`!', '`')
-            text = text[:item.start()]+META+text[item.end():]
+            segments.append((code, c))
-        i=item.start()+3
+            text = text[:item.start()] + META + text[item.end():]
-
+        i = item.start() + 3
 
     #############################################################
     # do h1,h2,h3,h4,h5,h6,b,i,ol,ul and normalize spaces
     #############################################################
 
     title = regex_title.search(text)
-    if not title: title='Title'
+    if not title:
-    else: title=title.group('t')
+        title = 'Title'
+    else:
+        title = title.group('t')
 
-    text = latex_escape(text,pound=False)
+    text = latex_escape(text, pound=False)
 
-    texts = text.split('## References',1)
+    texts = text.split('## References', 1)
     text = regex_anchor.sub('\\label{\g<t>}', texts[0])
-    if len(texts)==2:
+    if len(texts) == 2:
         text += '\n\\begin{thebibliography}{999}\n'
         text += regex_bibitem.sub('\n\\\\bibitem{\g<t>}', texts[1])
         text += '\n\\end{thebibliography}\n'
 
     text = '\n'.join(t.strip() for t in text.split('\n'))
     for regex, sub in regex_maps:
-        text = regex.sub(sub,text)
+        text = regex.sub(sub, text)
-    text=text.replace('#','\\#')
+    text = text.replace('#', '\\#')
-    text=text.replace('`',"'")
+    text = text.replace('`', "'")
 
     #############################################################
     # process tables and blockquotes
     #############################################################
     while True:
         item = regex_table.search(text)
-        if not item: break
+        if not item:
+            break
         c = item.group('c') or ''
-        if 'table' in allowed and not c in allowed['table']: c = ''
+        if 'table' in allowed and c not in allowed['table']:
+            c = ''
         content = item.group('t')
         if ' | ' in content:
-            rows = content.replace('\n','\\\\\n').replace(' | ',' & ')
+            rows = content.replace('\n', '\\\\\n').replace(' | ', ' & ')
-            row0,row2 = rows.split('\\\\\n',1)
+            row0, row2 = rows.split('\\\\\n', 1)
-            cols=row0.count(' & ')+1
+            cols = row0.count(' & ') + 1
-            cal='{'+''.join('l' for j in range(cols))+'}'
+            cal = '{' + ''.join('l' for j in range(cols)) + '}'
-            tabular = '\\begin{center}\n{\\begin{tabular}'+cal+'\\hline\n' + row0+'\\\\ \\hline\n'+row2 + ' \\\\ \\hline\n\\end{tabular}}\n\\end{center}'
+            tabular = '\\begin{center}\n{\\begin{tabular}' + cal + '\\hline\n' + row0 + '\\\\ \\hline\n' + row2 + ' \\\\ \\hline\n\\end{tabular}}\n\\end{center}'
-            if row2.count('\n')>20: tabular='\\newpage\n'+tabular
+            if row2.count('\n') > 20:
+                tabular = '\\newpage\n' + tabular
             text = text[:item.start()] + tabular + text[item.end():]
         else:
             text = text[:item.start()] + '\\begin{quote}' + content + '\\end{quote}' + text[item.end():]
@@ -126,29 +136,32 @@ def render(text,
     #############################################################
 
     def sub(x):
-        f=image_mapper(x.group('k'))
+        f = image_mapper(x.group('k'))
-        if not f: return None
+        if not f:
-        return '\n\\begin{center}\\includegraphics[width=8cm]{%s}\\end{center}\n' % (f)
+            return None
-    text = regex_image_width.sub(sub,text)
+        return '\n\\begin{center}\\includegraphics[width=8cm]{%s}\\end{center}\n' % f
-    text = regex_image.sub(sub,text)
+
+    text = regex_image_width.sub(sub, text)
+    text = regex_image.sub(sub, text)
 
     text = regex_link.sub('{\\\\footnotesize\\href{\g<k>}{\g<t>}}', text)
-    text = regex_commas.sub('\g<t>',text)
+    text = regex_commas.sub('\g<t>', text)
-    text = regex_noindent.sub('\n\\\\noindent \g<t>',text)
+    text = regex_noindent.sub('\n\\\\noindent \g<t>', text)
 
-    ### fix paths in images
+    # ## fix paths in images
-    regex=re.compile('\\\\_\w*\.(eps|png|jpg|gif)')
+    regex = re.compile('\\\\_\w*\.(eps|png|jpg|gif)')
     while True:
-        match=regex.search(text)
+        match = regex.search(text)
-        if not match: break
+        if not match:
-        text=text[:match.start()]+text[match.start()+1:]
+            break
-    #text = regex_quote_left.sub('``',text)
+        text = text[:match.start()] + text[match.start() + 1:]
-    #text = regex_quote_right.sub("''",text)
+    # text = regex_quote_left.sub('``',text)
+    # text = regex_quote_right.sub("''",text)
 
     if chapters:
-        text=text.replace(r'\section*{',r'\chapter*{')
+        text = text.replace(r'\section*{', r'\chapter*{')
-        text=text.replace(r'\section{',r'\chapter{')
+        text = text.replace(r'\section{', r'\chapter{')
-        text=text.replace(r'subsection{',r'section{')
+        text = text.replace(r'subsection{', r'section{')
 
     #############################################################
     # process all code text
@@ -156,57 +169,64 @@ def render(text,
     parts = text.split(META)
     text = parts[0]
     authors = []
-    for i,(code,b) in enumerate(segments):
+    for i, (code, b) in enumerate(segments):
-        if code==None:
+        if code is None:
             html = META
         else:
-            if b=='hidden':
+            if b == 'hidden':
-                html=''
+                html = ''
-            elif b=='author':
+            elif b == 'author':
                 author = latex_escape(code.strip())
                 authors.append(author)
-                html=''
+                html = ''
-            elif b=='inxx':
+            elif b == 'inxx':
-                html='\inxx{%s}' % latex_escape(code)
+                html = '\inxx{%s}' % latex_escape(code)
-            elif b=='cite':
+            elif b == 'cite':
-                html='~\cite{%s}' % latex_escape(code.strip())
+                html = '~\cite{%s}' % latex_escape(code.strip())
-            elif b=='ref':
+            elif b == 'ref':
-                html='~\ref{%s}' % latex_escape(code.strip())
+                html = '~\ref{%s}' % latex_escape(code.strip())
-            elif b=='latex':
+            elif b == 'latex':
                 if '\n' in code:
-                    html='\n\\begin{equation}\n%s\n\\end{equation}\n' % code.strip()
+                    html = '\n\\begin{equation}\n%s\n\\end{equation}\n' % code.strip()
                 else:
-                    html='$%s$' % code.strip()
+                    html = '$%s$' % code.strip()
-            elif b=='latex_eqnarray':
+            elif b == 'latex_eqnarray':
-                code=code.strip()
+                code = code.strip()
-                code='\\\\'.join(x.replace('=','&=&',1) for x in code.split('\\\\'))
+                code = '\\\\'.join(x.replace('=', '&=&', 1) for x in code.split('\\\\'))
-                html='\n\\begin{eqnarray}\n%s\n\\end{eqnarray}\n' % code
+                html = '\n\\begin{eqnarray}\n%s\n\\end{eqnarray}\n' % code
             elif b.startswith('latex_'):
-                key=b[6:]
+                key = b[6:]
-                html='\\begin{%s}%s\\end{%s}' % (key,code,key)
+                html = '\\begin{%s}%s\\end{%s}' % (key, code, key)
             elif b in extra:
-                if code[:1]=='\n': code=code[1:]
+                if code[:1] == '\n':
-                if code[-1:]=='\n': code=code[:-1]
+                    code = code[1:]
+                if code[-1:] == '\n':
+                    code = code[:-1]
                 html = extra[b](code)
-            elif code[:1]=='\n' or code[:-1]=='\n':
+            elif code[:1] == '\n' or code[:-1] == '\n':
-                if code[:1]=='\n': code=code[1:]
+                if code[:1] == '\n':
-                if code[-1:]=='\n': code=code[:-1]
+                    code = code[1:]
+                if code[-1:] == '\n':
+                    code = code[:-1]
                 if code.startswith('<') or code.startswith('{{') or code.startswith('http'):
                     html = '\\begin{lstlisting}[keywords={}]\n%s\n\\end{lstlisting}' % code
                 else:
                     html = '\\begin{lstlisting}\n%s\n\\end{lstlisting}' % code
             else:
-                if code[:1]=='\n': code=code[1:]
+                if code[:1] == '\n':
-                if code[-1:]=='\n': code=code[:-1]
+                    code = code[1:]
+                if code[-1:] == '\n':
+                    code = code[:-1]
                 html = '{\\ft %s}' % latex_escape(code)
         try:
-            text = text+html+parts[i+1]
+            text = text + html + parts[i + 1]
         except:
             text = text + '... WIKI PROCESSING ERROR ...'
             break
-    text =  text.replace(' ~\\cite','~\\cite')
+    text = text.replace(' ~\\cite', '~\\cite')
     return text, title, authors
 
+
 WRAPPER = """
 \\documentclass[12pt]{article}
 \\usepackage{hyperref}
@@ -239,12 +259,14 @@ WRAPPER = """
 \\end{document}
 """
 
-def markmin2latex(data, image_mapper=lambda x:x, extra={},
+
+def markmin2latex(data, image_mapper=lambda x: x, extra={},
                   wrapper=WRAPPER):
     body, title, authors = render(data, extra=extra, image_mapper=image_mapper)
-    author = '\n\\and\n'.join(a.replace('\n','\\\\\n\\footnotesize ') for a in authors)
+    author = '\n\\and\n'.join(a.replace('\n', '\\\\\n\\footnotesize ') for a in authors)
     return wrapper % dict(title=title, author=author, body=body)
 
+
 if __name__ == '__main__':
     parser = OptionParser()
     parser.add_option("-i", "--info", dest="info",
@@ -252,40 +274,39 @@ if __name__ == '__main__':
     parser.add_option("-t", "--test", dest="test", action="store_true",
                       default=False)
     parser.add_option("-n", "--no_wrapper", dest="no_wrapper",
-                      action="store_true",default=False)
+                      action="store_true", default=False)
-    parser.add_option("-c", "--chapters", dest="chapters",action="store_true",
+    parser.add_option("-c", "--chapters", dest="chapters", action="store_true",
-                      default=False,help="switch section for chapter")
+                      default=False, help="switch section for chapter")
     parser.add_option("-w", "--wrapper", dest="wrapper", default=False,
                       help="latex file containing header and footer")
 
     (options, args) = parser.parse_args()
     if options.info:
         import markmin2html
+
         markmin2latex(markmin2html.__doc__)
     elif options.test:
         doctest.testmod()
     else:
         if options.wrapper:
-            fwrapper = open(options.wrapper,'rb')
+            fwrapper = open(options.wrapper, 'rb')
             try:
                 wrapper = fwrapper.read()
             finally:
                 fwrapper.close()
         elif options.no_wrapper:
-            wrapper  = '%(body)s'
+            wrapper = '%(body)s'
         else:
             wrapper = WRAPPER
         for f in args:
-            fargs = open(f,'r')
+            fargs = open(f, 'r')
             content_data = []
             try:
                 content_data.append(fargs.read())
             finally:
                 fargs.close()
         content = '\n'.join(content_data)
-        output= markmin2latex(content,
+        output = markmin2latex(content,
-                              wrapper=wrapper,
+                               wrapper=wrapper,
-                              chapters=options.chapters)
+                               chapters=options.chapters)
         print output
-
-

gluon/contrib/markmin/markmin2pdf.py

@@ -13,21 +13,22 @@ from markmin2latex import markmin2latex
 
 __all__ = ['markmin2pdf']
 
-def removeall(path):
 
-    ERROR_STR= """Error removing %(path)s, %(error)s """
+def removeall(path):
+    ERROR_STR = """Error removing %(path)s, %(error)s """
+
     def rmgeneric(path, __func__):
         try:
             __func__(path)
         except OSError, (errno, strerror):
-            print ERROR_STR % {'path' : path, 'error': strerror }
+            print ERROR_STR % {'path': path, 'error': strerror}
 
-    files=[path]
+    files = [path]
 
     while files:
-        file=files[0]
+        file = files[0]
         if os.path.isfile(file):
-            f=os.remove
+            f = os.remove
             rmgeneric(file, os.remove)
             del files[0]
         elif os.path.isdir(file):
@@ -36,7 +37,7 @@ def removeall(path):
                 rmgeneric(file, os.rmdir)
                 del files[0]
             else:
-                files = [os.path.join(file,x) for x in nested] + files
+                files = [os.path.join(file, x) for x in nested] + files
 
 
 def latex2pdf(latex, pdflatex='pdflatex', passes=3):
@@ -49,13 +50,13 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
     - passes:   defines how often pdflates should be run in the texfile.
     """
 
-    pdflatex=pdflatex
+    pdflatex = pdflatex
-    passes=passes
+    passes = passes
-    warnings=[]
+    warnings = []
 
     # setup the envoriment
     tmpdir = mkdtemp()
-    texfile = open(tmpdir+'/test.tex','wb')
+    texfile = open(tmpdir + '/test.tex', 'wb')
     texfile.write(latex)
     texfile.seek(0)
     texfile.close()
@@ -63,8 +64,8 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
 
     # start doing some work
     for i in range(0, passes):
-        logfd,logname = mkstemp()
+        logfd, logname = mkstemp()
-        outfile=os.fdopen(logfd)
+        outfile = os.fdopen(logfd)
         try:
             ret = subprocess.call([pdflatex,
                                    '-interaction=nonstopmode',
@@ -75,18 +76,18 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
                                   stderr=subprocess.PIPE)
         finally:
             outfile.close()
-        re_errors=re.compile('^\!(.*)$',re.M)
+        re_errors = re.compile('^\!(.*)$', re.M)
-        re_warnings=re.compile('^LaTeX Warning\:(.*)$',re.M)
+        re_warnings = re.compile('^LaTeX Warning\:(.*)$', re.M)
         flog = open(logname)
         try:
             loglines = flog.read()
         finally:
             flog.close()
-        errors=re_errors.findall(loglines)
+        errors = re_errors.findall(loglines)
-        warnings=re_warnings.findall(loglines)
+        warnings = re_warnings.findall(loglines)
         os.unlink(logname)
 
-    pdffile=texfile.rsplit('.',1)[0]+'.pdf'
+    pdffile = texfile.rsplit('.', 1)[0] + '.pdf'
     if os.path.isfile(pdffile):
         fpdf = open(pdffile, 'rb')
         try:
@@ -100,31 +101,31 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
 
 
 def markmin2pdf(text, image_mapper=lambda x: None, extra={}):
-    return latex2pdf(markmin2latex(text,image_mapper=image_mapper, extra=extra))
+    return latex2pdf(markmin2latex(text, image_mapper=image_mapper, extra=extra))
 
 
 if __name__ == '__main__':
     import sys
     import doctest
     import markmin2html
-    if sys.argv[1:2]==['-h']:
+
+    if sys.argv[1:2] == ['-h']:
         data, warnings, errors = markmin2pdf(markmin2html.__doc__)
         if errors:
-            print 'ERRORS:'+'\n'.join(errors)
+            print 'ERRORS:' + '\n'.join(errors)
-            print 'WARNGINS:'+'\n'.join(warnings)
+            print 'WARNGINS:' + '\n'.join(warnings)
         else:
             print data
-    elif len(sys.argv)>1:
+    elif len(sys.argv) > 1:
-        fargv = open(sys.argv[1],'rb')
+        fargv = open(sys.argv[1], 'rb')
         try:
             data, warnings, errors = markmin2pdf(fargv.read())
         finally:
             fargv.close()
         if errors:
-            print 'ERRORS:'+'\n'.join(errors)
+            print 'ERRORS:' + '\n'.join(errors)
-            print 'WARNGINS:'+'\n'.join(warnings)
+            print 'WARNGINS:' + '\n'.join(warnings)
         else:
             print data
     else:
         doctest.testmod()
-

gluon/contrib/redis_scheduler.py

@@ -9,6 +9,17 @@ Scheduler with redis backend
 ---------------------------------
 """
 
+import os
+import time
+import socket
+import datetime
+import logging
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+from gluon.scheduler import *
+from gluon.scheduler import _decode_dict
+from gluon.contrib.redis_utils import RWatchError
+
 USAGE = """
 ## Example
 
@@ -35,11 +46,6 @@ mysched = RScheduler(db, dict(demo1=demo1,demo2=demo2), ...., redis_conn=rconn)
 
 """
 
-import os
-import time
-import socket
-import datetime
-import logging
 
 path = os.getcwd()
 
@@ -47,20 +53,19 @@ if 'WEB2PY_PATH' not in os.environ:
     os.environ['WEB2PY_PATH'] = path
 
 try:
-    from gluon.contrib.simplejson import loads, dumps
+    # try external module
-except:
     from simplejson import loads, dumps
+except ImportError:
+    try:
+        # try stdlib (Python >= 2.6)
+        from json import loads, dumps
+    except:
+        # fallback to pure-Python module
+        from gluon.contrib.simplejson import loads, dumps
 
 IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
-logger = logging.getLogger('web2py.rscheduler.%s' % IDENTIFIER)
+logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
-
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-from gluon.scheduler import *
-from gluon.scheduler import _decode_dict
-from gluon.contrib.redis_utils import RWatchError
-
 
 POLLING = 'POLLING'
 
@@ -111,8 +116,7 @@ class RScheduler(Scheduler):
         self._application = current.request.application or 'appname'
 
     def _nkey(self, key):
-        """Helper to restrict all keys to a namespace
+        """Helper to restrict all keys to a namespace and track them."""
-        and track them"""
         prefix = 'w2p:rsched:%s' % self._application
         allkeys = '%s:allkeys' % prefix
         newkey = "%s:%s" % (prefix, key)
@@ -120,10 +124,7 @@ class RScheduler(Scheduler):
         return newkey
 
     def prune_all(self):
-        """
+        """Global housekeeping."""
-        Just to be fair and implement a method
-        that does housekeeping
-        """
         all_keys = self._nkey('allkeys')
         with self.r_server.pipeline() as pipe:
             while True:
@@ -148,8 +149,9 @@ class RScheduler(Scheduler):
 
     def send_heartbeat(self, counter):
         """
-        workers coordination has evolved into something is not that
+        Workers coordination in redis.
-        easy. Here we try to do what we need in a single transaction,
+        It has evolved into something is not that easy.
+        Here we try to do what we need in a single transaction,
         and retry that transaction if something goes wrong
         """
         with self.r_server.pipeline() as pipe:
@@ -167,7 +169,9 @@ class RScheduler(Scheduler):
 
     def inner_send_heartbeat(self, counter, pipe):
         """
-        Does a few things:
+        Do a few things in the "maintenance" thread.
+
+        Specifically:
         - registers the workers
         - accepts commands sent to workers (KILL, TERMINATE, PICK, DISABLED, etc)
         - adjusts sleep
@@ -269,6 +273,8 @@ class RScheduler(Scheduler):
 
     def being_a_ticker(self, pipe):
         """
+        Elects a ticker.
+
         This is slightly more convoluted than the original
         but if far more efficient
         """
@@ -311,7 +317,9 @@ class RScheduler(Scheduler):
 
     def assign_tasks(self, db):
         """
-        The real beauty. We don't need to ASSIGN tasks, we just put
+        The real beauty.
+
+        We don't need to ASSIGN tasks, we just put
         them into the relevant queue
         """
         st, sd = db.scheduler_task, db.scheduler_task_deps
@@ -375,9 +383,6 @@ class RScheduler(Scheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -437,6 +442,7 @@ class RScheduler(Scheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def pop_task(self, db):
+        """Lift a task off a queue."""
         r_server = self.r_server
         st = self.db.scheduler_task
         task = None
@@ -533,7 +539,9 @@ class RScheduler(Scheduler):
 
     def report_task(self, task, task_report):
         """
-        Needs overwriting only because we need to pop from the
+        Override.
+
+        Needs it only because we need to pop from the
         running tasks
         """
         r_server = self.r_server
@@ -558,12 +566,12 @@ class RScheduler(Scheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
+        is_expired = (task.stop_time and
-                      and task.next_run_time > task.stop_time
+                      task.next_run_time > task.stop_time and
-                      and True or False)
+                      True or False)
-        status = (task.run_again and is_expired and EXPIRED
+        status = (task.run_again and is_expired and EXPIRED or
-                  or task.run_again and not is_expired
+                  task.run_again and not is_expired and
-                  and QUEUED or COMPLETED)
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             # assigned calculations
             d = dict(status=status,
@@ -579,10 +587,10 @@ class RScheduler(Scheduler):
             st_mapping = {'FAILED': 'FAILED',
                           'TIMEOUT': 'TIMEOUT',
                           'STOPPED': 'FAILED'}[task_report.status]
-            status = (task.retry_failed
+            status = (task.retry_failed and
-                      and task.times_failed < task.retry_failed
+                      task.times_failed < task.retry_failed and
-                      and QUEUED or task.retry_failed == -1
+                      QUEUED or task.retry_failed == -1 and
-                      and QUEUED or st_mapping)
+                      QUEUED or st_mapping)
             db(st.id == task.task_id).update(
                 times_failed=st.times_failed + 1,
                 next_run_time=task.next_run_time,
@@ -596,7 +604,7 @@ class RScheduler(Scheduler):
         r_server.hdel(running_dict, task.task_id)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -620,8 +628,8 @@ class RScheduler(Scheduler):
                 time.sleep(0.5)
 
     def get_workers(self, only_ticker=False):
-        """ Returns a dict holding worker_name : {**columns}
+        """Return a dict holding worker_name : {**columns}
-        representing all "registered" workers
+        representing all "registered" workers.
         only_ticker returns only the worker running as a TICKER,
         if there is any
         """

gluon/dal.py

@@ -74,12 +74,12 @@ def _default_validators(db, field):
         return requires
     # does not get here for reference and list:reference
     if field.unique:
-        requires.insert(0,validators.IS_NOT_IN_DB(db, field))
+        requires.insert(0, validators.IS_NOT_IN_DB(db, field))
-    excluded_fields = ['string','upload','text','password','boolean']
+    excluded_fields = ['string', 'upload', 'text', 'password', 'boolean']
     if (field.notnull or field.unique) and not field_type in excluded_fields:
-        requires.insert(0,validators.IS_NOT_EMPTY())
+        requires.insert(0, validators.IS_NOT_EMPTY())
     elif not field.notnull and not field.unique and requires:
-        requires[0] = validators.IS_EMPTY_OR(requires[0])
+        requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field in ('string', 'text', 'password') else None)
     return requires
 
 from gluon.serializers import custom_json, xml
@@ -93,7 +93,7 @@ DAL.uuid = lambda x: web2py_uuid()
 DAL.representers = {
     'rows_render': sqlhtml.represent,
     'rows_xml': sqlhtml.SQLTABLE
-    }
+}
 DAL.Field = Field
 DAL.Table = Table
 

gluon/html.py

@@ -308,7 +308,7 @@ def URL(a=None,
         else:
             function = f
 
-        # if the url gets a static resource, don't force extention
+        # if the url gets a static resource, don't force extension
         if controller == 'static':
             extension = None
             # add static version to url
@@ -317,7 +317,7 @@ def URL(a=None,
                 response = current.response
                 if response.static_version and response.static_version_urls:
                     args = [function] + args
-                    function = '_'+str(response.static_version)
+                    function = '_' + str(response.static_version)
 
         if '.' in function:
             function, extension = function.rsplit('.', 1)
@@ -330,18 +330,16 @@ def URL(a=None,
     if args:
         if url_encode:
             if encode_embedded_slash:
-                other = '/' + '/'.join([urllib.quote(str(
+                other = '/' + '/'.join([urllib.quote(str(x), '') for x in args])
-                    x), '') for x in args])
             else:
-                other = args and urllib.quote(
+                other = args and urllib.quote('/' + '/'.join([str(x) for x in args]))
-                    '/' + '/'.join([str(x) for x in args]))
         else:
             other = args and ('/' + '/'.join([str(x) for x in args]))
     else:
         other = ''
 
     if other.endswith('/'):
-        other += '/'    # add trailing slash to make last trailing empty arg explicit
+        other += '/'  # add trailing slash to make last trailing empty arg explicit
 
     list_vars = []
     for (key, vals) in sorted(vars.items()):
@@ -364,11 +362,11 @@ def URL(a=None,
         h_args = '/%s/%s/%s%s' % (application, controller, function2, other)
 
         # how many of the vars should we include in our hash?
-        if hash_vars is True:       # include them all
+        if hash_vars is True:  # include them all
             h_vars = list_vars
-        elif hash_vars is False:    # include none of them
+        elif hash_vars is False:  # include none of them
             h_vars = ''
-        else:                       # include just those specified
+        else:  # include just those specified
             if hash_vars and not isinstance(hash_vars, (list, tuple)):
                 hash_vars = [hash_vars]
             h_vars = [(k, v) for (k, v) in list_vars if k in hash_vars]
@@ -663,7 +661,7 @@ class XML(XmlComponent):
         """
         to be considered experimental since the behavior of this method
         is questionable
-        another option could be `TAG(self.text).elements(*args,**kwargs)`
+        another option could be `TAG(self.text).elements(*args, **kwargs)`
         """
         return []
 
@@ -760,7 +758,7 @@ class DIV(XmlComponent):
         Examples:
 
         >>> a=DIV()
-        >>> a.insert(0,SPAN('x'))
+        >>> a.insert(0, SPAN('x'))
         >>> print a
         <div><span>x</span></div>
         """
@@ -856,7 +854,7 @@ class DIV(XmlComponent):
         """
         components = []
         for c in self.components:
-            if isinstance(c, (allowed_parents,CAT)):
+            if isinstance(c, (allowed_parents, CAT)):
                 pass
             elif wrap_lambda:
                 c = wrap_lambda(c)
@@ -955,7 +953,6 @@ class DIV(XmlComponent):
         # get the xml for the inner components
         co = join([xmlescape(component) for component in
                    self.components])
-
         return (fa, co)
 
     def xml(self):
@@ -990,7 +987,7 @@ class DIV(XmlComponent):
 
         Examples:
 
-        >>> markdown = lambda text,tag=None,attributes={}: \
+        >>> markdown = lambda text, tag=None, attributes={}: \
                         {None: re.sub('\s+',' ',text), \
                          'h1':'#'+text+'\\n\\n', \
                          'p':text+'\\n'}.get(tag,text)
@@ -1027,7 +1024,7 @@ class DIV(XmlComponent):
         Examples:
 
         >>> a = DIV(DIV(SPAN('x'),3,DIV(SPAN('y'))))
-        >>> for c in a.elements('span',first_only=True): c[0]='z'
+        >>> for c in a.elements('span', first_only=True): c[0]='z'
         >>> print a
         <div><div><span>z</span>3<div><span>y</span></div></div></div>
         >>> for c in a.elements('span'): c[0]='z'
@@ -1059,7 +1056,7 @@ class DIV(XmlComponent):
 
         >>> a = DIV(DIV(SPAN('x', _class='abc'), DIV(SPAN('y', _class='abc'), SPAN('z', _class='abc'))))
         >>> b = a.elements('span.abc', replace=P('x', _class='xyz'))
-        >>> print a
+        >>> print a  # We should .xml() here instead of print
         <div><div><p class="xyz">x</p><div><p class="xyz">x</p><p class="xyz">x</p></div></div></div>
 
         "replace" can be a callable, which will be passed the original element and
@@ -1171,13 +1168,13 @@ class DIV(XmlComponent):
                 return i
             else:
                 self[i] = replace(self[i]) if callable(replace) else replace
-                return i+1
+                return i + 1
         # loop the components
         if find_text or find_components:
             i = 0
             while i < len(self.components):
                 c = self[i]
-                j = i+1
+                j = i + 1
                 if check and find_text and isinstance(c, str) and \
                         ((is_regex and find_text.search(c)) or (str(find_text) in c)):
                     j = replace_component(i)
@@ -2573,7 +2570,7 @@ class MENU(DIV):
                         item[3], select, prefix=CAT(prefix, item[0], '/'))
         select['_onchange'] = 'window.location=this.value'
         # avoid to wrap the select if no custom items are present
-        html = DIV(select,  self.serialize(custom_items)) if len(custom_items) else select
+        html = DIV(select, self.serialize(custom_items)) if len(custom_items) else select
         return html
 
     def xml(self):
@@ -2805,7 +2802,7 @@ class MARKMIN(XmlComponent):
         self.extra = extra or {}
         self.allowed = allowed or {}
         self.sep = sep
-        self.url = URL if url == True else url
+        self.url = URL if url is True else url
         self.environment = environment
         self.latex = latex
         self.autolinks = autolinks

gluon/languages.py

@@ -1002,17 +1002,19 @@ def update_all_languages(application_path):
             findT(application_path, language[:-3])
 
 
-def update_from_langfile(target, source):
+def update_from_langfile(target, source, force_update=False):
     """this will update untranslated messages in target from source (where both are language files)
     this can be used as first step when creating language file for new but very similar language
         or if you want update your app from welcome app of newer web2py version
         or in non-standard scenarios when you work on target and from any reason you have partial translation in source
+    Args:
+        force_update: if False existing translations remain unchanged, if True existing translations will update from source
     """
     src = read_dict(source)
     sentences = read_dict(target)
     for key in sentences:
         val = sentences[key]
-        if not val or val == key:
+        if not val or val == key or force_update:
             new_val = src.get(key)
             if new_val and new_val != val:
                 sentences[key] = new_val

gluon/scheduler.py

@@ -9,6 +9,26 @@ Background processes made simple
 ---------------------------------
 """
 
+
+import os
+import time
+import multiprocessing
+import sys
+import threading
+import traceback
+import signal
+import socket
+import datetime
+import logging
+import optparse
+import tempfile
+import types
+import Queue
+from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
+from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+
 USAGE = """
 ## Example
 
@@ -67,20 +87,6 @@ sudo restart web2py-scheduler
 sudo status web2py-scheduler
 """
 
-import os
-import time
-import multiprocessing
-import sys
-import threading
-import traceback
-import signal
-import socket
-import datetime
-import logging
-import optparse
-import types
-import Queue
-
 path = os.getcwd()
 
 if 'WEB2PY_PATH' not in os.environ:
@@ -101,12 +107,6 @@ IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
 logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
-from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
-from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-
-
 QUEUED = 'QUEUED'
 ASSIGNED = 'ASSIGNED'
 RUNNING = 'RUNNING'
@@ -168,24 +168,25 @@ class TaskReport(object):
 
 
 class JobGraph(object):
-    """Experimental: with JobGraph you can specify
+    """Experimental: dependencies amongs tasks"""
-    dependencies amongs tasks"""
 
     def __init__(self, db, job_name):
         self.job_name = job_name or 'job_0'
         self.db = db
 
     def add_deps(self, task_parent, task_child):
-        """Creates a dependency between task_parent and task_child"""
+        """Create a dependency between task_parent and task_child."""
         self.db.scheduler_task_deps.insert(task_parent=task_parent,
                                            task_child=task_child,
                                            job_name=self.job_name)
 
-    def validate(self, job_name):
+    def validate(self, job_name=None):
-        """Validates if all tasks job_name can be completed, i.e. there
+        """Validate if all tasks job_name can be completed.
-        are no mutual dependencies among tasks.
+
+        Checks if there are no mutual dependencies among tasks.
         Commits at the end if successfull, or it rollbacks the entire
-        transaction. Handle with care!"""
+        transaction. Handle with care!
+        """
         db = self.db
         sd = db.scheduler_task_deps
         if job_name:
@@ -223,14 +224,6 @@ class JobGraph(object):
             db.rollback()
             return None
 
-
-def demo_function(*argv, **kwargs):
-    """ test function """
-    for i in range(argv[0]):
-        print 'click', i
-        time.sleep(1)
-    return 'done'
-
 # the two functions below deal with simplejson decoding as unicode, esp for the dict decode
 # and subsequent usage as function Keyword arguments unicode variable names won't work!
 # borrowed from http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-unicode-ones-from-json-in-python
@@ -261,11 +254,12 @@ def _decode_dict(dct):
 
 
 def executor(queue, task, out):
-    """The function used to execute tasks in the background process"""
+    """The function used to execute tasks in the background process."""
     logger.debug('    task started')
 
     class LogOutput(object):
-        """Facility to log output at intervals"""
+        """Facility to log output at intervals."""
+
         def __init__(self, out_queue):
             self.out_queue = out_queue
             self.stdout = sys.stdout
@@ -280,7 +274,11 @@ def executor(queue, task, out):
         def write(self, data):
             self.out_queue.put(data)
 
-    W2P_TASK = Storage({'id': task.task_id, 'uuid': task.uuid})
+    W2P_TASK = Storage({
+                       'id': task.task_id,
+                       'uuid': task.uuid,
+                       'run_id': task.run_id
+                       })
     stdout = LogOutput(out)
     try:
         if task.app:
@@ -318,6 +316,11 @@ def executor(queue, task, out):
             result = eval(task.function)(
                 *loads(task.args, object_hook=_decode_dict),
                 **loads(task.vars, object_hook=_decode_dict))
+        if len(result) >= 1024:
+            fd, temp_path = tempfile.mkstemp(suffix='.w2p_sched')
+            with os.fdopen(fd, 'w') as f:
+                f.write(result)
+            result = 'w2p_special:%s' % temp_path
         queue.put(TaskReport('COMPLETED', result=result))
     except BaseException, e:
         tb = traceback.format_exc()
@@ -335,7 +338,7 @@ class MetaScheduler(threading.Thread):
         self.empty_runs = 0
 
     def async(self, task):
-        """Starts the background process
+        """Start the background process.
 
         Args:
             task : a `Task` object
@@ -410,6 +413,12 @@ class MetaScheduler(threading.Thread):
             else:
                 logger.debug('  task completed or failed')
                 tr = queue.get()
+        result = tr.result
+        if result and result.startswith('w2p_special'):
+            temp_path = result.replace('w2p_special:', '', 1)
+            with open(temp_path) as f:
+                tr.result = f.read()
+            os.unlink(temp_path)
         tr.output = task_output
         return tr
 
@@ -444,50 +453,23 @@ class MetaScheduler(threading.Thread):
         self.start()
 
     def send_heartbeat(self, counter):
-        print 'thum'
+        raise NotImplementedError
-        time.sleep(1)
 
     def pop_task(self):
         """Fetches a task ready to be executed"""
-        return Task(
+        raise NotImplementedError
-            app=None,
-            function='demo_function',
-            timeout=7,
-            args='[2]',
-            vars='{}')
 
     def report_task(self, task, task_report):
         """Creates a task report"""
-        print 'reporting task'
+        raise NotImplementedError
-        pass
 
     def sleep(self):
-        pass
+        raise NotImplementedError
 
     def loop(self):
         """Main loop, fetching tasks and starting executor's background
         processes"""
-        try:
+        raise NotImplementedError
-            self.start_heartbeats()
-            while True and self.have_heartbeat:
-                logger.debug('looping...')
-                task = self.pop_task()
-                if task:
-                    self.empty_runs = 0
-                    self.report_task(task, self.async(task))
-                else:
-                    self.empty_runs += 1
-                    logger.debug('sleeping...')
-                    if self.max_empty_runs != 0:
-                        logger.debug('empty runs %s/%s',
-                                     self.empty_runs, self.max_empty_runs)
-                        if self.empty_runs >= self.max_empty_runs:
-                            logger.info(
-                                'empty runs limit reached, killing myself')
-                            self.die()
-                    self.sleep()
-        except KeyboardInterrupt:
-            self.die()
 
 
 TASK_STATUS = (QUEUED, RUNNING, COMPLETED, FAILED, TIMEOUT, STOPPED, EXPIRED)
@@ -594,11 +576,11 @@ class Scheduler(MetaScheduler):
         return True
 
     def now(self):
-        """Shortcut that fetches current time based on UTC preferences"""
+        """Shortcut that fetches current time based on UTC preferences."""
         return self.utc_time and datetime.datetime.utcnow() or datetime.datetime.now()
 
     def set_requirements(self, scheduler_task):
-        """Called to set defaults for lazy_tables connections"""
+        """Called to set defaults for lazy_tables connections."""
         from gluon import current
         if hasattr(current, 'request'):
             scheduler_task.application_name.default = '%s/%s' % (
@@ -606,7 +588,7 @@ class Scheduler(MetaScheduler):
             )
 
     def define_tables(self, db, migrate):
-        """Defines Scheduler tables structure"""
+        """Define Scheduler tables structure."""
         from pydal.base import DEFAULT
         logger.debug('defining tables (migrate=%s)', migrate)
         now = self.now
@@ -693,14 +675,14 @@ class Scheduler(MetaScheduler):
 
     @staticmethod
     def total_seconds(td):
-        # backport for py2.6
+        """Backport for py2.6."""
         if hasattr(td, 'total_seconds'):
             return td.total_seconds()
         else:
             return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6
 
     def loop(self, worker_name=None):
-        """Main loop
+        """Main loop.
 
         This works basically as a neverending loop that:
 
@@ -752,7 +734,8 @@ class Scheduler(MetaScheduler):
             self.die()
 
     def wrapped_assign_tasks(self, db):
-        """Commodity function to call `assign_tasks` and trap exceptions
+        """Commodity function to call `assign_tasks` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `assign_task` after 0.5 seconds
         """
@@ -773,7 +756,8 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -793,7 +777,7 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def pop_task(self, db):
-        """Grabs a task ready to be executed from the queue"""
+        """Grab a task ready to be executed from the queue."""
         now = self.now()
         st = self.db.scheduler_task
         if self.is_a_ticker and self.do_assign_tasks:
@@ -874,7 +858,8 @@ class Scheduler(MetaScheduler):
             uuid=task.uuid)
 
     def wrapped_report_task(self, task, task_report):
-        """Commodity function to call `report_task` and trap exceptions
+        """Commodity function to call `report_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -891,8 +876,10 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def report_task(self, task, task_report):
-        """Takes care of storing the result according to preferences
+        """Take care of storing the result according to preferences.
-        and deals with logic for repeating tasks"""
+
+        Deals with logic for repeating tasks.
+        """
         db = self.db
         now = self.now()
         st = db.scheduler_task
@@ -914,12 +901,12 @@ class Scheduler(MetaScheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
+        is_expired = (task.stop_time and
-                      and task.next_run_time > task.stop_time
+                      task.next_run_time > task.stop_time and
-                      and True or False)
+                      True or False)
-        status = (task.run_again and is_expired and EXPIRED
+        status = (task.run_again and is_expired and EXPIRED or
-                  or task.run_again and not is_expired
+                  task.run_again and not is_expired and
-                  and QUEUED or COMPLETED)
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             d = dict(status=status,
                      next_run_time=task.next_run_time,
@@ -945,27 +932,26 @@ class Scheduler(MetaScheduler):
         logger.info('task completed (%s)', task_report.status)
 
     def update_dependencies(self, db, task_id):
+        """Unblock execution paths for Jobs."""
         db(db.scheduler_task_deps.task_child == task_id).update(can_visit=True)
 
     def adj_hibernation(self):
-        """Used to increase the "sleep" interval for DISABLED workers"""
+        """Used to increase the "sleep" interval for DISABLED workers."""
         if self.w_stats.status == DISABLED:
             wk_st = self.w_stats.sleep
             hibernation = wk_st + HEARTBEAT if wk_st < MAXHIBERNATION else MAXHIBERNATION
             self.w_stats.sleep = hibernation
 
     def send_heartbeat(self, counter):
-        """This function is vital for proper coordination among available
+        """Coordination among available workers.
-        workers.
-        It:
 
+        It:
         - sends the heartbeat
         - elects a ticker among available workers (the only process that
             effectively dispatch tasks to workers)
         - deals with worker's statuses
         - does "housecleaning" for dead workers
         - triggers tasks assignment to workers
-
         """
         if not self.db_thread:
             logger.debug('thread building own DAL object')
@@ -1053,7 +1039,8 @@ class Scheduler(MetaScheduler):
         self.sleep()
 
     def being_a_ticker(self):
-        """Elects a TICKER process that assigns tasks to available workers.
+        """Elect a TICKER process that assigns tasks to available workers.
+
         Does its best to elect a worker that is not busy processing other tasks
         to allow a proper distribution of tasks among all active workers ASAP
         """
@@ -1087,7 +1074,7 @@ class Scheduler(MetaScheduler):
             return False
 
     def assign_tasks(self, db):
-        """Assigns task to workers, that can then pop them from the queue
+        """Assign task to workers, that can then pop them from the queue.
 
         Deals with group_name(s) logic, in order to assign linearly tasks
         to available workers for those groups
@@ -1137,9 +1124,6 @@ class Scheduler(MetaScheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -1151,8 +1135,8 @@ class Scheduler(MetaScheduler):
         # intelligence (like esteeming how many tasks will a worker complete
         # before the ticker reassign them around, but the gain is quite small
         # 50 is a sweet spot also for fast tasks, with sane heartbeat values
-        # NB: ticker reassign tasks every 5 cycles, so if a worker completes its
+        # NB: ticker reassign tasks every 5 cycles, so if a worker completes
-        # 50 tasks in less than heartbeat*5 seconds,
+        # its 50 tasks in less than heartbeat*5 seconds,
         # it won't pick new tasks until heartbeat*5 seconds pass.
 
         # If a worker is currently elaborating a long task, its tasks needs to
@@ -1165,7 +1149,7 @@ class Scheduler(MetaScheduler):
         x = 0
         for group in wkgroups.keys():
             tasks = all_available(st.group_name == group).select(
-                limitby=(0, limit), orderby = st.next_run_time)
+                limitby=(0, limit), orderby=st.next_run_time)
             # let's break up the queue evenly among workers
             for task in tasks:
                 x += 1
@@ -1183,8 +1167,6 @@ class Scheduler(MetaScheduler):
                         status=ASSIGNED,
                         assigned_worker_name=assigned_wn
                     )
-                    if not task.task_name:
-                        d['task_name'] = task.function_name
                     db(
                         (st.id == task.id) &
                         (st.status.belongs((QUEUED, ASSIGNED)))
@@ -1204,14 +1186,13 @@ class Scheduler(MetaScheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def sleep(self):
-        """Calculates the number of seconds to sleep according to worker's
+        """Calculate the number of seconds to sleep."""
-        status and `heartbeat` parameter"""
         time.sleep(self.w_stats.sleep)
         # should only sleep until next available task
 
     def set_worker_status(self, group_names=None, action=ACTIVE,
                           exclude=None, limit=None, worker_name=None):
-        """Internal function to set worker's status"""
+        """Internal function to set worker's status."""
         ws = self.db.scheduler_worker
         if not group_names:
             group_names = self.group_names
@@ -1235,10 +1216,12 @@ class Scheduler(MetaScheduler):
                 self.db(ws.id.belongs(workers)).update(status=action)
 
     def disable(self, group_names=None, limit=None, worker_name=None):
-        """Sets DISABLED on the workers processing `group_names` tasks.
+        """Set DISABLED on the workers processing `group_names` tasks.
+
         A DISABLED worker will be kept alive but it won't be able to process
         any waiting tasks, essentially putting it to sleep.
-        By default, all group_names of Scheduler's instantation are selected"""
+        By default, all group_names of Scheduler's instantation are selected
+        """
         self.set_worker_status(
             group_names=group_names,
             action=DISABLED,
@@ -1283,8 +1266,9 @@ class Scheduler(MetaScheduler):
             pvars: "raw" kwargs to be passed to the function. Automatically
                 jsonified
             kwargs: all the parameters available (basically, every
-                `scheduler_task` column). If args and vars are here, they should
+                `scheduler_task` column). If args and vars are here, they
-                be jsonified already, and they will override pargs and pvars
+                should be jsonified already, and they will override pargs
+                and pvars
 
         Returns:
             a dict just as a normal validate_and_insert(), plus a uuid key

gluon/sqlhtml.py

@@ -1897,33 +1897,39 @@ class SQLFORM(FORM):
                 else:
                     field_type = field.type
 
-                operators = SELECT(*[OPTION(T(option), _value=option) for option in options], _class='form-control')
+                operators = SELECT(*[OPTION(T(option), _value=option) for option in options], 
+                                    _class='form-control')
                 _id = "%s_%s" % (value_id, name)
                 if field_type in ['boolean', 'double', 'time', 'integer']:
                     widget_ = SQLFORM.widgets[field_type]
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control')
+                    value_input = widget_.widget(field, field.default, _id=_id, 
+                                                 _class=widget_._class + ' form-control')
                 elif field_type == 'date':
                     iso_format = {'_data-w2p_date_format': '%Y-%m-%d'}
                     widget_ = SQLFORM.widgets.date
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control', **iso_format)
+                    value_input = widget_.widget(field, field.default, _id=_id, 
+                                                 _class=widget_._class + ' form-control', 
+                                                 **iso_format)
                 elif field_type == 'datetime':
                     iso_format = {'_data-w2p_datetime_format': '%Y-%m-%d %H:%M:%S'}
                     widget_ = SQLFORM.widgets.datetime
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control', **iso_format)
+                    value_input = widget_.widget(field, field.default, _id=_id, 
-                elif (field_type.startswith('reference ') or
+                                                 _class=widget_._class + ' form-control', 
-                      field_type.startswith('list:reference ')) and \
+                                                 **iso_format)
-                      hasattr(field.requires, 'options') or \
+                elif hasattr(field.requires, 'options'):
-                      hasattr(field.requires, 'options'):
                     value_input = SELECT(
                         *[OPTION(v, _value=k)
                           for k, v in field.requires.options()],
                          _class='form-control',
                          **dict(_id=_id))
-                elif field_type.startswith('reference ') or \
+                elif (field_type.startswith('integer') or
-                     field_type.startswith('list:integer') or \
+                      field_type.startswith('reference ') or
-                     field_type.startswith('list:reference '):
+                      field_type.startswith('list:integer') or
+                      field_type.startswith('list:reference ')):
                     widget_ = SQLFORM.widgets.integer
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control')
+                    value_input = widget_.widget(
+                        field, field.default, _id=_id, 
+                        _class=widget_._class + ' form-control')
                 else:
                     value_input = INPUT(
                         _type='text', _id=_id,
@@ -3037,7 +3043,16 @@ class SQLFORM(FORM):
             query = query & constraints[table._tablename]
         if isinstance(links, dict):
             links = links.get(table._tablename, [])
-        for key in 'columns,orderby,searchable,sortable,paginate,deletable,editable,details,selectable,create,fields'.split(','):
+        for key in ('fields', 'field_id', 'left', 'headers', 'orderby', 'groupby', 'searchable', 
+                    'sortable', 'paginate', 'deletable', 'editable', 'details', 'selectable',
+                    'create', 'csv', 'links', 'links_in_grid', 'upload', 'maxtextlengths',
+                    'maxtextlength', 'onvalidation', 'onfailure', 'oncreate', 'onupdate',
+                    'ondelete', 'sorter_icons', 'ui', 'showbuttontext', '_class', 'formname',
+                    'search_widget', 'advanced_search', 'ignore_rw', 'formstyle', 'exportclasses',
+                    'formargs', 'createargs', 'editargs', 'viewargs', 'selectable_submit_button',
+                    'buttons_placement', 'links_placement', 'noconfirm', 'cache_count', 'client_side_delete',
+                    'ignore_common_filters', 'auto_pagination', 'use_cursor'
+                   ):
             if isinstance(kwargs.get(key, None), dict):
                 if table._tablename in kwargs[key]:
                     kwargs[key] = kwargs[key][table._tablename]

gluon/tests/__init__.py

@@ -21,6 +21,8 @@ from test_contribs import *
 from test_web import *
 from test_dal import *
 from test_tools import *
+from test_appadmin import *
+from test_scheduler import *
 
 if sys.version[:3] == '2.7':
     from test_old_doctests import *

gluon/tests/test_appadmin.py

@@ -0,0 +1,175 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.sqlhtml
+"""
+import os
+import sys
+if sys.version < "2.7":
+    import unittest2 as unittest
+else:
+    import unittest
+
+from fix_path import fix_sys_path
+
+fix_sys_path(__file__)
+
+
+from compileapp import run_controller_in, run_view_in
+from languages import translator
+from gluon.storage import Storage, List
+import gluon.fileutils
+from gluon.dal import DAL, Field, Table
+from gluon.http import HTTP
+
+DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
+
+try:
+    import json
+except ImportError:
+    from gluon.contrib import simplejson as json
+
+
+def fake_check_credentials(foo):
+    return True
+
+
+class TestAppAdmin(unittest.TestCase):
+
+    def setUp(self):
+        from gluon.globals import Request, Response, Session, current
+        from gluon.html import A, DIV, FORM, MENU, TABLE, TR, INPUT, URL, XML
+        from gluon.validators import IS_NOT_EMPTY
+        from compileapp import LOAD
+        from gluon.http import HTTP, redirect
+        from gluon.tools import Auth
+        from gluon.sql import SQLDB
+        from gluon.sqlhtml import SQLTABLE, SQLFORM
+        self.original_check_credentials = gluon.fileutils.check_credentials
+        gluon.fileutils.check_credentials = fake_check_credentials
+        request = Request(env={})
+        request.application = 'welcome'
+        request.controller = 'appadmin'
+        request.function = self._testMethodName.split('_')[1]
+        request.folder = 'applications/welcome'
+        request.env.http_host = '127.0.0.1:8000'
+        request.env.remote_addr = '127.0.0.1'
+        response = Response()
+        session = Session()
+        T = translator('', 'en')
+        session.connect(request, response)
+        current.request = request
+        current.response = response
+        current.session = session
+        current.T = T
+        db = DAL(DEFAULT_URI, check_reserved=['all'])
+        auth = Auth(db)
+        auth.define_tables(username=True, signature=False)
+        db.define_table('t0', Field('tt'), auth.signature)
+        # Create a user
+        db.auth_user.insert(first_name='Bart',
+                            last_name='Simpson',
+                            username='user1',
+                            email='user1@test.com',
+                            password='password_123',
+                            registration_key=None,
+                            registration_id=None)
+        self.env = locals()
+
+    def tearDown(self):
+        gluon.fileutils.check_credentials = self.original_check_credentials
+
+    def run_function(self):
+        return run_controller_in(self.env['request'].controller, self.env['request'].function, self.env)
+
+    def run_view(self):
+        return run_view_in(self.env)
+
+    def test_index(self):
+        result = self.run_function()
+        self.assertTrue('db' in result['databases'])
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_select(self):
+        request = self.env['request']
+        request.args = List(['db'])
+        request.env.query_string = 'query=db.auth_user.id>0'
+        result = self.run_function()
+        self.assertTrue('table' in result and 'query' in result)
+        self.assertTrue(result['table'] == 'auth_user')
+        self.assertTrue(result['query'] == 'db.auth_user.id>0')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        result = self.run_function()
+        self.assertTrue('table' in result)
+        self.assertTrue('form' in result)
+        self.assertTrue(str(result['table']) is 'auth_user')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        data['first_name'] = 'Lisa'
+        data['last_name'] = 'Simpson'
+        data['username'] = 'lisasimpson'
+        data['password'] = 'password_123'
+        data['email'] = 'lisa@example.com'
+        request._vars = data
+        result = self.run_function()
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+        db = self.env['db']
+        lisa_record = db(db.auth_user.username == 'lisasimpson').select().first()
+        self.assertIsNotNone(lisa_record)
+        del data['_formkey']
+        del data['_formname']
+        del data['password']
+        for key in data:
+            self.assertEqual(data[key], lisa_record[key])
+
+    def test_update_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user', '1'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        for element in form.elements('input'):
+            data[element['_name']] = element['_value']
+        data['email'] = 'user1@example.com'
+        data['id'] = '1'
+        request._vars = data
+        self.assertRaises(HTTP, self.run_function)
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_cache.py

@@ -37,10 +37,11 @@ def tearDownModule():
         pass
 
 
-
 class TestCache(unittest.TestCase):
 
-    def testCacheInRam(self):
+    # TODO: test_CacheAbstract(self):
+
+    def test_CacheInRam(self):
 
         # defaults to mode='http'
         cache = CacheInRam()
@@ -53,22 +54,21 @@ class TestCache(unittest.TestCase):
         cache.clear()
         self.assertEqual(cache('a', lambda: 3, 100), 3)
         self.assertEqual(cache('a', lambda: 4, 0), 4)
-        #test singleton behaviour
+        # test singleton behaviour
         cache = CacheInRam()
         cache.clear()
         self.assertEqual(cache('a', lambda: 3, 100), 3)
         self.assertEqual(cache('a', lambda: 4, 0), 4)
-        #test key deletion
+        # test key deletion
         cache('a', None)
         self.assertEqual(cache('a', lambda: 5, 100), 5)
-        #test increment
+        # test increment
         self.assertEqual(cache.increment('a'), 6)
         self.assertEqual(cache('a', lambda: 1, 100), 6)
         cache.increment('b')
         self.assertEqual(cache('b', lambda: 'x', 100), 1)
 
-
+    def test_CacheOnDisk(self):
-    def testCacheOnDisk(self):
 
         # defaults to mode='http'
         s = Storage({'application': 'admin',
@@ -83,30 +83,36 @@ class TestCache(unittest.TestCase):
         cache.clear()
         self.assertEqual(cache('a', lambda: 3, 100), 3)
         self.assertEqual(cache('a', lambda: 4, 0), 4)
-        #test singleton behaviour
+        # test singleton behaviour
         cache = CacheOnDisk(s)
         cache.clear()
         self.assertEqual(cache('a', lambda: 3, 100), 3)
         self.assertEqual(cache('a', lambda: 4, 0), 4)
-        #test key deletion
+        # test key deletion
         cache('a', None)
         self.assertEqual(cache('a', lambda: 5, 100), 5)
-        #test increment
+        # test increment
         self.assertEqual(cache.increment('a'), 6)
         self.assertEqual(cache('a', lambda: 1, 100), 6)
         cache.increment('b')
         self.assertEqual(cache('b', lambda: 'x', 100), 1)
 
-    def testCacheWithPrefix(self):
+    # TODO: def test_CacheAction(self):
+
+    # TODO: def test_Cache(self):
+
+    # TODO: def test_lazy_cache(self):
+
+    def test_CacheWithPrefix(self):
         s = Storage({'application': 'admin',
                      'folder': 'applications/admin'})
         cache = Cache(s)
-        prefix = cache.with_prefix(cache.ram,'prefix')
+        prefix = cache.with_prefix(cache.ram, 'prefix')
         self.assertEqual(prefix('a', lambda: 1, 0), 1)
         self.assertEqual(prefix('a', lambda: 2, 100), 1)
         self.assertEqual(cache.ram('prefixa', lambda: 2, 100), 1)
 
-    def testRegex(self):
+    def test_Regex(self):
         cache = CacheInRam()
         self.assertEqual(cache('a1', lambda: 1, 0), 1)
         self.assertEqual(cache('a2', lambda: 2, 100), 2)
@@ -114,7 +120,7 @@ class TestCache(unittest.TestCase):
         self.assertEqual(cache('a1', lambda: 2, 0), 2)
         self.assertEqual(cache('a2', lambda: 3, 100), 3)
 
-    def testDALcache(self):
+    def test_DALcache(self):
         s = Storage({'application': 'admin',
                      'folder': 'applications/admin'})
         cache = Cache(s)

gluon/tests/test_dal.py

@@ -106,16 +106,18 @@ class TestDALAdapters(unittest.TestCase):
     def test_mysql(self):
         if os.environ.get('APPVEYOR'):
             return
-        os.environ["DB"] = "mysql://root:@localhost/pydal"
+        if os.environ.get('TRAVIS'):
-        result = self._run_tests()
+            os.environ["DB"] = "mysql://root:@localhost/pydal"
-        self.assertTrue(result)
+            result = self._run_tests()
+            self.assertTrue(result)
 
     def test_pg8000(self):
         if os.environ.get('APPVEYOR'):
             return
-        os.environ["DB"] = "postgres:pg8000://postgres:@localhost/pydal"
+        if os.environ.get('TRAVIS'):
-        result = self._run_tests()
+            os.environ["DB"] = "postgres:pg8000://postgres:@localhost/pydal"
-        self.assertTrue(result)
+            result = self._run_tests()
+            self.assertTrue(result)
 
 
 if __name__ == '__main__':

gluon/tests/test_html.py

@@ -14,11 +14,14 @@ from html import *
 from html import verifyURL
 from html import truncate_string
 from storage import Storage
+from html import XML_pickle, XML_unpickle
+from html import TAG_pickler, TAG_unpickler
 
 
 class TestBareHelpers(unittest.TestCase):
 
-    # TODO: def test_xmlescape(self):
+    # xmlescape() = covered by other tests
+
     # TODO: def test_call_as_list(self):
 
     def test_truncate_string(self):
@@ -26,6 +29,8 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(truncate_string('Lorem ipsum dolor sit amet, consectetur adipiscing elit, '
                                          'sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.',
                                          length=30), 'Lorem ipsum dolor sit amet,...')
+        self.assertEqual(truncate_string('Short text shorter than the length parameter.', length=100),
+                         'Short text shorter than the length parameter.')
         # French text
         self.assertEqual(truncate_string('Un texte en français avec des accents et des caractères bizarre.', length=30),
                          'Un texte en français avec d...')
@@ -93,6 +98,11 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
         rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'p': (1, 3), 'q': 2}, hmac_key='key', hash_vars='p')
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
+        # test url_encode
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=False)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li\xc3\xa9=2&ma\xc3\xaf=1&ma\xc3\xaf=3')
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=True)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li%C3%A9=2&ma%C3%AF=1&ma%C3%AF=3')
         # test CRLF detection
         self.assertRaises(SyntaxError, URL, *['a\n', 'c', 'f'])
         self.assertRaises(SyntaxError, URL, *['a\r', 'c', 'f'])
@@ -168,10 +178,13 @@ class TestBareHelpers(unittest.TestCase):
         # bug check for the sanitizer for closing no-close tags
         self.assertEqual(XML('<p>Test</p><br/><p>Test</p><br/>', sanitize=True),
                          XML('<p>Test</p><br /><p>Test</p><br />'))
+        # basic flatten test
+        self.assertEqual(XML('<p>Test</p>').flatten(), '<p>Test</p>')
+        self.assertEqual(XML('<p>Test</p>').flatten(render=lambda text, tag, attr: text), '<p>Test</p>')
 
-    # TODO: def test_XML_unpickle(self):
+    def test_XML_pickle_unpickle(self):
-
+        # weird test
-    # TODO: def test_XML_pickle(self):
+        self.assertEqual(XML_unpickle(XML_pickle('data to be pickle')[1][0]), 'data to be pickle')
 
     def test_DIV(self):
         # Empty DIV()
@@ -199,7 +212,31 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(a.xml(), '<div class="abc"><span>a</span><div>b</div></div>')
         self.assertEqual([el.xml() for el in s.siblings()], ['<div>b</div>'])
         self.assertEqual(s.sibling().xml(), '<div>b</div>')
+        # siblings with wrong args
         self.assertEqual(s.siblings('a'), [])
+        # siblings with good args
+        self.assertEqual(s.siblings('div')[0].xml(), '<div>b</div>')
+        # Check for siblings with wrong kargs and value
+        self.assertEqual(s.siblings(a='d'), [])
+        # Check for siblings with good kargs and value
+        # Can't figure this one out what is a right value here??
+        # Commented for now...
+        # self.assertEqual(s.siblings(div='<div>b</div>'), ???)
+        # No other sibling should return None
+        self.assertEqual(DIV(P('First element')).element('p').sibling(), None)
+        # --------------------------------------------------------------------------------------------------------------
+        # This use unicode to hit xmlescape() line :
+        #     """
+        #     elif isinstance(data, unicode):
+        #         data = data.encode('utf8', 'xmlcharrefreplace')
+        #     """
+        self.assertEqual(DIV(u'Texte en français avec des caractères accentués...').xml(),
+                         '<div>Texte en fran\xc3\xa7ais avec des caract\xc3\xa8res accentu\xc3\xa9s...</div>')
+        # --------------------------------------------------------------------------------------------------------------
+        self.assertEqual(DIV('Test with an ID', _id='id-of-the-element').xml(),
+                         '<div id="id-of-the-element">Test with an ID</div>')
+        self.assertEqual(DIV().element('p'), None)
+
         # Corner case for raise coverage of one line
         # I think such assert fail cause of python 2.6
         # Work under python 2.7
@@ -207,6 +244,9 @@ class TestBareHelpers(unittest.TestCase):
         #     DIV(BR('<>')).xml()
         # self.assertEqual(cm.exception[0], '<br/> tags cannot have components')
 
+        # test .get('attrib')
+        self.assertEqual(DIV('<p>Test</p>', _class="class_test").get('_class'), 'class_test')
+
     def test_CAT(self):
         # Empty CAT()
         self.assertEqual(CAT().xml(), '')
@@ -215,9 +255,10 @@ class TestBareHelpers(unittest.TestCase):
         # CAT(' ')
         self.assertEqual(CAT(' ').xml(), ' ')
 
-    # TODO: def test_TAG_unpickler(self):
+    def test_TAG_pickler_unpickler(self):
-
+        # weird test
-    # TODO: def test_TAG_pickler(self):
+        self.assertEqual(TAG_unpickler(TAG_pickler(TAG.div('data to be pickle'))[1][0]).xml(),
+                         '<div>data to be pickle</div>')
 
     def test_TAG(self):
         self.assertEqual(TAG.first(TAG.second('test'), _key=3).xml(),
@@ -225,6 +266,9 @@ class TestBareHelpers(unittest.TestCase):
         # ending in underscore "triggers" <input /> style
         self.assertEqual(TAG.first_(TAG.second('test'), _key=3).xml(),
                          '<first key="3" />')
+        # unicode test for TAG
+        self.assertEqual(TAG.div(u'Texte en français avec des caractères accentués...').xml(),
+                         '<div>Texte en fran\xc3\xa7ais avec des caract\xc3\xa8res accentu\xc3\xa9s...</div>')
 
     def test_HTML(self):
         self.assertEqual(HTML('<>', _a='1', _b='2').xml(),
@@ -259,6 +303,8 @@ class TestBareHelpers(unittest.TestCase):
                          '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd">\n<html a="1" b="2" lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">&lt;&gt;</html>')
         self.assertEqual(XHTML('<>', _a='1', _b='2', doctype='xmlns').xml(),
                          'xmlns\n<html a="1" b="2" lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">&lt;&gt;</html>')
+        self.assertEqual(XHTML('<>', _a='1', _b='2', _xmlns='xmlns').xml(),
+                         '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n<html a="1" b="2" lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">&lt;&gt;</html>')
 
     def test_HEAD(self):
         self.assertEqual(HEAD('<>', _a='1', _b='2').xml(),
@@ -290,6 +336,8 @@ class TestBareHelpers(unittest.TestCase):
     def test_STYLE(self):
         self.assertEqual(STYLE('<>', _a='1', _b='2').xml(),
                          '<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
+        # Try to hit : return DIV.xml(self)
+        self.assertEqual(STYLE().xml(), '<style></style>')
 
     def test_IMG(self):
         self.assertEqual(IMG(_a='1', _b='2').xml(),
@@ -351,18 +399,12 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(HR(_a='1', _b='2').xml(), '<hr a="1" b="2" />')
 
     def test_A(self):
-        self.assertEqual(
+        self.assertEqual(A('<>', _a='1', _b='2').xml(),
-            A('<>', _a='1', _b='2').xml(),
+                         '<a a="1" b="2">&lt;&gt;</a>')
-            '<a a="1" b="2">&lt;&gt;</a>'
+        self.assertEqual(A('a', cid='b').xml(),
-            )
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', callback='b', _id='c').xml(),
-            A('a', cid='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>'
-            )
         # Callback with no id trigger web2py_uuid() call
         from html import web2pyHTMLParser
         a = A('a', callback='b').xml()
@@ -370,38 +412,22 @@ class TestBareHelpers(unittest.TestCase):
             uuid_generated = tag.attributes['_id']
         self.assertEqual(a,
                          '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="{id}">a</a>'.format(id=uuid_generated))
-        self.assertEqual(
+        self.assertEqual(A('a', delete='tr').xml(),
-            A('a', delete='tr').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>'
+        self.assertEqual(A('a', _id='b', target='<self>').xml(),
-            )
+                         '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', component='b').xml(),
-            A('a', _id='b', target='<self>').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>'
+        self.assertEqual(A('a', _id='b', callback='c', noconfirm=True).xml(),
-            )
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', cid='b').xml(),
-            A('a', component='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>'
+        self.assertEqual(A('a', cid='b', _disable_with='processing...').xml(),
-            )
+                         '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
-            A('a', _id='b', callback='c', noconfirm=True).xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>'
+        self.assertEqual(A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
-            )
+                         '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
-        self.assertEqual(
-            A('a', cid='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b', _disable_with='processing...').xml(),
-            '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
-            '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
 
     def test_BUTTON(self):
         self.assertEqual(BUTTON('test', _type='button').xml(),
@@ -515,8 +541,9 @@ class TestBareHelpers(unittest.TestCase):
                          '<iframe a="1" b="2">&lt;&gt;</iframe>')
 
     def test_INPUT(self):
-        self.assertEqual(INPUT(_a='1', _b='2').xml(),
+        self.assertEqual(INPUT(_a='1', _b='2').xml(), '<input a="1" b="2" type="text" />')
-                         '<input a="1" b="2" type="text" />')
+        # list value
+        self.assertEqual(INPUT(_value=[1, 2, 3]).xml(), '<input type="text" value="[1, 2, 3]" />')
 
     def test_TEXTAREA(self):
         self.assertEqual(TEXTAREA('<>', _a='1', _b='2').xml(),
@@ -560,6 +587,12 @@ class TestBareHelpers(unittest.TestCase):
                                 OPTION('option 2', _value='2'),
                                 _multiple='multiple').xml(),
                          '<select multiple="multiple"><option selected="selected" value="1">option 1</option><option value="2">option 2</option></select>')
+        # More then one select with mutilple
+        self.assertEqual(SELECT(OPTION('option 1', _value='1', _selected='selected'),
+                                OPTION('option 2', _value='2', _selected='selected'),
+                                _multiple='multiple').xml(),
+                         '<select multiple="multiple"><option selected="selected" value="1">option 1</option><option selected="selected" value="2">option 2</option></select>'
+                         )
         # OPTGROUP
         self.assertEqual(SELECT(OPTGROUP(OPTION('option 1', _value='1'),
                                          OPTION('option 2', _value='2'),
@@ -574,6 +607,11 @@ class TestBareHelpers(unittest.TestCase):
         # String value
         self.assertEqual(SELECT('Option 1', 'Option 2').xml(),
                          '<select><option value="Option 1">Option 1</option><option value="Option 2">Option 2</option></select>')
+        # list as a value
+        self.assertEqual(SELECT(OPTION('option 1', _value=[1, 2, 3]),
+                                OPTION('option 2', _value=[4, 5, 6], _selected='selected'),
+                                _multiple='multiple').xml(),
+                         '<select multiple="multiple"><option value="[1, 2, 3]">option 1</option><option selected="selected" value="[4, 5, 6]">option 2</option></select>')
 
     def test_FIELDSET(self):
         self.assertEqual(FIELDSET('<>', _a='1', _b='2').xml(),
@@ -586,17 +624,40 @@ class TestBareHelpers(unittest.TestCase):
     def test_FORM(self):
         self.assertEqual(FORM('<>', _a='1', _b='2').xml(),
                          '<form a="1" action="#" b="2" enctype="multipart/form-data" method="post">&lt;&gt;</form>')
+        # These 2 crash AppVeyor and Travis with: "ImportError: No YAML serializer available"
+        # self.assertEqual(FORM('<>', _a='1', _b='2').as_yaml(),
+        #                  "accepted: null\nattributes: {_a: '1', _action: '#', _b: '2', _enctype: multipart/form-data, _method: post}\ncomponents: [<>]\nerrors: {}\nlatest: {}\nparent: null\nvars: {}\n")
+        # self.assertEqual(FORM('<>', _a='1', _b='2').as_xml(),
+        #                  '<?xml version="1.0" encoding="UTF-8"?><document><errors></errors><vars></vars><parent>None</parent><attributes><_enctype>multipart/form-data</_enctype><_action>#</_action><_b>2</_b><_a>1</_a><_method>post</_method></attributes><components><item>&amp;lt;&amp;gt;</item></components><accepted>None</accepted><latest></latest></document>')
 
     def test_BEAUTIFY(self):
         self.assertEqual(BEAUTIFY(['a', 'b', {'hello': 'world'}]).xml(),
                          '<div><table><tr><td><div>a</div></td></tr><tr><td><div>b</div></td></tr><tr><td><div><table><tr><td style="font-weight:bold;vertical-align:top;">hello</td><td style="vertical-align:top;">:</td><td><div>world</div></td></tr></table></div></td></tr></table></div>')
+        # unicode
+        self.assertEqual(BEAUTIFY([P(u'àéèûôç'), 'a', 'b', {'hello': 'world'}]).xml(),
+                         '<div><table><tr><td><div><p>\xc3\xa0\xc3\xa9\xc3\xa8\xc3\xbb\xc3\xb4\xc3\xa7</p></div></td></tr><tr><td><div>a</div></td></tr><tr><td><div>b</div></td></tr><tr><td><div><table><tr><td style="font-weight:bold;vertical-align:top;">hello</td><td style="vertical-align:top;">:</td><td><div>world</div></td></tr></table></div></td></tr></table></div>')
 
     def test_MENU(self):
         self.assertEqual(MENU([('Home', False, '/welcome/default/index', [])]).xml(),
                          '<ul class="web2py-menu web2py-menu-vertical"><li class="web2py-menu-first"><a href="/welcome/default/index">Home</a></li></ul>')
+        # Multiples entries menu
+        self.assertEqual(MENU([('Home', False, '/welcome/default/index', []),
+                               ('Item 1', False, '/welcome/default/func_one', []),
+                               ('Item 2', False, '/welcome/default/func_two', []),
+                               ('Item 3', False, '/welcome/default/func_three', []),
+                               ('Item 4', False, '/welcome/default/func_four', [])]).xml(),
+                         '<ul class="web2py-menu web2py-menu-vertical"><li class="web2py-menu-first"><a href="/welcome/default/index">Home</a></li><li><a href="/welcome/default/func_one">Item 1</a></li><li><a href="/welcome/default/func_two">Item 2</a></li><li><a href="/welcome/default/func_three">Item 3</a></li><li class="web2py-menu-last"><a href="/welcome/default/func_four">Item 4</a></li></ul>'
+                         )
         # mobile=True
         self.assertEqual(MENU([('Home', False, '/welcome/default/index', [])], mobile=True).xml(),
                          '<select class="web2py-menu web2py-menu-vertical" onchange="window.location=this.value"><option value="/welcome/default/index">Home</option></select>')
+        # Multiples entries menu for mobile
+        self.assertEqual(MENU([('Home', False, '/welcome/default/index', []),
+                               ('Item 1', False, '/welcome/default/func_one', []),
+                               ('Item 2', False, '/welcome/default/func_two', []),
+                               ('Item 3', False, '/welcome/default/func_three', []),
+                               ('Item 4', False, '/welcome/default/func_four', [])], mobile=True).xml(),
+                         '<select class="web2py-menu web2py-menu-vertical" onchange="window.location=this.value"><option value="/welcome/default/index">Home</option><option value="/welcome/default/func_one">Item 1</option><option value="/welcome/default/func_two">Item 2</option><option value="/welcome/default/func_three">Item 3</option><option value="/welcome/default/func_four">Item 4</option></select>')
 
     # TODO: def test_embed64(self):
 

gluon/tests/test_recfile.py

@@ -23,8 +23,8 @@ class TestRecfile(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree('tests')
 
-    def testgeneration(self):
+    def test_generation(self):
-        for k in range(20):
+        for k in range(10):
             teststring = 'test%s' % k
             filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
             with recfile.open(filename, "w") as g:
@@ -35,6 +35,40 @@ class TestRecfile(unittest.TestCase):
             recfile.remove(filename)
             is_there = recfile.exists(filename)
             self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = str(uuid.uuid4()) + '.test'
+            with recfile.open(filename, "w", path='tests') as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r", path='tests').read(), teststring)
+            is_there = recfile.exists(filename, path='tests')
+            self.assertTrue(is_there)
+            recfile.remove(filename, path='tests')
+            is_there = recfile.exists(filename, path='tests')
+            self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = os.path.join('tests', str(uuid.uuid4()), str(uuid.uuid4()) + '.test')
+            with recfile.open(filename, "w") as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r").read(), teststring)
+            is_there = recfile.exists(filename)
+            self.assertTrue(is_there)
+            recfile.remove(filename)
+            is_there = recfile.exists(filename)
+            self.assertFalse(is_there)
+
+    def test_existing(self):
+        filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
+        with open(filename, 'w') as g:
+            g.write('this file exists')
+        self.assertTrue(recfile.exists(filename))
+        self.assertTrue(hasattr(recfile.open(filename, "r"), 'read'))
+        recfile.remove(filename, path='tests')
+        self.assertFalse(recfile.exists(filename))
+        self.assertRaises(IOError, recfile.remove, filename)
+        self.assertRaises(IOError, recfile.open, filename, "r")
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_scheduler.py

@@ -0,0 +1,526 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.scheduler
+"""
+import os
+import unittest
+import glob
+import datetime
+import sys
+
+from fix_path import fix_sys_path
+
+
+fix_sys_path(__file__)
+
+from gluon.storage import Storage
+from gluon.languages import translator
+from gluon.scheduler import JobGraph, Scheduler
+from gluon.dal import DAL
+
+
+class BaseTestScheduler(unittest.TestCase):
+    def setUp(self):
+        self.db = None
+        self.cleanfolder()
+        from gluon import current
+        s = Storage({'application': 'welcome',
+                     'folder': 'applications/welcome',
+                     'controller': 'default'})
+        current.request = s
+        T = translator('', 'en')
+        current.T = T
+        self.db = DAL('sqlite://dummy2.db', check_reserved=['all'])
+
+    def cleanfolder(self):
+        if self.db:
+            self.db.close()
+        try:
+            os.unlink('dummy2.db')
+        except:
+            pass
+        tfiles = glob.glob('*_scheduler*.table')
+        for a in tfiles:
+            os.unlink(a)
+
+    def tearDown(self):
+        self.cleanfolder()
+        try:
+            self.inner_teardown()
+        except:
+            pass
+
+
+class TestsForJobGraph(BaseTestScheduler):
+
+    def testJobGraph(self):
+        s = Scheduler(self.db)
+        myjob = JobGraph(self.db, 'job_1')
+        fname = 'foo'
+        # We have a few items to wear, and there's an "order" to respect...
+        # Items are: watch, jacket, shirt, tie, pants, undershorts, belt, shoes, socks
+        # Now, we can't put on the tie without wearing the shirt first, etc...
+        watch = s.queue_task(fname, task_name='watch')
+        jacket = s.queue_task(fname, task_name='jacket')
+        shirt = s.queue_task(fname, task_name='shirt')
+        tie = s.queue_task(fname, task_name='tie')
+        pants = s.queue_task(fname, task_name='pants')
+        undershorts = s.queue_task(fname, task_name='undershorts')
+        belt = s.queue_task(fname, task_name='belt')
+        shoes = s.queue_task(fname, task_name='shoes')
+        socks = s.queue_task(fname, task_name='socks')
+        # before the tie, comes the shirt
+        myjob.add_deps(tie.id, shirt.id)
+        # before the belt too comes the shirt
+        myjob.add_deps(belt.id, shirt.id)
+        # before the jacket, comes the tie
+        myjob.add_deps(jacket.id, tie.id)
+        # before the belt, come the pants
+        myjob.add_deps(belt.id, pants.id)
+        # before the shoes, comes the pants
+        myjob.add_deps(shoes.id, pants.id)
+        # before the pants, comes the undershorts
+        myjob.add_deps(pants.id, undershorts.id)
+        # before the shoes, comes the undershorts
+        myjob.add_deps(shoes.id, undershorts.id)
+        # before the jacket, comes the belt
+        myjob.add_deps(jacket.id, belt.id)
+        # before the shoes, comes the socks
+        myjob.add_deps(shoes.id, socks.id)
+
+        ## results in the following topological sort
+        # 9,3,6 --> 4,5 --> 8,7 --> 2
+        # socks, shirt, undershorts
+        # tie, pants
+        # shoes, belt
+        # jacket
+        known_toposort = [
+            set([socks.id, shirt.id, undershorts.id]),
+            set([tie.id, pants.id]),
+            set([shoes.id, belt.id]),
+            set([jacket.id])
+        ]
+        toposort = myjob.validate('job_1')
+        self.assertEqual(toposort, known_toposort)
+        # add a cyclic dependency, jacket to undershorts
+        myjob.add_deps(undershorts.id, jacket.id)
+        # no exceptions raised, but result None
+        self.assertEqual(myjob.validate('job_1'), None)
+
+    def testJobGraphFailing(self):
+        s = Scheduler(self.db)
+        myjob = JobGraph(self.db, 'job_1')
+        fname = 'foo'
+        # We have a few items to wear, and there's an "order" to respect...
+        # Items are: watch, jacket, shirt, tie, pants, undershorts, belt, shoes, socks
+        # Now, we can't put on the tie without wearing the shirt first, etc...
+        watch = s.queue_task(fname, task_name='watch')
+        jacket = s.queue_task(fname, task_name='jacket')
+        shirt = s.queue_task(fname, task_name='shirt')
+        tie = s.queue_task(fname, task_name='tie')
+        pants = s.queue_task(fname, task_name='pants')
+        undershorts = s.queue_task(fname, task_name='undershorts')
+        belt = s.queue_task(fname, task_name='belt')
+        shoes = s.queue_task(fname, task_name='shoes')
+        socks = s.queue_task(fname, task_name='socks')
+        # before the tie, comes the shirt
+        myjob.add_deps(tie.id, shirt.id)
+        # before the belt too comes the shirt
+        myjob.add_deps(belt.id, shirt.id)
+        # before the jacket, comes the tie
+        myjob.add_deps(jacket.id, tie.id)
+        # before the belt, come the pants
+        myjob.add_deps(belt.id, pants.id)
+        # before the shoes, comes the pants
+        myjob.add_deps(shoes.id, pants.id)
+        # before the pants, comes the undershorts
+        myjob.add_deps(pants.id, undershorts.id)
+        # before the shoes, comes the undershorts
+        myjob.add_deps(shoes.id, undershorts.id)
+        # before the jacket, comes the belt
+        myjob.add_deps(jacket.id, belt.id)
+        # before the shoes, comes the socks
+        myjob.add_deps(shoes.id, socks.id)
+        # add a cyclic dependency, jacket to undershorts
+        myjob.add_deps(undershorts.id, jacket.id)
+        # no exceptions raised, but result None
+        self.assertEqual(myjob.validate('job_1'), None)
+        # and no deps added
+        deps_inserted = self.db(self.db.scheduler_task_deps.id>0).count()
+        self.assertEqual(deps_inserted, 0)
+
+    def testJobGraphDifferentJobs(self):
+        s = Scheduler(self.db)
+        myjob1 = JobGraph(self.db, 'job_1')
+        myjob2 = JobGraph(self.db, 'job_2')
+        fname = 'foo'
+        # We have a few items to wear, and there's an "order" to respect...
+        # Items are: watch, jacket, shirt, tie, pants, undershorts, belt, shoes, socks
+        # Now, we can't put on the tie without wearing the shirt first, etc...
+        watch = s.queue_task(fname, task_name='watch')
+        jacket = s.queue_task(fname, task_name='jacket')
+        shirt = s.queue_task(fname, task_name='shirt')
+        tie = s.queue_task(fname, task_name='tie')
+        pants = s.queue_task(fname, task_name='pants')
+        undershorts = s.queue_task(fname, task_name='undershorts')
+        belt = s.queue_task(fname, task_name='belt')
+        shoes = s.queue_task(fname, task_name='shoes')
+        socks = s.queue_task(fname, task_name='socks')
+        # before the tie, comes the shirt
+        myjob1.add_deps(tie.id, shirt.id)
+        # before the belt too comes the shirt
+        myjob1.add_deps(belt.id, shirt.id)
+        # before the jacket, comes the tie
+        myjob1.add_deps(jacket.id, tie.id)
+        # before the belt, come the pants
+        myjob1.add_deps(belt.id, pants.id)
+        # before the shoes, comes the pants
+        myjob2.add_deps(shoes.id, pants.id)
+        # before the pants, comes the undershorts
+        myjob2.add_deps(pants.id, undershorts.id)
+        # before the shoes, comes the undershorts
+        myjob2.add_deps(shoes.id, undershorts.id)
+        # before the jacket, comes the belt
+        myjob2.add_deps(jacket.id, belt.id)
+        # before the shoes, comes the socks
+        myjob2.add_deps(shoes.id, socks.id)
+        # every job by itself can be completed
+        self.assertNotEqual(myjob1.validate('job_1'), None)
+        self.assertNotEqual(myjob1.validate('job_2'), None)
+        # and, implicitly, every queued task can be too
+        self.assertNotEqual(myjob1.validate(), None)
+        # add a cyclic dependency, jacket to undershorts
+        myjob2.add_deps(undershorts.id, jacket.id)
+        # every job can still be completed by itself
+        self.assertNotEqual(myjob1.validate('job_1'), None)
+        self.assertNotEqual(myjob1.validate('job_2'), None)
+        # but trying to see if every task will ever be completed fails
+        self.assertEqual(myjob2.validate(), None)
+
+
+class TestsForSchedulerAPIs(BaseTestScheduler):
+
+    def testQueue_Task(self):
+
+        def isnotqueued(result):
+            self.assertEqual(result.id, None)
+            self.assertEqual(result.uuid, None)
+            self.assertEqual(len(result.errors.keys()) > 0, True)
+
+        def isqueued(result):
+            self.assertNotEqual(result.id, None)
+            self.assertNotEqual(result.uuid, None)
+            self.assertEqual(len(result.errors.keys()), 0)
+
+        s = Scheduler(self.db)
+        fname = 'foo'
+        watch = s.queue_task(fname, task_name='watch')
+        # queuing a task returns id, errors, uuid
+        self.assertEqual(set(watch.keys()), set(['id', 'uuid', 'errors']))
+        # queueing nothing isn't allowed
+        self.assertRaises(TypeError, s.queue_task, *[])
+        # passing pargs and pvars wrongly
+        # # pargs as dict
+        isnotqueued(s.queue_task(fname, dict(a=1), dict(b=1)))
+        # # pvars as list
+        isnotqueued(s.queue_task(fname, ['foo', 'bar'], ['foo', 'bar']))
+        # two tasks with the same uuid won't be there
+        isqueued(s.queue_task(fname, uuid='a'))
+        isnotqueued(s.queue_task(fname, uuid='a'))
+        # # #FIXME add here every parameter
+
+    def testTask_Status(self):
+        s = Scheduler(self.db)
+        fname = 'foo'
+        watch = s.queue_task(fname, task_name='watch')
+        # fetch status by id
+        by_id = s.task_status(watch.id)
+        # fetch status by uuid
+        by_uuid = s.task_status(watch.uuid)
+        # fetch status by query
+        by_query = s.task_status(self.db.scheduler_task.function_name == 'foo')
+        self.assertEqual(by_id, by_uuid)
+        self.assertEqual(by_id, by_query)
+        # fetch status by anything else throws
+        self.assertRaises(SyntaxError, s.task_status, *[[1, 2]])
+        # adding output returns the joined set, plus "result"
+        rtn = s.task_status(watch.id, output=True)
+        self.assertEqual(set(rtn.keys()), set(['scheduler_run', 'scheduler_task', 'result']))
+
+
+class testForSchedulerRunnerBase(BaseTestScheduler):
+
+    def inner_teardown(self):
+        from gluon import current
+        fdest = os.path.join(current.request.folder, 'models', 'scheduler.py')
+        os.unlink(fdest)
+        additional_files = [
+            os.path.join(current.request.folder, 'private', 'demo8.pholder')
+        ]
+        for f in additional_files:
+            try:
+                os.unlink(f)
+            except:
+                pass
+
+    def writefunction(self, content, initlines=None):
+        from gluon import current
+        fdest = os.path.join(current.request.folder, 'models', 'scheduler.py')
+        if initlines is None:
+            initlines = """
+import os
+import time
+from gluon.scheduler import Scheduler
+db_dal = os.path.abspath(os.path.join(request.folder, '..', '..', 'dummy2.db'))
+sched_dal = DAL('sqlite://%s' % db_dal, folder=os.path.dirname(db_dal))
+sched = Scheduler(sched_dal, max_empty_runs=15, migrate=False, heartbeat=1)
+            """
+        with open(fdest, 'w') as q:
+            q.write(initlines)
+            q.write(content)
+
+    def exec_sched(self):
+        import subprocess
+        call_args = [sys.executable, 'web2py.py', '--no-banner', '-D', '20','-K', 'welcome']
+        ret = subprocess.call(call_args, env=dict(os.environ))
+        return ret
+
+    def fetch_results(self, sched, task):
+        info = sched.task_status(task.id)
+        task_runs = self.db(self.db.scheduler_run.task_id == task.id).select()
+        return info, task_runs
+
+    def exec_asserts(self, stmts, tag):
+        for stmt in stmts:
+            self.assertEqual(stmt[1], True, msg="%s - %s" % (tag, stmt[0]))
+
+
+class TestsForSchedulerRunner(testForSchedulerRunnerBase):
+
+    def testRepeats_and_Expired_and_Prio(self):
+        s = Scheduler(self.db)
+        repeats = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), repeats=2, period=5)
+        a_while_ago = datetime.datetime.now() - datetime.timedelta(seconds=60)
+        expired = s.queue_task('demo4', stop_time=a_while_ago)
+        prio1 = s.queue_task('demo1', ['scheduled_first'])
+        prio2 = s.queue_task('demo1', ['scheduled_second'], next_run_time=a_while_ago)
+        self.db.commit()
+        self.writefunction(r"""
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # repeats check
+        task, task_run = self.fetch_results(s, repeats)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are COMPLETED ", (task_run[0].status == task_run[1].status == 'COMPLETED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'REPEATS')
+
+        # expired check
+        task, task_run = self.fetch_results(s, expired)
+        res = [
+            ("task status expired", task.status == 'EXPIRED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task didn't run at all", len(task_run) == 0)
+        ]
+        self.exec_asserts(res, 'EXPIRATION')
+
+        # prio check
+        task1 = s.task_status(prio1.id, output=True)
+        task2 = s.task_status(prio2.id, output=True)
+        res = [
+            ("tasks status completed", task1.scheduler_task.status == task2.scheduler_task.status == 'COMPLETED'),
+            ("priority2 was executed before priority1" , task1.scheduler_run.id > task2.scheduler_run.id)
+        ]
+        self.exec_asserts(res, 'PRIORITY')
+
+    def testNoReturn_and_Timeout_and_Progress(self):
+        s = Scheduler(self.db)
+        noret1 = s.queue_task('demo5')
+        noret2 = s.queue_task('demo3')
+        timeout1 = s.queue_task('demo4', timeout=5)
+        timeout2 = s.queue_task('demo4')
+        progress = s.queue_task('demo6', sync_output=2)
+        self.db.commit()
+        self.writefunction(r"""
+def demo3():
+    time.sleep(15)
+    print 1/0
+    return None
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
+
+def demo5():
+    time.sleep(15)
+    print "I'm printing something"
+    rtn = dict(a=1, b=2)
+
+def demo6():
+    time.sleep(5)
+    print '50%'
+    time.sleep(5)
+    print '!clear!100%'
+    return 1
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # noreturn check
+        task1, task_run1 = self.fetch_results(s, noret1)
+        task2, task_run2 = self.fetch_results(s, noret2)
+        res = [
+            ("tasks no_returns1 completed", task1.status == 'COMPLETED'),
+            ("tasks no_returns2 failed", task2.status == 'FAILED'),
+            ("no_returns1 doesn't have a scheduler_run record", len(task_run1) == 0),
+            ("no_returns2 has a scheduler_run record FAILED", (len(task_run2) == 1 and task_run2[0].status == 'FAILED')),
+        ]
+        self.exec_asserts(res, 'NO_RETURN')
+
+        # timeout check
+        task1 = s.task_status(timeout1.id, output=True)
+        task2 = s.task_status(timeout2.id, output=True)
+        res = [
+            ("tasks timeouts1 timeoutted", task1.scheduler_task.status == 'TIMEOUT'),
+            ("tasks timeouts2 completed", task2.scheduler_task.status == 'COMPLETED')
+        ]
+        self.exec_asserts(res, 'TIMEOUT')
+
+        # progress check
+        task1 = s.task_status(progress.id, output=True)
+        res = [
+            ("tasks percentages completed", task1.scheduler_task.status == 'COMPLETED'),
+            ("output contains only 100%", task1.scheduler_run.run_output.strip() == "100%")
+        ]
+        self.exec_asserts(res, 'PROGRESS')
+
+    def testDrift_and_env_and_immediate(self):
+        s = Scheduler(self.db)
+        immediate = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), immediate=True)
+        env = s.queue_task('demo7')
+        drift = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), period=93, prevent_drift=True)
+        self.db.commit()
+        self.writefunction(r"""
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+import random
+def demo7():
+    time.sleep(random.randint(1,5))
+    print W2P_TASK, request.now
+    return W2P_TASK.id, W2P_TASK.uuid, W2P_TASK.run_id
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # immediate check, can only check that nothing breaks
+        task1 = s.task_status(immediate.id)
+        res = [
+            ("tasks status completed", task1.status == 'COMPLETED'),
+        ]
+        self.exec_asserts(res, 'IMMEDIATE')
+
+        # drift check
+        task, task_run = self.fetch_results(s, drift)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("next_run_time is exactly start_time + period", (task.next_run_time == task.start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'DRIFT')
+
+        # env check
+        task1 = s.task_status(env.id, output=True)
+        res = [
+            ("task %s returned W2P_TASK correctly" % (task1.scheduler_task.id),  task1.result == [task1.scheduler_task.id, task1.scheduler_task.uuid, task1.scheduler_run.id]),
+        ]
+        self.exec_asserts(res, 'ENV')
+
+
+    def testRetryFailed(self):
+        s = Scheduler(self.db)
+        failed = s.queue_task('demo2', retry_failed=1, period=1)
+        failed_consecutive = s.queue_task('demo8', retry_failed=2, repeats=2, period=1)
+        self.db.commit()
+        self.writefunction(r"""
+def demo2():
+    1/0
+
+def demo8():
+    placeholder = os.path.join(request.folder, 'private', 'demo8.pholder')
+    with open(placeholder, 'a') as g:
+        g.write('\nplaceholder for demo8 created')
+    num_of_lines = 0
+    with open(placeholder) as f:
+        num_of_lines = len([a for a in f.read().split('\n') if a])
+    print 'number of lines', num_of_lines
+    if num_of_lines <= 2:
+       1/0
+    else:
+        os.unlink(placeholder)
+    return 1
+""")
+        ret = self.exec_sched()
+        # process finished just fine
+        self.assertEqual(ret, 0)
+        # failed - checks
+        task, task_run = self.fetch_results(s, failed)
+        res = [
+            ("task status failed", task.status == 'FAILED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task times_failed is 2", task.times_failed == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are FAILED", (task_run[0].status == task_run[1].status == 'FAILED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'FAILED')
+
+        # failed consecutive - checks
+        task, task_run = self.fetch_results(s, failed_consecutive)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task times_failed is 0", task.times_failed == 0),
+            ("task ran 6 times", len(task_run) == 6),
+            ("scheduler_run records for COMPLETED is 2", len([run.status for run in task_run if run.status == 'COMPLETED']) == 2),
+            ("scheduler_run records for FAILED is 4", len([run.status for run in task_run if run.status == 'FAILED']) == 4),
+        ]
+        self.exec_asserts(res, 'FAILED_CONSECUTIVE')
+
+    def testHugeResult(self):
+        s = Scheduler(self.db)
+        huge_result = s.queue_task('demo10', retry_failed=1, period=1)
+        self.db.commit()
+        self.writefunction(r"""
+def demo10():
+    res = 'a' * 99999
+    return dict(res=res)
+""")
+        ret = self.exec_sched()
+        # process finished just fine
+        self.assertEqual(ret, 0)
+        # huge_result - checks
+        task = s.task_status(huge_result.id, output=True)
+        res = [
+            ("task status completed", task.scheduler_task.status == 'COMPLETED'),
+            ("task times_run is 1", task.scheduler_task.times_run == 1),
+            ("result is the correct one", task.result == dict(res='a' * 99999))
+        ]
+        self.exec_asserts(res, 'HUGE_RESULT')
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_template.py

@@ -9,10 +9,11 @@ from fix_path import fix_sys_path
 
 fix_sys_path(__file__)
 
+import template
 from template import render
 
 
-class TestVirtualFields(unittest.TestCase):
+class TestTemplate(unittest.TestCase):
 
     def testRun(self):
         self.assertEqual(render(content='{{for i in range(n):}}{{=i}}{{pass}}',
@@ -61,6 +62,80 @@ class TestVirtualFields(unittest.TestCase):
         self.assertRaises(
             SyntaxError, render, content='{{pass\n=list((1,2,\n3))}}')
 
+    def testWithDummyFileSystem(self):
+        from os.path import join as pjoin
+        import contextlib
+        from StringIO import StringIO
+        from gluon.restricted import RestrictedError
+
+        @contextlib.contextmanager
+        def monkey_patch(module, fn_name, patch):
+            try:
+                unpatch = getattr(module, fn_name)
+            except AttributeError:
+                unpatch = None
+            setattr(module, fn_name, patch)
+            try:
+                yield
+            finally:
+                if unpatch is None:
+                    delattr(module, fn_name)
+                else:
+                    setattr(module, fn_name, unpatch)
+
+        def dummy_open(path, mode):
+            if path == pjoin('views', 'layout.html'):
+                return StringIO("{{block left_sidebar}}left{{end}}"
+                                "{{include}}"
+                                "{{block right_sidebar}}right{{end}}")
+            elif path == pjoin('views', 'layoutbrackets.html'):
+                return StringIO("[[block left_sidebar]]left[[end]]"
+                                "[[include]]"
+                                "[[block right_sidebar]]right[[end]]")
+            elif path == pjoin('views', 'default', 'index.html'):
+                return StringIO("{{extend 'layout.html'}}"
+                                "{{block left_sidebar}}{{super}} {{end}}"
+                                "to"
+                                "{{block right_sidebar}} {{super}}{{end}}")
+            elif path == pjoin('views', 'default', 'indexbrackets.html'):
+                return StringIO("[[extend 'layoutbrackets.html']]"
+                                "[[block left_sidebar]][[super]] [[end]]"
+                                "to"
+                                "[[block right_sidebar]] [[super]][[end]]")
+            elif path == pjoin('views', 'default', 'missing.html'):
+                return StringIO("{{extend 'wut'}}"
+                                "{{block left_sidebar}}{{super}} {{end}}"
+                                "to"
+                                "{{block right_sidebar}} {{super}}{{end}}")
+            elif path == pjoin('views', 'default', 'noescape.html'):
+                return StringIO("""{{=NOESCAPE('<script></script>')}}""")
+            raise IOError
+
+        with monkey_patch(template, 'open', dummy_open):
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'index.html'),
+                       path='views'),
+                'left to right')
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'indexbrackets.html'),
+                       path='views', delimiters=('[[', ']]')),
+                'left to right')
+            self.assertRaises(
+                RestrictedError,
+                render,
+                filename=pjoin('views', 'default', 'missing.html'),
+                path='views')
+            response = template.DummyResponse()
+            response.delimiters = ('[[', ']]')
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'indexbrackets.html'),
+                       path='views', context={'response': response}),
+                'left to right')
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'noescape.html'),
+                       context={'NOESCAPE': template.NOESCAPE}),
+                '<script></script>')
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_validators.py

@@ -160,6 +160,14 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ({u'a': 100}, None))
         rtn = IS_JSON()('spam1234')
         self.assertEqual(rtn, ('spam1234', 'Invalid json'))
+        rtn = IS_JSON(native_json=True)('{"a": 100}')
+        self.assertEqual(rtn, ('{"a": 100}', None))
+        rtn = IS_JSON().formatter(None)
+        self.assertEqual(rtn, None)
+        rtn = IS_JSON().formatter({'a': 100})
+        self.assertEqual(rtn, '{"a": 100}')
+        rtn = IS_JSON(native_json=True).formatter({'a': 100})
+        self.assertEqual(rtn, {'a': 100})
 
     def test_IS_IN_SET(self):
         rtn = IS_IN_SET(['max', 'john'])('max')
@@ -170,17 +178,29 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (('max', 'john'), None))
         rtn = IS_IN_SET(['max', 'john'], multiple=True)(('bill', 'john'))
         self.assertEqual(rtn, (('bill', 'john'), 'Value not allowed'))
-        rtn = IS_IN_SET(('id1','id2'), ['first label','second label'])('id1') # Traditional way
+        rtn = IS_IN_SET(('id1', 'id2'), ['first label', 'second label'])('id1')  # Traditional way
         self.assertEqual(rtn, ('id1', None))
-        rtn = IS_IN_SET({'id1':'first label', 'id2':'second label'})('id1')
+        rtn = IS_IN_SET({'id1': 'first label', 'id2': 'second label'})('id1')
         self.assertEqual(rtn, ('id1', None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)(None)
+        self.assertEqual(rtn, ([], None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)('')
+        self.assertEqual(rtn, ([], None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)('id1')
+        self.assertEqual(rtn, (['id1'], None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=(1, 2))(None)
+        self.assertEqual(rtn, ([], 'oops'))
         import itertools
-        rtn = IS_IN_SET(itertools.chain(['1','3','5'],['2','4','6']))('1')
+        rtn = IS_IN_SET(itertools.chain(['1', '3', '5'], ['2', '4', '6']))('1')
         self.assertEqual(rtn, ('1', None))
-        rtn = IS_IN_SET([('id1','first label'), ('id2','second label')])('id1') # Redundant way
+        rtn = IS_IN_SET([('id1', 'first label'), ('id2', 'second label')])('id1')  # Redundant way
         self.assertEqual(rtn, ('id1', None))
-        rtn = IS_IN_SET([('id1','first label'), ('id2','second label')]).options(zero=False)
+        rtn = IS_IN_SET([('id1', 'first label'), ('id2', 'second label')]).options(zero=False)
         self.assertEqual(rtn, [('id1', 'first label'), ('id2', 'second label')])
+        rtn = IS_IN_SET(['id1', 'id2']).options(zero=False)
+        self.assertEqual(rtn, [('id1', 'id1'), ('id2', 'id2')])
+        rtn = IS_IN_SET(['id2', 'id1'], sort=True).options(zero=False)
+        self.assertEqual(rtn, [('id1', 'id1'), ('id2', 'id2')])
 
     def test_IS_IN_DB(self):
         from gluon.dal import DAL, Field
@@ -190,29 +210,100 @@ class TestValidators(unittest.TestCase):
         costanza_id = db.person.insert(name='costanza')
         rtn = IS_IN_DB(db, 'person.id', '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
-        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')(george_id+costanza_id)
+        rtn = IS_IN_DB(db, 'person.name', '%(name)s')('george')
-        self.assertEqual(rtn, (george_id+costanza_id, 'oops'))
+        self.assertEqual(rtn, ('george', None))
+        rtn = IS_IN_DB(db, db.person, '%(name)s')(george_id)
+        self.assertEqual(rtn, (george_id, None))
+        rtn = IS_IN_DB(db(db.person.id > 0), db.person, '%(name)s')(george_id)
+        self.assertEqual(rtn, (george_id, None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')(george_id + costanza_id)
+        self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
         rtn = IS_IN_DB(db, db.person.id, '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
-        rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops')(george_id+costanza_id)
+        rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops')(george_id + costanza_id)
-        self.assertEqual(rtn, (george_id+costanza_id, 'oops'))
+        self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
-        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True)([george_id,costanza_id])
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True)([george_id, costanza_id])
-        self.assertEqual(rtn, ([george_id,costanza_id], None))
+        self.assertEqual(rtn, ([george_id, costanza_id], None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, error_message='oops')("I'm not even an id")
+        self.assertEqual(rtn, (["I'm not even an id"], 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, delimiter=',')('%d,%d' % (george_id, costanza_id))
-        self.assertEqual(rtn, ( ('%d,%d' % (george_id, costanza_id)).split(','), None))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 3), delimiter=',')('%d,%d' % (george_id, costanza_id))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 2), delimiter=',', error_message='oops')('%d,%d' % (george_id, costanza_id))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), 'oops'))
         rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops').options(zero=False)
         self.assertEqual(sorted(rtn), [('%d' % george_id, 'george'), ('%d' % costanza_id, 'costanza')])
+        rtn = IS_IN_DB(db, db.person.id, db.person.name, error_message='oops', sort=True).options(zero=True)
+        self.assertEqual(rtn, [('', ''), ('%d' % costanza_id, 'costanza'), ('%d' % george_id, 'george')])
+        # Test using the set it made for options
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops')
+        vldtr.options()
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', multiple=True)
+        vldtr.options()
+        rtn = vldtr(['george', 'costanza'])
+        self.assertEqual(rtn, (['george', 'costanza'], None))
+        # Test it works with self reference
+        db.define_table('category',
+                        Field('parent_id', 'reference category', requires=IS_EMPTY_OR(IS_IN_DB(db, 'category.id', '%(name)s'))),
+                        Field('name')
+                        )
+        ret = db.category.validate_and_insert(name='seinfeld')
+        self.assertFalse(list(ret.errors))
+        ret = db.category.validate_and_insert(name='characters', parent_id=ret.id)
+        self.assertFalse(list(ret.errors))
+        rtn = IS_IN_DB(db, 'category.id', '%(name)s')(ret.id)
+        self.assertEqual(rtn, (ret.id, None))
+        # Test _and
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', _and=IS_LENGTH(maxsize=7, error_message='bad'))
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', 'bad'))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr.options() # test theset with _and
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        # Test auto_add
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', auto_add=True)('jerry')
+        self.assertEqual(rtn, (3, None))
         db.person.drop()
+        db.category.drop()
 
     def test_IS_NOT_IN_DB(self):
         from gluon.dal import DAL, Field
         db = DAL('sqlite:memory')
-        db.define_table('person', Field('name'))
+        db.define_table('person', Field('name'), Field('nickname'))
         db.person.insert(name='george')
+        db.person.insert(name='costanza', nickname='T Bone')
         rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops')('george')
         self.assertEqual(rtn, ('george', 'oops'))
+        rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops', allowed_override=['george'])('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops')('  ')
+        self.assertEqual(rtn, ('  ', 'oops'))
         rtn = IS_NOT_IN_DB(db, 'person.name')('jerry')
         self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db, 'person.name')(u'jerry')
+        self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db(db.person.id > 0), 'person.name')(u'jerry')
+        self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db, db.person, error_message='oops')(1)
+        self.assertEqual(rtn, ('1', 'oops'))
+        vldtr = IS_NOT_IN_DB(db, 'person.name', error_message='oops')
+        vldtr.set_self_id({'name': 'costanza', 'nickname': 'T Bone'})
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', 'oops'))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', None))
+
         db.person.drop()
 
     def test_IS_INT_IN_RANGE(self):
@@ -365,7 +456,7 @@ class TestValidators(unittest.TestCase):
     def test_IS_ALPHANUMERIC(self):
         rtn = IS_ALPHANUMERIC()('1')
         self.assertEqual(rtn, ('1', None))
-        rtn =  IS_ALPHANUMERIC()('')
+        rtn = IS_ALPHANUMERIC()('')
         self.assertEqual(rtn, ('', None))
         rtn = IS_ALPHANUMERIC()('A_a')
         self.assertEqual(rtn, ('A_a', None))
@@ -373,62 +464,62 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('!', 'Enter only letters, numbers, and underscore'))
 
     def test_IS_EMAIL(self):
-        rtn =  IS_EMAIL()('a@b.com')
+        rtn = IS_EMAIL()('a@b.com')
         self.assertEqual(rtn, ('a@b.com', None))
-        rtn =  IS_EMAIL()('abc@def.com')
+        rtn = IS_EMAIL()('abc@def.com')
         self.assertEqual(rtn, ('abc@def.com', None))
-        rtn =  IS_EMAIL()('abc@3def.com')
+        rtn = IS_EMAIL()('abc@3def.com')
         self.assertEqual(rtn, ('abc@3def.com', None))
-        rtn =  IS_EMAIL()('abc@def.us')
+        rtn = IS_EMAIL()('abc@def.us')
         self.assertEqual(rtn, ('abc@def.us', None))
-        rtn =  IS_EMAIL()('abc@d_-f.us')
+        rtn = IS_EMAIL()('abc@d_-f.us')
         self.assertEqual(rtn, ('abc@d_-f.us', None))
-        rtn =  IS_EMAIL()('@def.com')           # missing name
+        rtn = IS_EMAIL()('@def.com')           # missing name
         self.assertEqual(rtn, ('@def.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('"abc@def".com')      # quoted name
+        rtn = IS_EMAIL()('"abc@def".com')      # quoted name
         self.assertEqual(rtn, ('"abc@def".com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc+def.com')        # no @
+        rtn = IS_EMAIL()('abc+def.com')        # no @
         self.assertEqual(rtn, ('abc+def.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def.x')          # one-char TLD
+        rtn = IS_EMAIL()('abc@def.x')          # one-char TLD
         self.assertEqual(rtn, ('abc@def.x', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def.12')         # numeric TLD
+        rtn = IS_EMAIL()('abc@def.12')         # numeric TLD
         self.assertEqual(rtn, ('abc@def.12', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def..com')       # double-dot in domain
+        rtn = IS_EMAIL()('abc@def..com')       # double-dot in domain
         self.assertEqual(rtn, ('abc@def..com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@.def.com')       # dot starts domain
+        rtn = IS_EMAIL()('abc@.def.com')       # dot starts domain
         self.assertEqual(rtn, ('abc@.def.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@def.c_m')        # underscore in TLD
+        rtn = IS_EMAIL()('abc@def.c_m')        # underscore in TLD
         self.assertEqual(rtn, ('abc@def.c_m', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('NotAnEmail')         # missing @
+        rtn = IS_EMAIL()('NotAnEmail')         # missing @
         self.assertEqual(rtn, ('NotAnEmail', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('abc@NotAnEmail')     # missing TLD
+        rtn = IS_EMAIL()('abc@NotAnEmail')     # missing TLD
         self.assertEqual(rtn, ('abc@NotAnEmail', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('customer/department@example.com')
+        rtn = IS_EMAIL()('customer/department@example.com')
         self.assertEqual(rtn, ('customer/department@example.com', None))
-        rtn =  IS_EMAIL()('$A12345@example.com')
+        rtn = IS_EMAIL()('$A12345@example.com')
         self.assertEqual(rtn, ('$A12345@example.com', None))
-        rtn =  IS_EMAIL()('!def!xyz%abc@example.com')
+        rtn = IS_EMAIL()('!def!xyz%abc@example.com')
         self.assertEqual(rtn, ('!def!xyz%abc@example.com', None))
-        rtn =  IS_EMAIL()('_Yosemite.Sam@example.com')
+        rtn = IS_EMAIL()('_Yosemite.Sam@example.com')
         self.assertEqual(rtn, ('_Yosemite.Sam@example.com', None))
-        rtn =  IS_EMAIL()('~@example.com')
+        rtn = IS_EMAIL()('~@example.com')
         self.assertEqual(rtn, ('~@example.com', None))
-        rtn =  IS_EMAIL()('.wooly@example.com')       # dot starts name
+        rtn = IS_EMAIL()('.wooly@example.com')       # dot starts name
         self.assertEqual(rtn, ('.wooly@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('wo..oly@example.com')      # adjacent dots in name
+        rtn = IS_EMAIL()('wo..oly@example.com')      # adjacent dots in name
         self.assertEqual(rtn, ('wo..oly@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('pootietang.@example.com')  # dot ends name
+        rtn = IS_EMAIL()('pootietang.@example.com')  # dot ends name
         self.assertEqual(rtn, ('pootietang.@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('.@example.com')            # name is bare dot
+        rtn = IS_EMAIL()('.@example.com')            # name is bare dot
         self.assertEqual(rtn, ('.@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('Ima.Fool@example.com')
+        rtn = IS_EMAIL()('Ima.Fool@example.com')
         self.assertEqual(rtn, ('Ima.Fool@example.com', None))
-        rtn =  IS_EMAIL()('Ima Fool@example.com')     # space in name
+        rtn = IS_EMAIL()('Ima Fool@example.com')     # space in name
         self.assertEqual(rtn, ('Ima Fool@example.com', 'Enter a valid email address'))
-        rtn =  IS_EMAIL()('localguy@localhost')       # localhost as domain
+        rtn = IS_EMAIL()('localguy@localhost')       # localhost as domain
         self.assertEqual(rtn, ('localguy@localhost', None))
         # test for banned
-        rtn =  IS_EMAIL(banned='^.*\.com(|\..*)$')('localguy@localhost')       # localhost as domain
+        rtn = IS_EMAIL(banned='^.*\.com(|\..*)$')('localguy@localhost')       # localhost as domain
         self.assertEqual(rtn, ('localguy@localhost', None))
         rtn = IS_EMAIL(banned='^.*\.com(|\..*)$')('abc@example.com')
         self.assertEqual(rtn, ('abc@example.com', 'Enter a valid email address'))
@@ -437,6 +528,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('localguy@localhost', 'Enter a valid email address'))
         rtn = IS_EMAIL(forced='^.*\.edu(|\..*)$')('localguy@example.edu')
         self.assertEqual(rtn, ('localguy@example.edu', None))
+        # test for not a string at all
+        rtn = IS_EMAIL(error_message='oops')(42)
+        self.assertEqual(rtn, (42, 'oops'))
 
     def test_IS_LIST_OF_EMAILS(self):
         emails = ['localguy@localhost', '_Yosemite.Sam@example.com']
@@ -495,16 +589,16 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('', 'Enter time as hh:mm:ss (seconds, am, pm optional)'))
 
     def test_IS_DATE(self):
-        v = IS_DATE(format="%m/%d/%Y",error_message="oops")
+        v = IS_DATE(format="%m/%d/%Y", error_message="oops")
         rtn = v('03/03/2008')
         self.assertEqual(rtn, (datetime.date(2008, 3, 3), None))
         rtn = v('31/03/2008')
         self.assertEqual(rtn, ('31/03/2008', 'oops'))
-        rtn = IS_DATE(format="%m/%d/%Y",error_message="oops").formatter(datetime.date(1834, 12, 14))
+        rtn = IS_DATE(format="%m/%d/%Y", error_message="oops").formatter(datetime.date(1834, 12, 14))
         self.assertEqual(rtn, '12/14/1834')
 
     def test_IS_DATETIME(self):
-        v = IS_DATETIME(format="%m/%d/%Y %H:%M",error_message="oops")
+        v = IS_DATETIME(format="%m/%d/%Y %H:%M", error_message="oops")
         rtn = v('03/03/2008 12:40')
         self.assertEqual(rtn, (datetime.datetime(2008, 3, 3, 12, 40), None))
         rtn = v('31/03/2008 29:40')
@@ -512,16 +606,20 @@ class TestValidators(unittest.TestCase):
         # Test timezone is removed and value is properly converted
         #
         # https://github.com/web2py/web2py/issues/1094
+
         class DummyTimezone(datetime.tzinfo):
 
             ONE = datetime.timedelta(hours=1)
 
             def utcoffset(self, dt):
                 return DummyTimezone.ONE
+
             def tzname(self, dt):
                 return "UTC+1"
+
             def dst(self, dt):
                 return DummyTimezone.ONE
+
             def localize(self, dt, is_dst=False):
                 return dt.replace(tzinfo=self)
         v = IS_DATETIME(format="%Y-%m-%d %H:%M", error_message="oops", timezone=DummyTimezone())
@@ -529,65 +627,65 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (datetime.datetime(1982, 12, 14, 7, 0), None))
 
     def test_IS_DATE_IN_RANGE(self):
-        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008,1,1),
+        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008, 1, 1),
-                             maximum=datetime.date(2009,12,31),
+                             maximum=datetime.date(2009, 12, 31),
-                             format="%m/%d/%Y",error_message="oops")
+                             format="%m/%d/%Y", error_message="oops")
 
         rtn = v('03/03/2008')
         self.assertEqual(rtn, (datetime.date(2008, 3, 3), None))
         rtn = v('03/03/2010')
         self.assertEqual(rtn, ('03/03/2010', 'oops'))
-        rtn = v(datetime.date(2008,3,3))
+        rtn = v(datetime.date(2008, 3, 3))
         self.assertEqual(rtn, (datetime.date(2008, 3, 3), None))
-        rtn =  v(datetime.date(2010,3,3))
+        rtn = v(datetime.date(2010, 3, 3))
         self.assertEqual(rtn, (datetime.date(2010, 3, 3), 'oops'))
-        v = IS_DATE_IN_RANGE(maximum=datetime.date(2009,12,31),
+        v = IS_DATE_IN_RANGE(maximum=datetime.date(2009, 12, 31),
-            format="%m/%d/%Y")
+                             format="%m/%d/%Y")
         rtn = v('03/03/2010')
         self.assertEqual(rtn, ('03/03/2010', 'Enter date on or before 12/31/2009'))
-        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008,1,1),
+        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008, 1, 1),
-            format="%m/%d/%Y")
+                             format="%m/%d/%Y")
         rtn = v('03/03/2007')
         self.assertEqual(rtn, ('03/03/2007', 'Enter date on or after 01/01/2008'))
-        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008,1,1),
+        v = IS_DATE_IN_RANGE(minimum=datetime.date(2008, 1, 1),
-            maximum=datetime.date(2009,12,31),
+                             maximum=datetime.date(2009, 12, 31),
-            format="%m/%d/%Y")
+                             format="%m/%d/%Y")
         rtn = v('03/03/2007')
         self.assertEqual(rtn, ('03/03/2007', 'Enter date in range 01/01/2008 12/31/2009'))
 
     def test_IS_DATETIME_IN_RANGE(self):
         v = IS_DATETIME_IN_RANGE(
-            minimum=datetime.datetime(2008,1,1,12,20),
+            minimum=datetime.datetime(2008, 1, 1, 12, 20),
-            maximum=datetime.datetime(2009,12,31,12,20),
+            maximum=datetime.datetime(2009, 12, 31, 12, 20),
-            format="%m/%d/%Y %H:%M",error_message="oops")
+            format="%m/%d/%Y %H:%M", error_message="oops")
         rtn = v('03/03/2008 12:40')
         self.assertEqual(rtn, (datetime.datetime(2008, 3, 3, 12, 40), None))
         rtn = v('03/03/2010 10:34')
         self.assertEqual(rtn, ('03/03/2010 10:34', 'oops'))
-        rtn =  v(datetime.datetime(2008,3,3,0,0))
+        rtn = v(datetime.datetime(2008, 3, 3, 0, 0))
         self.assertEqual(rtn, (datetime.datetime(2008, 3, 3, 0, 0), None))
-        rtn =  v(datetime.datetime(2010,3,3,0,0))
+        rtn = v(datetime.datetime(2010, 3, 3, 0, 0))
         self.assertEqual(rtn, (datetime.datetime(2010, 3, 3, 0, 0), 'oops'))
-        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009,12,31,12,20),
+        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009, 12, 31, 12, 20),
-            format='%m/%d/%Y %H:%M:%S')
+                                 format='%m/%d/%Y %H:%M:%S')
         rtn = v('03/03/2010 12:20:00')
         self.assertEqual(rtn, ('03/03/2010 12:20:00', 'Enter date and time on or before 12/31/2009 12:20:00'))
-        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008,1,1,12,20),
+        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008, 1, 1, 12, 20),
-            format='%m/%d/%Y %H:%M:%S')
+                                 format='%m/%d/%Y %H:%M:%S')
         rtn = v('03/03/2007 12:20:00')
         self.assertEqual(rtn, ('03/03/2007 12:20:00', 'Enter date and time on or after 01/01/2008 12:20:00'))
-        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008,1,1,12,20),
+        v = IS_DATETIME_IN_RANGE(minimum=datetime.datetime(2008, 1, 1, 12, 20),
-            maximum=datetime.datetime(2009,12,31,12,20),
+                                 maximum=datetime.datetime(2009, 12, 31, 12, 20),
-            format='%m/%d/%Y %H:%M:%S')
+                                 format='%m/%d/%Y %H:%M:%S')
         rtn = v('03/03/2007 12:20:00')
         self.assertEqual(rtn, ('03/03/2007 12:20:00', 'Enter date and time in range 01/01/2008 12:20:00 12/31/2009 12:20:00'))
-        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009,12,31,12,20),
+        v = IS_DATETIME_IN_RANGE(maximum=datetime.datetime(2009, 12, 31, 12, 20),
-            format='%Y-%m-%d %H:%M:%S', error_message='oops')
+                                 format='%Y-%m-%d %H:%M:%S', error_message='oops')
         rtn = v('clearly not a date')
         self.assertEqual(rtn, ('clearly not a date', 'oops'))
 
     def test_IS_LIST_OF(self):
-        values = [0,1,2,3,4]
+        values = [0, 1, 2, 3, 4]
         rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10))(values)
         self.assertEqual(rtn, (values, None))
         values.append(11)
@@ -595,9 +693,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (values, 'Enter an integer between 0 and 9'))
         rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10))(1)
         self.assertEqual(rtn, ([1], None))
-        rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10), minimum=10)([1,2])
+        rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10), minimum=10)([1, 2])
         self.assertEqual(rtn, ([1, 2], 'Enter between 10 and 100 values'))
-        rtn =  IS_LIST_OF(IS_INT_IN_RANGE(0, 10), maximum=2)([1,2,3])
+        rtn = IS_LIST_OF(IS_INT_IN_RANGE(0, 10), maximum=2)([1, 2, 3])
         self.assertEqual(rtn, ([1, 2, 3], 'Enter between 0 and 2 values'))
         # regression test for issue 742
         rtn = IS_LIST_OF(minimum=1)('')
@@ -656,67 +754,69 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('a bc', 'Must be slug'))
 
     def test_ANY_OF(self):
-        rtn = ANY_OF([IS_EMAIL(),IS_ALPHANUMERIC()])('a@b.co')
+        rtn = ANY_OF([IS_EMAIL(), IS_ALPHANUMERIC()])('a@b.co')
         self.assertEqual(rtn, ('a@b.co', None))
-        rtn = ANY_OF([IS_EMAIL(),IS_ALPHANUMERIC()])('abco')
+        rtn = ANY_OF([IS_EMAIL(), IS_ALPHANUMERIC()])('abco')
         self.assertEqual(rtn, ('abco', None))
-        rtn = ANY_OF([IS_EMAIL(),IS_ALPHANUMERIC()])('@ab.co')
+        rtn = ANY_OF([IS_EMAIL(), IS_ALPHANUMERIC()])('@ab.co')
         self.assertEqual(rtn, ('@ab.co', 'Enter only letters, numbers, and underscore'))
-        rtn = ANY_OF([IS_ALPHANUMERIC(),IS_EMAIL()])('@ab.co')
+        rtn = ANY_OF([IS_ALPHANUMERIC(), IS_EMAIL()])('@ab.co')
         self.assertEqual(rtn, ('@ab.co', 'Enter a valid email address'))
-        rtn = ANY_OF([IS_DATE(),IS_EMAIL()])('a@b.co')
+        rtn = ANY_OF([IS_DATE(), IS_EMAIL()])('a@b.co')
         self.assertEqual(rtn, ('a@b.co', None))
-        rtn = ANY_OF([IS_DATE(),IS_EMAIL()])('1982-12-14')
+        rtn = ANY_OF([IS_DATE(), IS_EMAIL()])('1982-12-14')
         self.assertEqual(rtn, (datetime.date(1982, 12, 14), None))
+        rtn = ANY_OF([IS_DATE(format='%m/%d/%Y'), IS_EMAIL()]).formatter(datetime.date(1834, 12, 14))
+        self.assertEqual(rtn, '12/14/1834')
 
     def test_IS_EMPTY_OR(self):
         rtn = IS_EMPTY_OR(IS_EMAIL())('abc@def.com')
         self.assertEqual(rtn, ('abc@def.com', None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL())('   ')
+        rtn = IS_EMPTY_OR(IS_EMAIL())('   ')
         self.assertEqual(rtn, (None, None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL(), null='abc')('   ')
+        rtn = IS_EMPTY_OR(IS_EMAIL(), null='abc')('   ')
         self.assertEqual(rtn, ('abc', None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL(), null='abc', empty_regex='def')('def')
+        rtn = IS_EMPTY_OR(IS_EMAIL(), null='abc', empty_regex='def')('def')
         self.assertEqual(rtn, ('abc', None))
-        rtn =  IS_EMPTY_OR(IS_EMAIL())('abc')
+        rtn = IS_EMPTY_OR(IS_EMAIL())('abc')
         self.assertEqual(rtn, ('abc', 'Enter a valid email address'))
-        rtn =  IS_EMPTY_OR(IS_EMAIL())(' abc ')
+        rtn = IS_EMPTY_OR(IS_EMAIL())(' abc ')
         self.assertEqual(rtn, ('abc', 'Enter a valid email address'))
-        rtn = IS_EMPTY_OR(IS_IN_SET([('id1','first label'), ('id2','second label')], zero='zero')).options(zero=False)
+        rtn = IS_EMPTY_OR(IS_IN_SET([('id1', 'first label'), ('id2', 'second label')], zero='zero')).options(zero=False)
-        self.assertEqual(rtn, [('', ''),('id1', 'first label'), ('id2', 'second label')])
+        self.assertEqual(rtn, [('', ''), ('id1', 'first label'), ('id2', 'second label')])
-        rtn = IS_EMPTY_OR(IS_IN_SET([('id1','first label'), ('id2','second label')], zero='zero')).options()
+        rtn = IS_EMPTY_OR(IS_IN_SET([('id1', 'first label'), ('id2', 'second label')], zero='zero')).options()
-        self.assertEqual(rtn, [('', 'zero'),('id1', 'first label'), ('id2', 'second label')])
+        self.assertEqual(rtn, [('', 'zero'), ('id1', 'first label'), ('id2', 'second label')])
 
     def test_CLEANUP(self):
         rtn = CLEANUP()('helloò')
         self.assertEqual(rtn, ('hello', None))
 
     def test_CRYPT(self):
-        rtn = str(CRYPT(digest_alg='md5',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='md5', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^md5\$.{16}\$.{32}$')
-        rtn = str(CRYPT(digest_alg='sha1',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha1', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha1\$.{16}\$.{40}$')
-        rtn = str(CRYPT(digest_alg='sha256',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha256', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha256\$.{16}\$.{64}$')
-        rtn = str(CRYPT(digest_alg='sha384',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha384', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha384\$.{16}\$.{96}$')
-        rtn = str(CRYPT(digest_alg='sha512',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='sha512', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^sha512\$.{16}\$.{128}$')
         alg = 'pbkdf2(1000,20,sha512)'
-        rtn = str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg=alg, salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^pbkdf2\(1000,20,sha512\)\$.{16}\$.{40}$')
-        rtn = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0])
+        rtn = str(CRYPT(digest_alg='md5', key='mykey', salt=True)('test')[0])
         self.assertRegexpMatches(rtn, r'^md5\$.{16}\$.{32}$')
-        a = str(CRYPT(digest_alg='sha1',salt=False)('test')[0])
+        a = str(CRYPT(digest_alg='sha1', salt=False)('test')[0])
-        self.assertEqual(CRYPT(digest_alg='sha1',salt=False)('test')[0], a)
+        self.assertEqual(CRYPT(digest_alg='sha1', salt=False)('test')[0], a)
-        self.assertEqual(CRYPT(digest_alg='sha1',salt=False)('test')[0], a[6:])
+        self.assertEqual(CRYPT(digest_alg='sha1', salt=False)('test')[0], a[6:])
-        self.assertEqual(CRYPT(digest_alg='md5',salt=False)('test')[0], a)
+        self.assertEqual(CRYPT(digest_alg='md5', salt=False)('test')[0], a)
-        self.assertEqual(CRYPT(digest_alg='md5',salt=False)('test')[0], a[6:])
+        self.assertEqual(CRYPT(digest_alg='md5', salt=False)('test')[0], a[6:])
 
     def test_IS_STRONG(self):
         rtn = IS_STRONG(es=True)('Abcd1234')
         self.assertEqual(rtn, ('Abcd1234',
-         'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|'))
+                               'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|'))
         rtn = IS_STRONG(es=True)('Abcd1234!')
         self.assertEqual(rtn, ('Abcd1234!', None))
         rtn = IS_STRONG(es=True, entropy=1)('a')
@@ -737,33 +837,34 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('********', None))
         rtn = IS_STRONG(es=True, max=4)('abcde')
         self.assertEqual(rtn,
-            ('abcde',
+                         ('abcde',
-             '|'.join(['Minimum length is 8',
+                          '|'.join(['Minimum length is 8',
-                      'Maximum length is 4',
+                                    'Maximum length is 4',
-                      'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
+                                    'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
-                      'Must include at least 1 upper case',
+                                    'Must include at least 1 upper case',
-                      'Must include at least 1 number']))
+                                    'Must include at least 1 number']))
-            )
+                         )
         rtn = IS_STRONG(es=True)('abcde')
         self.assertEqual(rtn,
-            ('abcde',
+                         ('abcde',
-             '|'.join(['Minimum length is 8',
+                          '|'.join(['Minimum length is 8',
-                      'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
+                                    'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
-                      'Must include at least 1 upper case',
+                                    'Must include at least 1 upper case',
-                      'Must include at least 1 number']))
+                                    'Must include at least 1 number']))
-            )
+                         )
         rtn = IS_STRONG(upper=0, lower=0, number=0, es=True)('Abcde1')
         self.assertEqual(rtn,
-            ('Abcde1',
+                         ('Abcde1',
-             '|'.join(['Minimum length is 8',
+                          '|'.join(['Minimum length is 8',
-                      'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
+                                    'Must include at least 1 of the following: ~!@#$%^&*()_+-=?<>,.:;{}[]|',
-                      'May not include any upper case letters',
+                                    'May not include any upper case letters',
-                      'May not include any lower case letters',
+                                    'May not include any lower case letters',
-                      'May not include any numbers']))
+                                    'May not include any numbers']))
-            )
+                         )
 
     def test_IS_IMAGE(self):
         class DummyImageFile(object):
+
             def __init__(self, filename, ext, width, height):
                 from StringIO import StringIO
                 import struct
@@ -771,7 +872,7 @@ class TestValidators(unittest.TestCase):
                 self.file = StringIO()
                 if ext == 'bmp':
                     self.file.write(b'BM')
-                    self.file.write(b' '*16)
+                    self.file.write(b' ' * 16)
                     self.file.write(struct.pack('<LL', width, height))
                 elif ext == 'gif':
                     self.file.write(b'GIF87a')
@@ -782,7 +883,7 @@ class TestValidators(unittest.TestCase):
                     self.file.write(struct.pack('!xHH', height, width))
                 elif ext == 'png':
                     self.file.write(b'\211PNG\r\n\032\n')
-                    self.file.write(b' '*4)
+                    self.file.write(b' ' * 4)
                     self.file.write(b'IHDR')
                     self.file.write(struct.pack('!LL', width, height))
                 self.file.seek(0)
@@ -823,10 +924,10 @@ class TestValidators(unittest.TestCase):
         rtn = IS_IMAGE(error_message='oops')(img)
         self.assertEqual(rtn, (img, 'oops'))
 
-
     def test_IS_UPLOAD_FILENAME(self):
         import cgi
         from StringIO import StringIO
+
         def gen_fake(filename):
             formdata_file_data = """
 ---123
@@ -934,7 +1035,7 @@ this is the content of the fake file
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPV6(subnets='fb00::/8')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'Enter valid IPv6 address'))
-        rtn = IS_IPV6(subnets=['fc00::/8','2001::/32'])('2001::8ffa:fe22:b3af')
+        rtn = IS_IPV6(subnets=['fc00::/8', '2001::/32'])('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPV6(subnets='invalidsubnet')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'invalid subnet provided'))
@@ -1009,7 +1110,7 @@ this is the content of the fake file
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPADDRESS(subnets='fb00::/8')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'Enter valid IP address'))
-        rtn = IS_IPADDRESS(subnets=['fc00::/8','2001::/32'])('2001::8ffa:fe22:b3af')
+        rtn = IS_IPADDRESS(subnets=['fc00::/8', '2001::/32'])('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', None))
         rtn = IS_IPADDRESS(subnets='invalidsubnet')('2001::8ffa:fe22:b3af')
         self.assertEqual(rtn, ('2001::8ffa:fe22:b3af', 'invalid subnet provided'))

gluon/tools.py

@@ -820,12 +820,12 @@ class Recaptcha(DIV):
     Examples:
         Use as::
 
-            form = FORM(Recaptcha(public_key='...',private_key='...'))
+            form = FORM(Recaptcha(public_key='...', private_key='...'))
 
         or::
 
             form = SQLFORM(...)
-            form.append(Recaptcha(public_key='...',private_key='...'))
+            form.append(Recaptcha(public_key='...', private_key='...'))
 
     """
 
@@ -984,17 +984,17 @@ class Recaptcha2(DIV):
     Examples:
         Use as::
 
-            form = FORM(Recaptcha2(public_key='...',private_key='...'))
+            form = FORM(Recaptcha2(public_key='...', private_key='...'))
 
         or::
 
             form = SQLFORM(...)
-            form.append(Recaptcha2(public_key='...',private_key='...'))
+            form.append(Recaptcha2(public_key='...', private_key='...'))
 
         to protect the login page instead, use::
 
             from gluon.tools import Recaptcha2
-            auth.settings.captcha = Recaptcha2(request, public_key='...',private_key='...')
+            auth.settings.captcha = Recaptcha2(request, public_key='...', private_key='...')
 
     """
 
@@ -1217,7 +1217,7 @@ class AuthJWT(object):
         self.auth = auth
         self.algorithm = algorithm
         if self.algorithm not in ('HS256', 'HS384', 'HS512'):
-            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
+            raise NotImplementedError('Algorithm %s not allowed' % algorithm)
         self.verify_expiration = verify_expiration
         self.leeway = leeway
         self.expiration = expiration
@@ -1314,6 +1314,7 @@ class AuthJWT(object):
         We (mis)use the heavy default auth mechanism to avoid any further computation,
         while sticking to a somewhat-stable Auth API.
         """
+        # TODO: Check the following comment
         ## is the following safe or should we use
         ## calendar.timegm(datetime.datetime.utcnow().timetuple())
         ## result seem to be the same (seconds since epoch, in UTC)
@@ -1395,9 +1396,10 @@ class AuthJWT(object):
             self.alter_payload(payload)
             ret = {'token': self.generate_token(payload)}
         elif ret is None:
-            raise HTTP(
+            raise HTTP(401,
-                401, u'Not Authorized - need to be logged in, to pass a token for refresh or username and password for login',
+                       u'Not Authorized - need to be logged in, to pass a token '
-                **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
+                       u'for refresh or username and password for login',
+                       **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
         response.headers['Content-Type'] = 'application/json'
         return serializers.json(ret)
 
@@ -1450,9 +1452,9 @@ class Auth(object):
         everybody_group_id=None,
         manager_actions={},
         auth_manager_role=None,
-        two_factor_authentication_group = None,
+        two_factor_authentication_group=None,
-        auth_two_factor_enabled = False,
+        auth_two_factor_enabled=False,
-        auth_two_factor_tries_left = 3,
+        auth_two_factor_tries_left=3,
         login_captcha=None,
         register_captcha=None,
         pre_registration_div=None,
@@ -1469,7 +1471,7 @@ class Auth(object):
         on_failed_authentication=lambda x: redirect(x),
         formstyle=None,
         label_separator=None,
-        logging_enabled = True,
+        logging_enabled=True,
         allow_delete_accounts=False,
         password_field='password',
         table_user_name='auth_user',
@@ -1735,6 +1737,7 @@ class Auth(object):
             host = host_names[0]
         else:
             host = 'localhost'
+        return host
 
     def __init__(self, environment=None, db=None, mailer=True,
                  hmac_key=None, controller='default', function='user',
@@ -1742,7 +1745,7 @@ class Auth(object):
                  csrf_prevention=True, propagate_extension=None,
                  url_index=None, jwt=None, host_names=None):
 
-        ## next two lines for backward compatibility
+        # next two lines for backward compatibility
         if not db and environment and isinstance(environment, DAL):
             db = environment
         self.db = db
@@ -1779,7 +1782,7 @@ class Auth(object):
 
         url_index = url_index or URL(controller, 'index')
         url_login = URL(controller, function, args='login',
-                        extension = propagate_extension)
+                        extension=propagate_extension)
         # ## what happens after registration?
 
         settings = self.settings = Settings()
@@ -1834,9 +1837,9 @@ class Auth(object):
             hmac_key=hmac_key,
             formstyle=current.response.formstyle,
             label_separator=current.response.form_label_separator,
-            two_factor_methods = [],
+            two_factor_methods=[],
-            two_factor_onvalidation = [],
+            two_factor_onvalidation=[],
-            host = host,
+            host=host,
         )
         settings.lock_keys = True
         # ## these are messages that can be customized
@@ -1931,7 +1934,7 @@ class Auth(object):
                        'reset_password', 'request_reset_password',
                        'change_password', 'profile', 'groups',
                        'impersonate', 'not_authorized', 'confirm_registration',
-                       'bulk_register','manage_tokens','jwt'):
+                       'bulk_register', 'manage_tokens', 'jwt'):
             if len(request.args) >= 2 and args[0] == 'impersonate':
                 return getattr(self, args[0])(request.args[1])
             else:
@@ -1973,10 +1976,8 @@ class Auth(object):
         else:
             next = '?_next=' + urllib.quote(URL(args=request.args,
                                                 vars=request.get_vars))
-        href = lambda function: '%s/%s%s' % (action, function, next
+        href = lambda function: \
-                                             if referrer_actions is DEFAULT
+            '%s/%s%s' % (action, function, next if referrer_actions is DEFAULT or function in referrer_actions else '')
-                                             or function in referrer_actions
-                                             else '')
         if isinstance(prefix, str):
             prefix = T(prefix)
         if prefix:
@@ -1989,9 +1990,7 @@ class Auth(object):
         if self.user_id:  # User is logged in
             logout_next = self.settings.logout_next
             items.append({'name': T('Log Out'),
-                          'href': '%s/logout?_next=%s' % (action,
+                          'href': '%s/logout?_next=%s' % (action, urllib.quote(logout_next)),
-                                                          urllib.quote(
-                                                          logout_next)),
                           'icon': 'icon-off'})
             if 'profile' not in self.settings.actions_disabled:
                 items.append({'name': T('Profile'), 'href': href('profile'),
@@ -2024,8 +2023,8 @@ class Auth(object):
             if (self.settings.use_username and not
                     'retrieve_username' in self.settings.actions_disabled):
                 items.append({'name': T('Forgot username?'),
-                             'href': href('retrieve_username'),
+                              'href': href('retrieve_username'),
-                             'icon': 'icon-edit'})
+                              'icon': 'icon-edit'})
 
         def menu():  # For inclusion in MENU
             self.bar = [(items[0]['name'], False, items[0]['href'], [])]
@@ -2145,11 +2144,11 @@ class Auth(object):
             if self.user_id:
                 self.bar = SPAN(prefix, user_identifier, s1,
                                 Anr(items[0]['name'],
-                                _href=items[0]['href']), s3,
+                                    _href=items[0]['href']), s3,
                                 _class='auth_navbar')
             else:
                 self.bar = SPAN(s1, Anr(items[0]['name'],
-                                _href=items[0]['href']), s3,
+                                        _href=items[0]['href']), s3,
                                 _class='auth_navbar')
             for item in items[1:]:
                 self.bar.insert(-1, s2)
@@ -2206,11 +2205,10 @@ class Auth(object):
             if ('id' in fieldnames and
                 'modified_on' in fieldnames and
                 not current_record in fieldnames):
-                table._enable_record_versioning(
+                table._enable_record_versioning(archive_db=archive_db,
-                    archive_db=archive_db,
+                                                archive_name=archive_names,
-                    archive_name=archive_names,
+                                                current_record=current_record,
-                    current_record=current_record,
+                                                current_record_label=current_record_label)
-                    current_record_label=current_record_label)
 
     def define_signature(self):
         db = self.db
@@ -2474,13 +2472,11 @@ class Auth(object):
                     settings.table_token_name,
                     Field('user_id', reference_table_user, default=None,
                           label=self.messages.label_user_id),
-                    Field('expires_on', 'datetime', default=datetime.datetime(2999,12,31)),
+                    Field('expires_on', 'datetime', default=datetime.datetime(2999, 12, 31)),
-                    Field('token',writable=False,default=web2py_uuid,unique=True),
+                    Field('token', writable=False, default=web2py_uuid, unique=True),
                     *extra_fields,
-                    **dict(
+                    **dict(migrate=self.__get_migrate(settings.table_token_name, migrate),
-                        migrate=self.__get_migrate(
+                           fake_migrate=fake_migrate))
-                            settings.table_token_name, migrate),
-                        fake_migrate=fake_migrate))
         if not db._lazy_tables:
             settings.table_user = db[settings.table_user_name]
             settings.table_group = db[settings.table_group_name]
@@ -2531,9 +2527,7 @@ class Auth(object):
         # log messages should not be translated
         if type(description).__name__ == 'lazyT':
             description = description.m
-        self.table_event().insert(
+        self.table_event().insert(description=str(description % vars), origin=origin, user_id=user_id)
-            description=str(description % vars),
-            origin=origin, user_id=user_id)
 
     def get_or_create_user(self, keys, update_fields=['email'],
                            login=True, get=True):
@@ -2575,15 +2569,14 @@ class Auth(object):
                     update_keys[key] = keys[key]
             user.update_record(**update_keys)
         elif checks:
-            if not 'first_name' in keys and 'first_name' in table_user.fields:
+            if 'first_name' not in keys and 'first_name' in table_user.fields:
                 guess = keys.get('email', 'anonymous').split('@')[0]
                 keys['first_name'] = keys.get('username', guess)
             vars = table_user._filter_fields(keys)
             user_id = table_user.insert(**vars)
             user = table_user[user_id]
             if self.settings.create_user_groups:
-                group_id = self.add_group(
+                group_id = self.add_group(self.settings.create_user_groups % user)
-                    self.settings.create_user_groups % user)
                 self.add_membership(group_id, user_id)
             if self.settings.everybody_group_id:
                 self.add_membership(self.settings.everybody_group_id, user_id)
@@ -2698,8 +2691,7 @@ class Auth(object):
             fields[settings.passfield] = \
                 settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
         if not fields.get(settings.userfield):
-            raise ValueError('register_bare: ' +
+            raise ValueError('register_bare: userfield not provided or invalid')
-                             'userfield not provided or invalid')
         user = self.get_or_create_user(fields, login=False, get=False,
                                        update_fields=self.settings.update_fields)
         if not user:
@@ -2750,7 +2742,7 @@ class Auth(object):
             redirect(session._cas_service)
 
         def cas_onaccept(form, onaccept=onaccept):
-            if not onaccept is DEFAULT:
+            if onaccept is not DEFAULT:
                 onaccept(form)
             return allow_access(interactivelogin=True)
         return self.login(next, onvalidation, cas_onaccept, log)
@@ -2837,14 +2829,14 @@ class Auth(object):
         response = current.response
         session = current.session
 
-        ### use session for federated login
+        # use session for federated login
         snext = self.get_vars_next()
 
         if snext:
             session._auth_next = snext
         elif session._auth_next:
             snext = session._auth_next
-        ### pass
+        # pass
 
         if next is DEFAULT:
             # important for security
@@ -2950,7 +2942,7 @@ class Auth(object):
                                )
 
                 captcha = settings.login_captcha or \
-                    (settings.login_captcha != False and settings.captcha)
+                    (settings.login_captcha is not False and settings.captcha)
                 if captcha:
                     addrow(form, captcha.label, captcha, captcha.comment,
                            settings.formstyle, 'captcha__row')
@@ -2978,8 +2970,7 @@ class Auth(object):
                         elif temp_user.registration_key in ('disabled', 'blocked'):
                             response.flash = self.messages.login_disabled
                             return form
-                        elif (temp_user.registration_key is not None
+                        elif (temp_user.registration_key is not None and temp_user.registration_key.strip()):
-                              and temp_user.registration_key.strip()):
                             response.flash = \
                                 self.messages.registration_verifying
                             return form
@@ -3090,7 +3081,7 @@ class Auth(object):
                         subject=self.messages.retrieve_two_factor_code_subject,
                         message=self.messages.retrieve_two_factor_code.format(session.auth_two_factor))
                 else:
-                    #Check for all method. It is possible to have multiples
+                    # Check for all method. It is possible to have multiples
                     for two_factor_method in two_factor_methods:
                         try:
                             # By default we use session.auth_two_factor generated before.
@@ -3106,8 +3097,6 @@ class Auth(object):
                             hideerror=settings.hideerror):
                 accepted_form = True
 
-                accepted_form = True
-
                 '''
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
@@ -3251,12 +3240,16 @@ class Auth(object):
                 next = cas.logout_url(next)
 
         current.session.auth = None
+        self.user = None
         if self.settings.renew_session_onlogout:
             current.session.renew(clear_session=not self.settings.keep_session_onlogout)
         current.session.flash = self.messages.logged_out
         if next is not None:
             redirect(next)
 
+    def logout_bare(self):
+        self.logout(next=None, onlogout=None, log=None)
+
     def register(self,
                  next=DEFAULT,
                  onvalidation=DEFAULT,
@@ -3308,7 +3301,7 @@ class Auth(object):
 
         passfield = self.settings.password_field
         formstyle = self.settings.formstyle
-        try: # Make sure we have our original minimum length as other auth forms change it
+        try:  # Make sure we have our original minimum length as other auth forms change it
             table_user[passfield].requires[-1].min_length = self.settings.password_min_length
         except:
             pass
@@ -3349,7 +3342,7 @@ class Auth(object):
 
         key = web2py_uuid()
         if self.settings.registration_requires_approval:
-            key = 'pending-'+key
+            key = 'pending-' + key
 
         table_user.registration_key.default = key
         if form.accepts(request, session if self.csrf_prevention else None,
@@ -3358,12 +3351,10 @@ class Auth(object):
                         hideerror=self.settings.hideerror):
             description = self.messages.group_description % form.vars
             if self.settings.create_user_groups:
-                group_id = self.add_group(
+                group_id = self.add_group(self.settings.create_user_groups % form.vars, description)
-                    self.settings.create_user_groups % form.vars, description)
                 self.add_membership(group_id, form.vars.id)
             if self.settings.everybody_group_id:
-                self.add_membership(
+                self.add_membership(self.settings.everybody_group_id, form.vars.id)
-                    self.settings.everybody_group_id, form.vars.id)
             if self.settings.registration_requires_verification:
                 link = self.url(
                     self.settings.function, args=('verify_email', key), scheme=True)
@@ -3381,8 +3372,7 @@ class Auth(object):
                not self.settings.registration_requires_verification:
                 table_user[form.vars.id] = dict(registration_key='pending')
                 session.flash = self.messages.registration_pending
-            elif (not self.settings.registration_requires_verification or
+            elif (not self.settings.registration_requires_verification or self.settings.login_after_registration):
-                      self.settings.login_after_registration):
                 if not self.settings.registration_requires_verification:
                     table_user[form.vars.id] = dict(registration_key='')
                 session.flash = self.messages.registration_successful
@@ -3460,7 +3450,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_username_captcha or \
-                  (self.settings.retrieve_username_captcha != False and self.settings.captcha)
+                  (self.settings.retrieve_username_captcha is not False and self.settings.captcha)
         if not self.settings.mailer:
             response.flash = self.messages.function_disabled
             return ''
@@ -3618,7 +3608,7 @@ class Auth(object):
 
         if self.settings.prevent_password_reset_attacks:
             key = request.vars.key
-            if not key and len(request.args)>1:
+            if not key and len(request.args) > 1:
                 key = request.args[-1]
             if key:
                 session._reset_password_key = key
@@ -3728,7 +3718,7 @@ class Auth(object):
     def manage_tokens(self):
         if not self.user:
             redirect(self.settings.login_url)
-        table_token =self.table_token()
+        table_token = self.table_token()
         table_token.user_id.writable = False
         table_token.user_id.default = self.user.id
         table_token.token.writable = False
@@ -3792,8 +3782,7 @@ class Auth(object):
                   requires=self.table_user()[passfield].requires),
             Field('new_password2', 'password',
                   label=self.messages.verify_password,
-                  requires=[IS_EXPR(
+                  requires=[IS_EXPR('value==%s' % repr(request.vars.new_password),
-                      'value==%s' % repr(request.vars.new_password),
                                     self.messages.mismatched_password)]),
             submit_button=self.messages.password_reset_button,
             hidden=dict(_next=next),
@@ -3827,7 +3816,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_password_captcha or \
-                (self.settings.retrieve_password_captcha != False and self.settings.captcha)
+                (self.settings.retrieve_password_captcha is not False and self.settings.captcha)
 
         if next is DEFAULT:
             next = self.get_vars_next() or self.settings.request_reset_password_next
@@ -3872,7 +3861,7 @@ class Auth(object):
                         formname='reset_password', dbio=False,
                         onvalidation=onvalidation,
                         hideerror=self.settings.hideerror):
-            user = table_user(**{userfield:form.vars.get(userfield)})
+            user = table_user(**{userfield: form.vars.get(userfield)})
             key = user.registration_key
             if not user:
                 session.flash = self.messages['invalid_%s' % userfield]
@@ -4124,6 +4113,7 @@ class Auth(object):
             raise HTTP(401, "Not Authorized")
         current_id = auth.user.id
         requested_id = user_id
+        user = None
         if user_id is DEFAULT:
             user_id = current.request.post_vars.user_id
         if user_id and user_id != self.user.id and user_id != '0':
@@ -4152,7 +4142,10 @@ class Auth(object):
             return None
         if requested_id is DEFAULT and not request.post_vars:
             return SQLFORM.factory(Field('user_id', 'integer'))
-        return SQLFORM(table_user, user.id, readonly=True)
+        elif not user:
+            return None
+        else:
+            return SQLFORM(table_user, user.id, readonly=True)
 
     def update_groups(self):
         if not self.user:
@@ -4234,10 +4227,8 @@ class Auth(object):
                         else:
                             next = self.here()
                             current.session.flash = current.response.flash
-                            return call_or_redirect(
+                            return call_or_redirect(self.settings.on_failed_authentication,
-                                self.settings.on_failed_authentication,
+                                                    self.settings.login_url + '?_next=' + urllib.quote(next))
-                                self.settings.login_url +
-                                    '?_next=' + urllib.quote(next))
 
                 if callable(condition):
                     flag = condition()
@@ -4318,11 +4309,8 @@ class Auth(object):
         """
         Creates a group associated to a role
         """
-
+        group_id = self.table_group().insert(role=role, description=description)
-        group_id = self.table_group().insert(
+        self.log_event(self.messages['add_group_log'], dict(group_id=group_id, role=role))
-            role=role, description=description)
-        self.log_event(self.messages['add_group_log'],
-                       dict(group_id=group_id, role=role))
         return group_id
 
     def del_group(self, group_id):
@@ -4332,7 +4320,8 @@ class Auth(object):
         self.db(self.table_group().id == group_id).delete()
         self.db(self.table_membership().group_id == group_id).delete()
         self.db(self.table_permission().group_id == group_id).delete()
-        if group_id in self.user_groups: del self.user_groups[group_id]
+        if group_id in self.user_groups:
+            del self.user_groups[group_id]
         self.log_event(self.messages.del_group_log, dict(group_id=group_id))
 
     def id_group(self, role):
@@ -4364,7 +4353,6 @@ class Auth(object):
         """
         Checks if user is member of group_id or role
         """
-
         group_id = group_id or self.id_group(role)
         try:
             group_id = int(group_id)
@@ -4373,8 +4361,8 @@ class Auth(object):
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
-        if group_id and user_id and self.db((membership.user_id == user_id)
+        if group_id and user_id and self.db((membership.user_id == user_id) &
-                    & (membership.group_id == group_id)).select():
+                                            (membership.group_id == group_id)).select():
             r = True
         else:
             r = False
@@ -4397,8 +4385,8 @@ class Auth(object):
             user_id = self.user.id
         membership = self.table_membership()
         db = membership._db
-        record = db((membership.user_id==user_id)&
+        record = db((membership.user_id == user_id) &
-                    (membership.group_id==group_id),
+                    (membership.group_id == group_id),
                     ignore_common_filters=True).select().first()
         if record:
             if hasattr(record, 'is_active') and not record.is_active:
@@ -4421,15 +4409,18 @@ class Auth(object):
         """
 
         group_id = group_id or self.id_group(role)
+        try:
+            group_id = int(group_id)
+        except:
+            group_id = self.id_group(group_id)  # interpret group_id as a role
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
         self.log_event(self.messages['del_membership_log'],
                        dict(user_id=user_id, group_id=group_id))
-        ret = self.db(membership.user_id
+        ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
-                      == user_id)(membership.group_id
+        if group_id in self.user_groups:
-                                  == group_id).delete()
+            del self.user_groups[group_id]
-        if group_id in self.user_groups: del self.user_groups[group_id]
         return ret
 
     def has_permission(self,
@@ -4446,34 +4437,32 @@ class Auth(object):
         """
 
         if not group_id and self.settings.everybody_group_id and \
-                self.has_permission(
+                self.has_permission(name, table_name, record_id, user_id=None,
-            name, table_name, record_id, user_id=None,
+                                    group_id=self.settings.everybody_group_id):
-            group_id=self.settings.everybody_group_id):
                 return True
 
         if not user_id and not group_id and self.user:
             user_id = self.user.id
         if user_id:
             membership = self.table_membership()
-            rows = self.db(membership.user_id
+            rows = self.db(membership.user_id == user_id).select(membership.group_id)
-                           == user_id).select(membership.group_id)
             groups = set([row.group_id for row in rows])
             if group_id and group_id not in groups:
                 return False
         else:
             groups = set([group_id])
         permission = self.table_permission()
-        rows = self.db(permission.name == name)(permission.table_name
+        rows = self.db(permission.name ==
-                 == str(table_name))(permission.record_id
+                       name)(permission.table_name ==
-                 == record_id).select(permission.group_id)
+                             str(table_name))(permission.record_id ==
+                                              record_id).select(permission.group_id)
         groups_required = set([row.group_id for row in rows])
         if record_id:
-            rows = self.db(permission.name
+            rows = self.db(permission.name ==
-                            == name)(permission.table_name
+                           name)(permission.table_name ==
-                     == str(table_name))(permission.record_id
+                                 str(table_name))(permission.record_id ==
-                     == 0).select(permission.group_id)
+                                                  0).select(permission.group_id)
-            groups_required = groups_required.union(set([row.group_id
+            groups_required = groups_required.union(set([row.group_id for row in rows]))
-                    for row in rows]))
         if groups.intersection(groups_required):
             r = True
         else:
@@ -4497,14 +4486,14 @@ class Auth(object):
         permission = self.table_permission()
         if group_id == 0:
             group_id = self.user_group()
-        record = self.db((permission.group_id == group_id)&
+        record = self.db((permission.group_id == group_id) &
-                         (permission.name == name)&
+                         (permission.name == name) &
-                         (permission.table_name == str(table_name))&
+                         (permission.table_name == str(table_name)) &
                          (permission.record_id == long(record_id)),
-                          ignore_common_filters=True).select(
+                         ignore_common_filters=True
-            limitby=(0, 1), orderby_on_limitby=False).first()
+                         ).select(limitby=(0, 1), orderby_on_limitby=False).first()
         if record:
-            if hasattr(record, 'is_active') and not record.is_ctive:
+            if hasattr(record, 'is_active') and not record.is_active:
                 record.update_record(is_active=True)
             id = record.id
         else:
@@ -4531,10 +4520,11 @@ class Auth(object):
         self.log_event(self.messages['del_permission_log'],
                        dict(group_id=group_id, name=name,
                             table_name=table_name, record_id=record_id))
-        return self.db(permission.group_id == group_id)(permission.name
+        return self.db(permission.group_id ==
-                 == name)(permission.table_name
+                       group_id)(permission.name ==
-                           == str(table_name))(permission.record_id
+                                 name)(permission.table_name ==
-                 == long(record_id)).delete()
+                                       str(table_name))(permission.record_id ==
+                                                        long(record_id)).delete()
 
     def accessible_query(self, name, table, user_id=None):
         """
@@ -4561,10 +4551,9 @@ class Auth(object):
                 cquery = table
             tablenames = db._adapter.tables(cquery)
             for tablename in tablenames:
-                cquery &= self.accessible_query(name, tablename,
+                cquery &= self.accessible_query(name, tablename, user_id=user_id)
-                                                user_id=user_id)
             return cquery
-        if not isinstance(table, str) and\
+        if not isinstance(table, str) and \
                 self.has_permission(name, table, 0, user_id):
             return table.id > 0
         membership = self.table_membership()
@@ -4593,11 +4582,11 @@ class Auth(object):
         If you have a table (db.mytable) that needs full revision history you
         can just do::
 
-            form=crud.update(db.mytable,myrecord,onaccept=auth.archive)
+            form = crud.update(db.mytable, myrecord, onaccept=auth.archive)
 
         or::
 
-            form=SQLFORM(db.mytable,myrecord).process(onaccept=auth.archive)
+            form = SQLFORM(db.mytable, myrecord).process(onaccept=auth.archive)
 
         crud.archive will define a new table "mytable_archive" and store
         a copy of the current record (if archive_current=True)
@@ -4611,18 +4600,18 @@ class Auth(object):
         in a model::
 
             db.define_table('mytable_archive',
-                Field('current_record',db.mytable),
+                            Field('current_record', db.mytable),
-                db.mytable)
+                            db.mytable)
 
         Notice such table includes all fields of db.mytable plus one: current_record.
         crud.archive does not timestamp the stored record unless your original table
         has a fields like::
 
             db.define_table(...,
-                Field('saved_on','datetime',
+                Field('saved_on', 'datetime',
-                     default=request.now,update=request.now,writable=False),
+                      default=request.now, update=request.now, writable=False),
-                Field('saved_by',auth.user,
+                Field('saved_by', auth.user,
-                     default=auth.user_id,update=auth.user_id,writable=False),
+                      default=auth.user_id, update=auth.user_id, writable=False),
 
         there is nothing special about these fields since they are filled before
         the record is archived.
@@ -4631,15 +4620,14 @@ class Auth(object):
         you can do, for example::
 
             db.define_table('myhistory',
-                Field('parent_record',db.mytable),
+                Field('parent_record', db.mytable), db.mytable)
-                db.mytable)
 
         and use it as::
 
-            form=crud.update(db.mytable,myrecord,
+            form = crud.update(db.mytable, myrecord,
-                             onaccept=lambda form:crud.archive(form,
+                               onaccept=lambda form:crud.archive(form,
-                             archive_table=db.myhistory,
+                                                                 archive_table=db.myhistory,
-                             current_record='parent_record'))
+                                                                 current_record='parent_record'))
 
         """
         if not archive_current and not form.record:
@@ -4647,7 +4635,7 @@ class Auth(object):
         table = form.table
         if not archive_table:
             archive_table_name = '%s_archive' % table
-            if not archive_table_name in table._db:
+            if archive_table_name not in table._db:
                 table._db.define_table(
                     archive_table_name,
                     Field(current_record, table),
@@ -4902,18 +4890,16 @@ class Crud(object):
         self.deleted = False
         captcha = self.settings.update_captcha or self.settings.captcha
         if record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
-                         self.settings.formstyle, 'captcha__row')
         captcha = self.settings.create_captcha or self.settings.captcha
         if not record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
-                         self.settings.formstyle, 'captcha__row')
         if request.extension not in ('html', 'load'):
             (_session, _formname) = (None, None)
         else:
             (_session, _formname) = (
                 session, '%s/%s' % (table._tablename, form.record_id))
-        if not formname is DEFAULT:
+        if formname is not DEFAULT:
             _formname = formname
         keepvalues = self.settings.keepvalues
         if request.vars.delete_this_record:
@@ -5202,7 +5188,7 @@ class Crud(object):
                     elif validate:
                         value, error = field.validate(txtval)
                         if not error:
-                            ### TODO deal with 'starts with', 'ends with', 'contains' on GAE
+                            # TODO deal with 'starts with', 'ends with', 'contains' on GAE
                             query &= self.get_query(field, opval, value)
                         else:
                             row[3].append(DIV(error, _class='error'))
@@ -5215,7 +5201,7 @@ class Crud(object):
                 results = db(query).select(*selected, **attributes)
                 for r in refsearch:
                     results = results.find(r)
-            except:  # hmmm, we should do better here
+            except:  # TODO: hmmm, we should do better here
                 results = None
         return form, results
 
@@ -5227,7 +5213,7 @@ def fetch(url, data=None, headers=None,
           cookie=Cookie.SimpleCookie(),
           user_agent='Mozilla/5.0'):
     headers = headers or {}
-    if not data is None:
+    if data is not None:
         data = urllib.urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
@@ -5690,12 +5676,12 @@ class Service(object):
         methods = self.jsonrpc_procedures
         data = json_parser.loads(request.body.read())
         jsonrpc_2 = data.get('jsonrpc')
-        if jsonrpc_2: #hand over to version 2 of the protocol
+        if jsonrpc_2:  # hand over to version 2 of the protocol
             return self.serve_jsonrpc2(data)
         id, method, params = data.get('id'), data.get('method'), data.get('params', [])
         if id is None:
             return return_error(0, 100, 'missing id')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, 100, 'method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -5719,8 +5705,7 @@ class Service(object):
         def return_response(id, result):
             if not must_respond:
                 return None
-            return serializers.json({'jsonrpc': '2.0',
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'result': result})
-                'id': id, 'result': result})
 
         def return_error(id, code, message=None, data=None):
             error = {'code': code}
@@ -5731,9 +5716,7 @@ class Service(object):
                 error['message'] = message
             if data is not None:
                 error['data'] = data
-            return serializers.json({'jsonrpc': '2.0',
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'error': error})
-                                     'id': id,
-                                     'error': error})
 
         def validate(data):
             """
@@ -5793,7 +5776,7 @@ class Service(object):
             return return_error(None, e.code, e.info)
 
         id, method, params = data.get('id'), data['method'], data.get('params', '')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, -32601, data='Method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -5899,7 +5882,7 @@ class Service(object):
                     UL(LI("Location: %s" % dispatcher.location),
                        LI("Namespace: %s" % dispatcher.namespace),
                        LI("SoapAction: %s" % dispatcher.action),
-                    ),
+                       ),
                     H3("Sample SOAP XML Request Message:"),
                     CODE(sample_req_xml, language="xml"),
                     H3("Sample SOAP XML Response Message:"),
@@ -6033,7 +6016,7 @@ def prettydate(d, T=lambda x: x, utc=False):
         return T('1 year' + suffix)
     elif dt.days >= 60:
         return T('%d months' + suffix) % int(dt.days / 30)
-    elif dt.days > 21:
+    elif dt.days >= 27:  # 4 weeks ugly
         return T('1 month' + suffix)
     elif dt.days >= 14:
         return T('%d weeks' + suffix) % int(dt.days / 7)
@@ -6439,10 +6422,9 @@ class Wiki(object):
                 args += value['args']
                 db.define_table(key, *args, **value['vars'])
 
-        if self.settings.templates is None and not \
+        if self.settings.templates is None and not self.settings.manage_permissions:
-           self.settings.manage_permissions:
+            self.settings.templates = \
-            self.settings.templates = db.wiki_page.tags.contains('template') & \
+                db.wiki_page.tags.contains('template') & db.wiki_page.can_read.contains('everybody')
-                db.wiki_page.can_read.contains('everybody')
 
         def update_tags_insert(page, id, db=db):
             for tag in page.tags or []:
@@ -6465,8 +6447,10 @@ class Wiki(object):
             'wiki_editor' not in auth.user_groups.values() and
                 self.settings.groups == auth.user_groups.values()):
             group = db.auth_group(role='wiki_editor')
-            gid = group.id if group else db.auth_group.insert(
+            if group:
-                role='wiki_editor')
+                gid = group.id
+            else:
+                db.auth_group.insert(role='wiki_editor')
             auth.add_membership(gid)
 
         settings.lock_keys = True
@@ -6493,9 +6477,8 @@ class Wiki(object):
         groups = self.settings.groups
         return ('wiki_editor' in groups or
                 (page is None and 'wiki_author' in groups) or
-                not page is None and (
+                page is not None and (set(groups).intersection(set(page.can_edit)) or
-                set(groups).intersection(set(page.can_edit)) or
+                                      page.created_by == self.auth.user.id))
-                page.created_by == self.auth.user.id))
 
     def can_manage(self):
         if not self.auth.user:
@@ -6516,7 +6499,7 @@ class Wiki(object):
                     return True
         return False
 
-    ### END POLICY
+    # END POLICY
 
     def automenu(self):
         """adds the menu if not present"""
@@ -6734,20 +6717,15 @@ class Wiki(object):
         if self.settings.templates:
             fields.append(
                 Field("from_template", "reference wiki_page",
-                      requires=IS_EMPTY_OR(
+                      requires=IS_EMPTY_OR(IS_IN_DB(db(self.settings.templates), db.wiki_page._id, '%(slug)s')),
-                                   IS_IN_DB(db(self.settings.templates),
+                      comment=current.T("Choose Template or empty for new Page")))
-                                            db.wiki_page._id,
-                                            '%(slug)s')),
-                      comment=current.T(
-                        "Choose Template or empty for new Page")))
         form = SQLFORM.factory(*fields, **dict(_class="well"))
         form.element("[type=submit]").attributes["_value"] = \
             current.T("Create Page from Slug")
 
         if form.process().accepted:
-             form.vars.from_template = 0 if not form.vars.from_template \
+            form.vars.from_template = 0 if not form.vars.from_template else form.vars.from_template
-                 else form.vars.from_template
+            redirect(URL(args=('_edit', form.vars.slug, form.vars.from_template or 0)))  # added param
-             redirect(URL(args=('_edit', form.vars.slug, form.vars.from_template or 0)))  # added param
         return dict(content=form)
 
     def pages(self):
@@ -6838,25 +6816,25 @@ class Wiki(object):
                     mode = 0
                 if mode in (2, 3):
                     submenu.append((current.T('View Page'), None,
-                    URL(controller, function, args=slug)))
+                                    URL(controller, function, args=slug)))
                 if mode in (1, 3):
                     submenu.append((current.T('Edit Page'), None,
-                    URL(controller, function, args=('_edit', slug))))
+                                    URL(controller, function, args=('_edit', slug))))
                 if mode in (1, 2):
                     submenu.append((current.T('Edit Page Media'), None,
-                    URL(controller, function, args=('_editmedia', slug))))
+                                    URL(controller, function, args=('_editmedia', slug))))
 
             submenu.append((current.T('Create New Page'), None,
                             URL(controller, function, args=('_create'))))
             # Moved next if to inside self.auth.user check
             if self.can_manage():
                 submenu.append((current.T('Manage Pages'), None,
-                            URL(controller, function, args=('_pages'))))
+                                URL(controller, function, args=('_pages'))))
                 submenu.append((current.T('Edit Menu'), None,
-                            URL(controller, function, args=('_edit', 'wiki-menu'))))
+                                URL(controller, function, args=('_edit', 'wiki-menu'))))
             # Also moved inside self.auth.user check
             submenu.append((current.T('Search Pages'), None,
-                        URL(controller, function, args=('_search'))))
+                            URL(controller, function, args=('_search'))))
         return menu
 
     def search(self, tags=None, query=None, cloud=True, preview=True,
@@ -6874,7 +6852,7 @@ class Wiki(object):
             if request.vars.q:
                 tags = [v.strip() for v in request.vars.q.split(',')]
                 tags = [v.lower() for v in tags if v]
-        if tags or not query is None:
+        if tags or query is not None:
             db = self.auth.db
             count = db.wiki_tag.wiki_page.count()
             fields = [db.wiki_page.id, db.wiki_page.slug,

gluon/utils.py

@@ -83,11 +83,9 @@ def compare(a, b):
     """ Compares two strings and not vulnerable to timing attacks """
     if HAVE_COMPARE_DIGEST:
         return hmac.compare_digest(a, b)
-    if len(a) != len(b):
+    result = len(a) ^ len(b)
-        return False
+    for i in xrange(len(b)):
-    result = 0
+		result |= ord(a[i%len(a)]) ^ ord(b[i])
-    for x, y in zip(a, b):
-        result |= ord(x) ^ ord(y)
     return result == 0
 
 

gluon/validators.py

@@ -376,8 +376,8 @@ class IS_JSON(Validator):
     def __call__(self, value):
         try:
             if self.native_json:
-                simplejson.loads(value) # raises error in case of malformed json
+                simplejson.loads(value)  # raises error in case of malformed json
-                return (value, None) #  the serialized value is not passed
+                return (value, None)  # the serialized value is not passed
             else:
                 return (simplejson.loads(value), None)
         except JSONErrors:
@@ -459,7 +459,7 @@ class IS_IN_SET(Validator):
 
     def __call__(self, value):
         if self.multiple:
-            ### if below was values = re.compile("[\w\-:]+").findall(str(value))
+            # if below was values = re.compile("[\w\-:]+").findall(str(value))
             if not value:
                 values = []
             elif isinstance(value, (tuple, list)):
@@ -471,8 +471,6 @@ class IS_IN_SET(Validator):
         thestrset = [str(x) for x in self.theset]
         failures = [x for x in values if not str(x) in thestrset]
         if failures and self.theset:
-            if self.multiple and (value is None or value == ''):
-                return ([], None)
             return (value, translate(self.error_message))
         if self.multiple:
             if isinstance(self.multiple, (tuple, list)) and \
@@ -525,8 +523,8 @@ class IS_IN_DB(Validator):
             field = field._id
         elif isinstance(field, str):
             items = field.split('.')
-            if len(items)==1: items+=['id']
+            if len(items) == 1:
-            field = self.dbset.db[items[0]][items[1]]
+                field = items[0] + '.id'
 
         (ktable, kfield) = str(field).split('.')
         if not label:
@@ -536,16 +534,16 @@ class IS_IN_DB(Validator):
                 label = '%%(%s)s' % str(label).split('.')[-1]
             fieldnames = regex2.findall(label)
             if kfield not in fieldnames:
-                fieldnames.append(kfield) # kfield must be last
+                fieldnames.append(kfield)  # kfield must be last
         elif isinstance(label, Field):
-            fieldnames = [label.name, kfield] # kfield must be last
+            fieldnames = [label.name, kfield]  # kfield must be last
             label = '%%(%s)s' % label.name
         elif callable(label):
             fieldnames = '*'
         else:
             raise NotImplementedError
-        self.field = field # the lookup field
+
-        self.fieldnames = fieldnames # fields requires to build the formatting
+        self.fieldnames = fieldnames  # fields requires to build the formatting
         self.label = label
         self.ktable = ktable
         self.kfield = kfield
@@ -623,16 +621,16 @@ class IS_IN_DB(Validator):
             if isinstance(value, list):
                 values = value
             elif self.delimiter:
-                values = value.split(self.delimiter) # because of autocomplete
+                values = value.split(self.delimiter)  # because of autocomplete
             elif value:
                 values = [value]
             else:
                 values = []
 
-            if self.field.type in ('id','integer'):
+            if field.type in ('id', 'integer'):
                 new_values = []
                 for value in values:
-                    if not (isinstance(value,(int,long)) or value.isdigit()):
+                    if not (isinstance(value, (int, long)) or value.isdigit()):
                         if self.auto_add:
                             value = str(self.maybe_add(table, self.fieldnames[0], value))
                         else:
@@ -647,11 +645,10 @@ class IS_IN_DB(Validator):
                 if not [v for v in values if v not in self.theset]:
                     return (values, None)
             else:
-                from pydal.adapters import GoogleDatastoreAdapter
-
                 def count(values, s=self.dbset, f=field):
                     return s(f.belongs(map(int, values))).count()
-                if GoogleDatastoreAdapter is not None and isinstance(self.dbset.db._adapter, GoogleDatastoreAdapter):
+
+                if self.dbset.db._adapter.dbengine == "google:datastore":
                     range_ids = range(0, len(values), 30)
                     total = sum(count(values[i:i + 30]) for i in range_ids)
                     if total == len(values):
@@ -659,8 +656,8 @@ class IS_IN_DB(Validator):
                 elif count(values) == len(values):
                     return (values, None)
         else:
-            if self.field.type in ('id','integer'):
+            if field.type in ('id', 'integer'):
-                if isinstance(value,(int,long)) or value.isdigit():
+                if isinstance(value, (int, long)) or value.isdigit():
                     value = int(value)
                 elif self.auto_add:
                     value = self.maybe_add(table, self.fieldnames[0], value)
@@ -820,7 +817,7 @@ class IS_INT_IN_RANGE(Validator):
         if regex_isint.match(str(value)):
             v = int(value)
             if ((self.minimum is None or v >= self.minimum) and
-                (self.maximum is None or v < self.maximum)):
+                    (self.maximum is None or v < self.maximum)):
                 return (v, None)
         return (value, self.error_message)
 
@@ -894,7 +891,7 @@ class IS_FLOAT_IN_RANGE(Validator):
             else:
                 v = float(str(value).replace(self.dot, '.'))
             if ((self.minimum is None or v >= self.minimum) and
-                (self.maximum is None or v <= self.maximum)):
+                    (self.maximum is None or v <= self.maximum)):
                 return (v, None)
         except (ValueError, TypeError):
             pass
@@ -980,7 +977,7 @@ class IS_DECIMAL_IN_RANGE(Validator):
             else:
                 v = decimal.Decimal(str(value).replace(self.dot, '.'))
             if ((self.minimum is None or v >= self.minimum) and
-                (self.maximum is None or v <= self.maximum)):
+                    (self.maximum is None or v <= self.maximum)):
                 return (v, None)
         except (ValueError, TypeError, decimal.InvalidOperation):
             pass
@@ -1200,7 +1197,12 @@ class IS_EMAIL(Validator):
         self.error_message = error_message
 
     def __call__(self, value):
-        match = self.regex.match(value)
+        try:
+            match = self.regex.match(value)
+        except TypeError:
+            # Value may not be a string where we can look for matches.
+            # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
+            match = None
         if match:
             domain = value.split('@')[1]
             if (not self.banned or not self.banned.match(domain)) \
@@ -2344,6 +2346,7 @@ class IS_DATE_IN_RANGE(IS_DATE):
             (datetime.date(2010, 3, 3), 'oops')
 
     """
+
     def __init__(self,
                  minimum=None,
                  maximum=None,
@@ -2397,6 +2400,7 @@ class IS_DATETIME_IN_RANGE(IS_DATETIME):
             (datetime.datetime(2010, 3, 3, 0, 0), 'oops')
 
     """
+
     def __init__(self,
                  minimum=None,
                  maximum=None,
@@ -2510,7 +2514,7 @@ def urlify(s, maxlen=80, keep_underscores=False):
     if keep_underscores:
         s = re.sub('\s+', '-', s)         # whitespace to hyphens
         s = re.sub('[^\w\-]', '', s)
-                   # strip all but alphanumeric/underscore/hyphen
+        # strip all but alphanumeric/underscore/hyphen
     else:
         s = re.sub('[\s_]+', '-', s)      # whitespace & underscores to hyphens
         s = re.sub('[^a-z0-9\-]', '', s)  # strip all but alphanumeric/hyphen
@@ -2608,7 +2612,7 @@ class ANY_OF(Validator):
         # Use the formatter of the first subvalidator
         # that validates the value and has a formatter
         for validator in self.subs:
-            if hasattr(validator, 'formatter') and validator(value)[1] != None:
+            if hasattr(validator, 'formatter') and validator(value)[1] is None:
                 return validator.formatter(value)
 
 
@@ -2702,6 +2706,7 @@ class LazyCrypt(object):
     """
     Stores a lazy password hash
     """
+
     def __init__(self, crypt, password):
         """
         crypt is an instance of the CRYPT validator,
@@ -2757,8 +2762,8 @@ class LazyCrypt(object):
         # LazyCrypt objects comparison
         if isinstance(stored_password, self.__class__):
             return ((self is stored_password) or
-                   ((self.crypt.key == stored_password.crypt.key) and
+                    ((self.crypt.key == stored_password.crypt.key) and
-                   (self.password == stored_password.password)))
+                     (self.password == stored_password.password)))
 
         if self.crypt.key:
             if ':' in self.crypt.key:
@@ -3401,14 +3406,14 @@ class IS_IPV4(Validator):
                     ok = True
             if not (self.is_localhost is None or self.is_localhost ==
                     (number == self.localhost)):
-                    ok = False
+                ok = False
             if not (self.is_private is None or self.is_private ==
                     (sum([private_number[0] <= number <= private_number[1]
                           for private_number in self.private]) > 0)):
-                    ok = False
+                ok = False
             if not (self.is_automatic is None or self.is_automatic ==
                     (self.automatic[0] <= number <= self.automatic[1])):
-                    ok = False
+                ok = False
             if ok:
                 return (value, None)
         return (value, translate(self.error_message))
@@ -3692,6 +3697,7 @@ class IS_IPADDRESS(Validator):
         >>> IS_IPADDRESS(subnets='invalidsubnet')('2001::8ffa:fe22:b3af')
         ('2001::8ffa:fe22:b3af', 'invalid subnet provided')
     """
+
     def __init__(
             self,
             minip='0.0.0.0',
@@ -3754,7 +3760,7 @@ class IS_IPADDRESS(Validator):
                 is_private=self.is_private,
                 is_automatic=self.is_automatic,
                 error_message=self.error_message
-                )(value)
+            )(value)
         elif self.is_ipv6 or isinstance(ip, IPv6Address):
             retval = IS_IPV6(
                 is_private=self.is_private,
@@ -3766,7 +3772,7 @@ class IS_IPADDRESS(Validator):
                 is_teredo=self.is_teredo,
                 subnets=self.subnets,
                 error_message=self.error_message
-                )(value)
+            )(value)
         else:
             retval = (value, translate(self.error_message))
 

scripts/lang_update_from_langfile.py

@@ -30,8 +30,15 @@ if __name__ == '__main__':
         dest="source",
         help="Specify language file (ro) where seek for translations"
     )
+    parser.add_argument(
+        '-f', '--force-update',
+        dest="force_update",
+        action="store_true",
+        default=False,
+        help="without it: add new + translate untranslated, if used: in addition update items if translation differs"
+    )
     args = parser.parse_args()
 
-    update_from_langfile(args.target, args.source)
+    update_from_langfile(args.target, args.source, force_update=args.force_update)
 
     print '%s was updated.' % args.target

scripts/setup-web2py-nginx-uwsgi-ubuntu.sh

@@ -212,21 +212,20 @@ if [ "$nopassword" -eq 0 ]
 then
    sudo -u www-data python -c "from gluon.main import save_password; save_password('$PW',443)"
 fi
-systemctl enable emperor.uwsgi.service
+
-systemctl start emperor.uwsgi.service
+/etc/init.d/nginx start
-/etc/init.d/nginx restart
+start uwsgi-emperor
 
 echo <<EOF
-you can reload uwsgi with
+you can stop uwsgi and nginx with
 
-  systemctl restart emperor.uwsgi
+  sudo /etc/init.d/nginx stop
+  sudo stop uwsgi-emperor
  
-and stop it with
+and start it with
 
-  systemctl stop emperor.uwsgi
+  sudo /etc/init.d/nginx start
+  sudo start uwsgi-emperor
 
-to reload web2py only (without restarting uwsgi)
-
-  touch /etc/uwsgi/web2py.ini
 EOF