R-2.14.6

simplified oneall dname handling

.travis.yml

@@ -12,8 +12,22 @@ python:
   - 'pypy'
 
 install:
+  - |
+      if [ "$TRAVIS_PYTHON_VERSION" = "pypy" ]; then
+        export PYENV_ROOT="$HOME/.pyenv"
+        if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
+          pushd "$PYENV_ROOT" && git pull && popd
+        else
+          rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
+        fi
+        export PYPY_VERSION="5.0.1"
+        "$PYENV_ROOT/bin/pyenv" install --skip-existing "pypy-$PYPY_VERSION"
+        virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION"
+        source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate"
+      fi
   - pip install -e .
 
+
 before_script:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install --download-cache $HOME/.pip-cache unittest2; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache coverage; fi;

CHANGELOG

@@ -1,5 +1,12 @@
-## 2.14.1
+## 2.14.6
 
+- Increased test coverage (thanks Richard)
+- Fixed some newly discovered security issues in admin:
+  CSRF vulnerability in admin that allows disabling apps
+  Brute force password attack vulnerability in admin
+  (thanks Narendra and Leonel)  
+
+## 2.14.1-5
 
 - fixed two major security issues that caused the examples app to leak information
 - new Auth(…,host_names=[…]) to prevent host header injection
@@ -45,6 +52,7 @@
     sessiondb = RedisSession(redis_conn=rconn, session_expiry=False)
     session.connect(request, response, db = sessiondb)
 
+Many thanks to Richard and Simone for their work and dedication.
 
 ## 2.13.*
 

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

VERSION

@@ -1 +1 @@
-Version 2.14.2-stable+timestamp.2016.03.24.17.44.22
+Version 2.14.6-stable+timestamp.2016.05.09.19.18.48

applications/admin/controllers/default.py

@@ -74,8 +74,10 @@ def log_progress(app, mode='EDIT', filename=None, progress=0):
 def safe_open(a, b):
     if (DEMO_MODE or is_gae) and ('w' in b or 'a' in b):
         class tmp:
+
             def write(self, data):
                 pass
+
             def close(self):
                 pass
         return tmp()
@@ -119,6 +121,9 @@ def index():
         send = URL('site')
     if session.authorized:
         redirect(send)
+    elif failed_login_count() >= allowed_number_of_attempts:
+        time.sleep(2 ** allowed_number_of_attempts)
+        raise HTTP(403)
     elif request.vars.password:
         if verify_password(request.vars.password[:1024]):
             session.authorized = True
@@ -208,6 +213,7 @@ def site():
     file_or_appurl = 'file' in request.vars or 'appurl' in request.vars
 
     class IS_VALID_APPNAME(object):
+
         def __call__(self, value):
             if not re.compile('^\w+$').match(value):
                 return (value, T('Invalid application name'))
@@ -353,6 +359,7 @@ def pack():
         session.flash = T('internal error: %s', e)
         redirect(URL('site'))
 
+
 def pack_plugin():
     app = get_app()
     if len(request.args) == 2:
@@ -368,7 +375,6 @@ def pack_plugin():
         redirect(URL('plugin', args=request.args))
 
 
-
 def pack_exe(app, base, filenames=None):
     import urllib
     import zipfile
@@ -397,10 +403,20 @@ def pack_exe(app, base, filenames=None):
 def pack_custom():
     app = get_app()
     base = apath(app, r=request)
-    if request.post_vars.file:
 
+    def ignore(fs):
+        return [f for f in fs if not (
+                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
+    files = {}
+    for (r, d, f) in os.walk(base):
+        files[r] = {'folders': ignore(d), 'files': ignore(f)}
+
+    if request.post_vars.file:
+        valid_set = set(os.path.relpath(os.path.join(r, f), base) for r in files for f in files[r]['files'])
         files = request.post_vars.file
         files = [files] if not isinstance(files, list) else files
+        files = [file for file in files if file in valid_set]
+
         if request.post_vars.doexe is None:
             fname = 'web2py.app.%s.w2p' % app
             try:
@@ -417,12 +433,7 @@ def pack_custom():
                 redirect(URL(args=request.args))
         else:
             return pack_exe(app, base, files)
-    def ignore(fs):
-        return [f for f in fs if not (
-                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
-    files = {}
-    for (r,d,f) in os.walk(base):
-        files[r] = {'folders':ignore(d),'files':ignore(f)}
+
     return locals()
 
 
@@ -533,8 +544,8 @@ def delete():
         redirect(URL(sender, anchor=request.vars.id2))
     return dict(dialog=dialog, filename=filename)
 
-
 def enable():
+    if not URL.verify(request, hmac_key=session.hmac_key): raise HTTP(401)
     app = get_app()
     filename = os.path.join(apath(app, r=request), 'DISABLED')
     if is_gae:
@@ -546,6 +557,7 @@ def enable():
         safe_open(filename, 'wb').write('disabled: True\ntime-disabled: %s' % request.now)
         return SPAN(T('Enable'), _style='color:red')
 
+
 def peek():
     """ Visualize object code """
     app = get_app(request.vars.app)
@@ -764,8 +776,8 @@ def edit():
             view = request.args[3].replace('.html', '')
             view_link = URL(request.args[0], request.args[2], view)
     elif filetype == 'python' and request.args[1] == 'controllers':
-        ## it's a controller file.
-        ## Create links to all of the associated view files.
+        # it's a controller file.
+        # Create links to all of the associated view files.
         app = get_app()
         viewname = os.path.splitext(request.args[2])[0]
         viewpath = os.path.join(app, 'views', viewname)
@@ -820,12 +832,14 @@ def edit():
         else:
             return response.json(file_details)
 
+
 def todolist():
     """ Returns all TODO of the requested app
     """
     app = request.vars.app or ''
     app_path = apath('%(app)s' % {'app': app}, r=request)
     dirs = ['models', 'controllers', 'modules', 'private']
+
     def listfiles(app, dir, regexp='.*\.py$'):
         files = sorted(listdir(apath('%(app)s/%(dir)s/' % {'app': app, 'dir': dir}, r=request), regexp))
         files = [x.replace(os.path.sep, '/') for x in files if not x.endswith('.bak')]
@@ -850,6 +864,7 @@ def todolist():
 
     return {'todo': output, 'app': app}
 
+
 def editor_sessions():
     config = Config(os.path.join(request.folder, 'settings.cfg'),
                     section='editor_sessions', default_values={})
@@ -866,6 +881,7 @@ def editor_sessions():
 
     return response.render('default/editor_sessions.html', {'editor_sessions': preferences})
 
+
 def resolve():
     """
     """
@@ -1298,6 +1314,7 @@ def plugin():
                 languages=languages,
                 crontab=crontab)
 
+
 def create_file():
     """ Create files handler """
     if request.vars and not request.vars.token == session.token:
@@ -1473,11 +1490,13 @@ def listfiles(app, dir, regexp='.*\.py$'):
     files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')]
     return files
 
+
 def editfile(path, file, vars={}, app=None):
     args = (path, file) if 'app' in vars else (app, path, file)
     url = URL('edit', args=args, vars=vars)
     return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;')
 
+
 def files_menu():
     app = request.vars.app or 'welcome'
     dirs = [{'name': 'models', 'reg': '.*\.py$'},
@@ -1495,6 +1514,7 @@ def files_menu():
                                        _id=dir['name'] + '_files', _style="display: none;")))
     return dict(result_files=result_files)
 
+
 def upload_file():
     """ File uploading handler """
     if request.vars and not request.vars.token == session.token:
@@ -1867,7 +1887,7 @@ def bulk_register():
         redirect(URL('site'))
     return locals()
 
-### Begin experimental stuff need fixes:
+# Begin experimental stuff need fixes:
 # 1) should run in its own process - cannot os.chdir
 # 2) should not prompt user at console
 # 3) should give option to force commit and not reuqire manual merge
@@ -1934,6 +1954,7 @@ def git_push():
             redirect(URL('site'))
     return dict(app=app, form=form)
 
+
 def plugins():
     app = request.args(0)
     from serializers import loads_json
@@ -1948,12 +1969,16 @@ def plugins():
             session.plugins = []
     return dict(plugins=session.plugins["results"], app=request.args(0))
 
+
 def install_plugin():
     app = request.args(0)
     source = request.vars.source
     plugin = request.vars.plugin
     if not (source and app):
         raise HTTP(500, T("Invalid request"))
+    # make sure no XSS attacks in source
+    if not source.lower().split('://')[0] in ('http','https'):
+        raise HTTP(500, T("Invalid request"))
     form = SQLFORM.factory()
     result = None
     if form.process().accepted:

applications/admin/models/access.py

@@ -4,6 +4,7 @@ import time
 from gluon import portalocker
 from gluon.admin import apath
 from gluon.fileutils import read_file
+from gluon.utils import web2py_uuid
 # ###########################################################
 # ## make sure administrator is on localhost or https
 # ###########################################################
@@ -49,15 +50,18 @@ except IOError:
 def verify_password(password):
     session.pam_user = None
     if DEMO_MODE:
-        return True
+        ret = True
     elif not _config.get('password'):
-        return False
+        ret - False
     elif _config['password'].startswith('pam_user:'):
         session.pam_user = _config['password'][9:].strip()
         import gluon.contrib.pam
-        return gluon.contrib.pam.authenticate(session.pam_user, password)
+        ret = gluon.contrib.pam.authenticate(session.pam_user, password)
     else:
-        return _config['password'] == CRYPT()(password)[0]
+        ret = _config['password'] == CRYPT()(password)[0]
+    if ret:
+        session.hmac_key = web2py_uuid()
+    return ret
 
 
 # ###########################################################
@@ -100,13 +104,12 @@ def write_hosts_deny(denied_hosts):
     portalocker.unlock(f)
     f.close()
 
-
 def login_record(success=True):
     denied_hosts = read_hosts_deny()
     val = (0, 0)
     if success and request.client in denied_hosts:
         del denied_hosts[request.client]
-    elif not success and not request.is_local:
+    elif not success:
         val = denied_hosts.get(request.client, (0, 0))
         if time.time() - val[1] < expiration_failed_logins \
             and val[0] >= allowed_number_of_attempts:
@@ -117,6 +120,11 @@ def login_record(success=True):
     write_hosts_deny(denied_hosts)
     return val[0]
 
+def failed_login_count():
+    denied_hosts = read_hosts_deny()
+    val = denied_hosts.get(request.client, (0, 0))
+    return val[0]
+
 
 # ###########################################################
 # ## session expiration

applications/admin/views/default/site.html

@@ -56,7 +56,7 @@
         {{pass}}
           </ul>
         </div>
-        {{=button_enable(URL('enable',args=a), a) if a!='admin' else ''}}
+        {{=button_enable(URL('enable',args=a, hmac_key=session.hmac_key), a) if a!='admin' else ''}}
       </td>
     </tr>
     {{pass}}

applications/examples/controllers/default.py

@@ -19,8 +19,7 @@ def index():
 def what():
     import urllib
     try:
-        images = XML(urllib.urlopen(
-            'http://www.web2py.com/poweredby/default/images').read())
+        images = XML(urllib.urlopen('http://www.web2py.com/poweredby/default/images').read())
     except:
         images = []
     return response.render(images=images)
@@ -74,14 +73,15 @@ def license():
     filename = os.path.join(request.env.gluon_parent, 'LICENSE')
     return response.render(dict(license=MARKMIN(read_file(filename))))
 
+
 def version():
     if request.args(0) == 'raw':
         return request.env.web2py_version
     from gluon.fileutils import parse_version
     (a, b, c, pre_release, build) = parse_version(request.env.web2py_version)
-    return 'Version %i.%i.%i (%.4i-%.2i-%.2i %.2i:%.2i:%.2i) %s' % (
-        a,b,c,build.year,build.month,build.day,
-        build.hour,build.minute,build.second,pre_release)
+    return 'Version %i.%i.%i (%.4i-%.2i-%.2i %.2i:%.2i:%.2i) %s' % \
+           (a, b, c, build.year, build.month, build.day, build.hour, build.minute, build.second, pre_release)
+
 
 @cache.action(time_expire=300, cache_model=cache.ram, quick='P')
 def examples():

applications/examples/models/menu.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 
 response.menu = [
-    (T('Home'), False, URL('default', 'index')),
-    (T('About'), False, URL('default', 'what')),
-    (T('Download'), False, URL('default', 'download')),
-    (T('Docs & Resources'), False, URL('default', 'documentation')),
-    (T('Support'), False, URL('default', 'support')),
-    (T('Contributors'), False, URL('default', 'who'))]
+    (T('Home'), request.controller == 'default' and request.function == 'index', URL('default', 'index')),
+    (T('About'), request.controller == 'default' and request.function == 'what', URL('default', 'what')),
+    (T('Download'), request.controller == 'default' and request.function == 'download', URL('default', 'download')),
+    (T('Docs & Resources'), request.controller == 'default' and request.function == 'documentation', URL('default', 'documentation')),
+    (T('Support'), request.controller == 'default' and request.function == 'support', URL('default', 'support')),
+    (T('Contributors'), request.controller == 'default' and request.function == 'who', URL('default', 'who'))]
 
 #########################################################################
 ## Changes the menu active item

applications/examples/static/css/examples.css

@@ -0,0 +1,69 @@
+/* Gray the black as suggested by Anthony */
+h1,h2,h3,h4,h5,h6 {color: rgb(35, 35, 35); text-transform:none}
+.black {
+    color: rgb(35, 35, 35);
+    background-color: rgb(35, 35, 35);
+}
+
+/* Spacing between thead and tbody */
+/* Ref: http://stackoverflow.com/questions/9258754/spacing-between-thead-and-tbody */
+tbody:before {
+    content: "-";
+    display: block;
+    line-height: 1em;
+    color: transparent;
+}
+
+/* Improve buttons in download page */
+th, td {padding: 0}
+
+
+tbody tr:hover {background-color:transparent}
+tbody tr {border-bottom: none}
+p {text-align: left}
+p, li { line-height: 1.6em}
+
+/* Improve CODE() display though padding has no effect as some PRE are hardcoded somewhere can't find it */
+/* padding of 10px should make it... */
+pre {background-color: rgb(35, 35, 35)!important; border-radius:5px; color:white; padding: 10px}
+
+/* Improve buttons in download page */
+a.btn.btn180 {padding:10px; font-size:1.2em; width:200px}
+
+.menu .web2py-menu-active a {
+    color: #26a69a;
+}
+
+.spaced-vertical {
+    margin: 0 0.5em 0.5em 0;
+}
+
+.btn:hover,
+a.noeffect img:hover {
+    transition: scale .5s;
+    transform: scale(1.05);
+}
+
+.btn,
+a.noeffect img {
+    transition: all .2s ease-in-out;
+}
+
+/* Lower saturation of color #26a69a - 20 points lower */
+/* The below change to color #26a69a should come before other color change or they override all buttons background-color */
+/* The color should maybe change at stupid.css level as it herited from there also in stupid.css it would be better
+   to define this color at one place actually color is defined all over the place */
+a {color:#47a69d}
+.btn, button, [type=button], [type=submit] {background-color:#47a69d}
+.progress .determinate {background-color:#47a69d}
+.progress .indeterminate {background-color:#47a69d}
+a:not(.btn):not(.noeffect):hover {color:#47a69d}
+a:not(.btn):not(.noeffect):after {background-color:#47a69d}
+.tags > span {background-color:#47a69d}
+.tags.dismissible > span.off:hover {background-color:#47a69d}
+.aquamarine{background-color:#47a69d}
+
+/* Lower the saturation of 20 points */
+.green {background-color: #58cc65}
+.yellow {background-color: #ffe333}
+.red {background-color: #cc4229}

applications/examples/static/css/stupid.css

@@ -5,8 +5,9 @@
 ************/
 
 /*** basic styles ***/
-*, *:after, *:before {border:0; margin:0; padding:0; -webkit-box-sizing:border-box; -moz-box-sizing:border-box; box-sizing:border-box}
-html, body {max-width: 100vw !important; overflow-x: hidden !important}
+html {box-sizing:border-box;}
+*, *:after, *:before {border:0; margin:0; padding:0; box-sizing:inherit;}
+html, body {max-width: 100vw; overflow-x: hidden}
 body {font-family:"HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif}
 p, li {margin-bottom:0.5em}
 p {text-align:justify}
@@ -14,7 +15,7 @@ label, strong {font-weight:bold}
 ul {list-style-type:none; padding-left:20px}
 a {text-decoration:none; color:#26a69a; white-space:nowrap}
 a:hover {cursor:pointer}
-h1,h2,h3,h4,h5,h6{font-weight:strong; text-transform:uppercase}
+h1,h2,h3,h4,h5,h6{font-weight:bold; text-transform:uppercase}
 h1{font-size:4em; margin:1.0em 0 0.25em 0}
 h2{font-size:2.4em; margin:0.9em 0 0.25em 0}
 h3{font-size:1.8em; margin:0.8em 0 0.25em 0}
@@ -23,15 +24,13 @@ h5{font-size:1.4em; margin:0.6em 0 0.25em 0}
 h6{font-size:1.2em; margin:0.5em 0 0.25em 0}
 table {border-collapse:collapse}
 tbody tr:hover {background-color:#fbf6d9}
-td, th {padding:5px; vertical-align:top; text-align:left; border:0}
 thead tr {background-color:#f1f1f1}
 tbody tr {border-bottom:2px solid #f1f1f1}
-th {font-weight:string; padding:5px; vertical-align:bottom; text-align:left}
+td, th {padding: 5px; text-align: left; vertical-align:top}
 thead th {vertical-align:bottom}
-tbody th {vertical-align:top}
-header, footer {with:100%}
+header, main, footer {display:block; with:100%} /* IE fix */
 
-@media (max-width:599px) {
+@media all and (max-width:599px) {
   h1{font-size:2em}
   h2{font-size:1.8em}
   h3{font-size:1.6em}
@@ -41,20 +40,20 @@ header, footer {with:100%}
 }
 
 /*** buttons ***/
-.btn, button, [type=button], [type=submit] {padding:0.5em 1em !important; margin:0 0.5em 0.5em 0; line-height:2.4em; background-color:#26a69a; color:white}
+.btn, button, [type=button], [type=submit] {padding:0.5em 1em; margin:0 0.5em 0.5em 0; display:inline-block; background-color:#26a69a; color:white}
 .btn:hover, button:hover, [type=button]:hover, [type=submit]:hover {box-shadow:0 0 10px #666; text-decoration:none; cursor:pointer}
-.btn.small, table .btn {padding:0.25em 0.5em !important; font-size:0.8em; line-height:1.5em}
-.btn.large {padding:1em 2em !important; font-size:1.2em; line-height:4em}
+.btn.small, table .btn {padding:0.25em 0.5em; font-size:0.8em}
+.btn.large {padding:1em 2em; font-size:1.2em}
 .btn.oval {border-radius:50%}
 
 /*** helpers ***/
 .rounded {-moz-border-radius:5px; border-radius:5px}
-.padded {padding:10px 20px !important}
-.center {text-align:center !important; margin-left:auto; margin-right:auto}
+.padded {padding:10px 20px}
+.center {text-align:center; margin-left:auto; margin-right:auto}
 .center>div {text-align:left}
 .right {right:0; text-align:right}
-.middle div {vertical-align:middle !important}
-.bottom div {vertical-align:bottom !important}
+.middle div {vertical-align:middle}
+.bottom div {vertical-align:bottom}
 .xscroll {overflow-x:scroll}
 .yscroll {overflow-y:scroll}
 .nowrap {white-space:nowrap; overflow-x:hidden}
@@ -69,16 +68,17 @@ header, footer {with:100%}
 input:not([type]), input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]), [type=file]:before {outline:none; padding:0.5em 1em; margin:0.5px; border-bottom:1px solid #ddd; width:100%}
 textarea {width:100%; border:1px solid #ddd; padding:4px 8px; outline:none; outline:none}
 select {-webkit-appearance:none; outline:none; padding:0.5em 1em; border-radius:0; margin:0.5px; border-bottom:1px solid #ddd; width:100%;background-color:transparent}
-input, textarea, select, button {font-size:12px}
+input, textarea, select, button, .btn {font-size:12px}
 input:not([type]):hover, input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]):hover, select:hover, textarea:hover {background-color:#fbf6d9; transition:background-color 1s ease}
-input:invalid, input.error {background:#cc1f00!important;color:white}
+input:invalid, input.error {background:#cc1f00;color:white}
 
 /*** grid ***/
 .container {margin-right:-20px}
-.container>.quarter, .container>.half, .container>.third, .container>.twothirds, .container>.threequarters, .container>.fill{display:inline-block; padding:5px 20px 5px 0; vertical-align:top}
-.container>.fill {width:100%; margin-right:-20px}
+.container>.quarter, .container>.half, .container>.third, .container>.twothirds, .container>.threequarters {display:inline-block; padding: 0 20px 0 0; vertical-align:top}
+.container>.fill{display: inline-block}
 .container img, .container video {max-width:100%}
-@media (min-width:800px) {
+
+@media all and (min-width:800px) {
   .max900 {max-width:900px; margin-left:auto; margin-right:auto}
   .quarter {width:25%; margin-right:-5px}
   .half {width:50%; margin-right:-10px}
@@ -86,7 +86,7 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
   .twothirds {width:66.66%; margin-right:-13.33px}
   .threequarters {width:75%; margin-right:-15px}
 }
-@media (min-width:600px) and (max-width:799px) {
+@media all and (min-width:600px) and (max-width:799px) {
   .quarter.compressible {width:25%; margin-right:-5px}
   .half.compressible {width:50%; margin-right:-10px}
   .threequarters.compressible {width:75%; margin-right:-15px}
@@ -95,7 +95,7 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
   .twothirds {width:66.66%; margin-right:-13.33px}
   label.quarter:not(.compressible).right, label.half:not(.compressible).right, label.threequarters:not(.compressible).right   {float:left; text-align:left}
 }
-@media (max-width:599px) {  
+@media all and (max-width:599px) {  
   .quarter:not(.compressible), .half:not(.compressible), .third:not(.compressible), .twothirds:not(.compressible), .threequarters:not(.compressible) {width:100%;}
   label.quarter:not(.compressible).right, label.half:not(.compressible).right, label.threequarters:not(.compressible).right,
   label.third:not(.compressible).right, label.twothirds:not(.compressible).right   {float:left; text-align:left}
@@ -115,7 +115,7 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
   display:block;
   width:120%;
   background-color:#acece6;
-  border-radius:0 !important;
+  border-radius:0;
   background-clip:padding-box;
   overflow:hidden;
 }
@@ -193,17 +193,17 @@ input:invalid, input.error {background:#cc1f00!important;color:white}
 .menu ul ul {top:0; left:80%; z-index:1100}
 .menu li:hover > ul {visibility:visible; opacity:1}
 .menu>li>ul>li:first-child:before{content:''; position:absolute; width:1px; height:1px; border:10px solid transparent; left:50px; top:-20px; margin-left:-10px; border-bottom-color:white}
-.menu.dark ul {background:black; border:1px solid black}
+.menu.dark ul {background:#111111; border:1px solid #111111}
 .menu.dark ul a {color:white}
-.menu.dark>li>ul>li:first-child:before{border-bottom-color:black}
+.menu.dark>li>ul>li:first-child:before{border-bottom-color:#111111}
 
-@media (max-width:599px) {
+@media all and (max-width:599px) {
   header .menu li, header .menu ul {width: 100%}  
   header .menu.right {float:left; text-align:left}
   header .menu ul ul {top:2.5em; left:-1px}
 }
 
-@media (min-width:600px) {
+@media all and (min-width:600px) {
   .ham {display:none!important}
   .burger.accordion * {max-height:1000px; overflow:visible}
 }
@@ -268,8 +268,8 @@ a:not(.btn):not(.noeffect):after {
 [data-tooltip]:before, [data-tooltip]:after {display:none; position:absolute; top:0}
 [data-tooltip]:hover:after,[data-tooltip]:hover:before {display:block}
 [data-tooltip]:hover:before {
-  border-bottom:.6em solid black;
-  border-bottom:.6em solid black;
+  border-bottom:.6em solid #111111;
+  border-bottom:.6em solid #111111;
   border-left:7px solid transparent;
   border-right:7px solid transparent;
   content:"";
@@ -288,7 +288,7 @@ a:not(.btn):not(.noeffect):after {
   content:attr(data-tooltip);
   left:0;
   top:2px;
-  margin-left:-20;
+  margin-left:-20px;
   margin-top:1.5em;
   padding:5px 15px;
   white-space:pre-wrap;
@@ -339,9 +339,6 @@ a:not(.btn):not(.noeffect):after {
   transform: rotateY( 180deg );
 }
 
-/*** colors from http://clrs.cc/ ***/
-.navy{background-color:#001f3f!important;color:white}.blue{background-color:#0074d9!important;color:white}.aqua{background-color:#7fdbff!important;color:black}.teal{background-color:#39cccc!important;color:white}.olive{background-color:#3d9970!important;color:white}.green{background-color:#2ecc40!important;color:white}.aquamarine{background-color:#26a69a!important;color:white}.lime{background-color:#01ff70!important;color:black}.yellow{background-color:#ffdc00!important;color:black}.orange{background-color:#ff851b!important;color:white}.red{background-color:#cc1f00!important;color:white}.fuchsia{background-color:#f012be!important;color:white}.pink{background-color:#ee6e73!important;color:white}.purple{background-color:#b10dc9!important;color:white}.maroon{background-color:#85144b!important;color:white}.white{background-color:#fff!important;color:black;-webkit-box-shadow:inset 0px 0px 0px 1px #ddd;-moz-box-shadow:inset 0px 0px 0px 1px #ddd;box-shadow:inset 0px 0px 0px 1px #ddd}.gray{background-color:#aaa!important;color:white}.silver{background-color:#f1f1f1!important;color:black}.black{background-color:#000!important;color:white}.glass{background:rgba(255,255,255,0.5)!important;color:black}
-
 /**** tags ****/
 .tags > span {
   padding: 4px 9px;
@@ -350,10 +347,13 @@ a:not(.btn):not(.noeffect):after {
   background-color: #26a69a;
   border-radius: 5px;
   font-size:12px;
-  margin: 5px 5px 5px 0 !important;
-  line-height: 32px;
+  margin: 2px 5px 2px 0;
+  display: inline-block;
 }
 .tags.dismissible > span:hover {opacity: 0.5}
 .tags.dismissible > span:not(.off):after {content:" ✕"}
 .tags > span.off {background-color: #ccc}
 .tags.dismissible > span.off:hover {background-color:#26a69a}
+
+/*** colors from http://clrs.cc/ ***/
+.navy{background-color:#001f3f;color:white}.blue{background-color:#0074d9;color:white}.aqua{background-color:#7fdbff;color:#111111}.teal{background-color:#39cccc;color:white}.olive{background-color:#3d9970;color:white}.green{background-color:#2ecc40;color:white}.aquamarine{background-color:#26a69a;color:white}.lime{background-color:#01ff70;color:#111111}.yellow{background-color:#ffdc00;color:#111111}.orange{background-color:#ff851b;color:white}.red{background-color:#cc1f00;color:white}.fuchsia{background-color:#f012be;color:white}.pink{background-color:#ee6e73;color:white}.purple{background-color:#b10dc9;color:white}.maroon{background-color:#85144b;color:white}.white{background-color:#fff;color:#111111;-webkit-box-shadow:inset 0px 0px 0px 1px #ddd;-moz-box-shadow:inset 0px 0px 0px 1px #ddd;box-shadow:inset 0px 0px 0px 1px #ddd}.gray{background-color:#aaa;color:white}.silver{background-color:#f1f1f1;color:#111111}.black{background-color:#111111;color:white}.glass{background:rgba(255,255,255,0.5);color:#111111}

applications/examples/views/default/download.html

@@ -17,7 +17,7 @@
     <tbody>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="http://www.web2py.com/examples/static/web2py_win.zip">For Windows</a>
+          <a class="btn btn180 rounded green" href="http://www.web2py.com/examples/static/web2py_win.zip">For Windows</a>
         </td>
         <td>
           <a class="btn btn180 rounded yellow" href="http://www.web2py.com/examples/static/nightly/web2py_win.zip">For Windows</a>
@@ -28,7 +28,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="http://www.web2py.com/examples/static/web2py_osx.zip">For Mac</a>
+          <a class="btn btn180 rounded green" href="http://www.web2py.com/examples/static/web2py_osx.zip">For Mac</a>
         </td>
         <td>
           <a class="btn btn180 rounded yellow" href="http://www.web2py.com/examples/static/nightly/web2py_osx.zip">For Mac</a>
@@ -37,7 +37,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="http://www.web2py.com/examples/static/web2py_src.zip">Source Code</a>
+          <a class="btn btn180 rounded green" href="http://www.web2py.com/examples/static/web2py_src.zip">Source Code</a>
         </td>
         <td>
           <a class="btn btn180 rounded yellow" href="http://www.web2py.com/examples/static/nightly/web2py_src.zip">Source Code</a>
@@ -48,7 +48,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded red" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
+          <a class="btn btn180 rounded green" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
         </td>
         <td>
           <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>
@@ -84,7 +84,7 @@
 <p>Applications built with web2py can be released under any license the author wishes as long they do not contain web2py code. They can link unmodified web2py libraries and they can be distributed with official web2py binaries. In particular web2py applications can be distributed in closed source. The admin interface provides a button to byte-code compile.</p>
 <p>It is fine to distribute web2py (source or compiled) with your applications as long as you make it clear in the license where your application ends and web2py starts.</p>
 <p>web2py is copyrighted by Massimo Di Pierro. The web2py trademark is owned by Massimo Di Pierro.</p>
-<a class="btn btn-small" href="{{=URL('license')}}">read more</a>
+<a class="btn btn-small rounded" href="{{=URL('license')}}">read more</a>
 
 <h3>Artwork</h3>
 <center>

applications/examples/views/default/index.html

@@ -3,22 +3,22 @@
 <div class="container">
   <div class="twothirds">
     <div class="padded">
-      <h3>web2py<sup>TM</sup> Web Framework</h3>
+      <h3><img src="{{=URL('static/images', 'web2py_logo.png')}}"> Web Framework</h3>
       <p>Free open source full-stack framework for rapid development of fast, scalable, <a href="http://www.web2py.com/book/default/chapter/01#Security" target="_blank">secure</a> and portable database-driven web-based applications. Written and programmable in <a href="http://www.python.org" target="_blank">Python</a>.</p>
       <table width="100%">
 	<tr>
 	  <td>
-	    <a href="http://web2py.com/book">
+	    <a class="noeffect" href="http://web2py.com/book">
 	      <img src="{{=URL('static','images/book-5th.png')}}" />
 	    </a>
 	  </td>
 	  <td>
-	    <a href="https://vimeo.com/album/3016728">
+	    <a class="noeffect" href="https://vimeo.com/album/3016728">
 	      <img src="{{=URL('static','images/videos.png')}}" />
 	    </a>
 	  </td>
 	  <td>
-	    <a href="http://link.packtpub.com/SUlnrN">
+	    <a class="noeffect" href="http://link.packtpub.com/SUlnrN">
 	      <img src="{{=URL('static','images/book-recipes.png')}}" />
 	    </a>
 	  </td>
@@ -29,8 +29,8 @@
   </div>
   <div class="third">
     <div class="padded center">
-      <a href="http://www.infoworld.com/slideshow/24605/infoworlds-2012-technology-of-the-year-award-winners-183313#slide23">
-        <img src="{{=URL('static','images/infoworld2012.jpeg')}}">
+      <a class="noeffect" href="http://www.infoworld.com/slideshow/24605/infoworlds-2012-technology-of-the-year-award-winners-183313#slide23">
+        <img class="spaced-vertical" src="{{=URL('static','images/infoworld2012.jpeg')}}">
       </a>
       <a class="btn rounded red fill" href="{{=URL('download')}}">
         Download Now

applications/examples/views/default/support.html

@@ -32,7 +32,6 @@
       <li><a target="_blank" href="http://www.albendas.com">Albendas</a> (Spain)</li>
       <li><a target="_blank" href="http://www.appliedobjects.com">Applied Objects</a> (New Zealand)</li>
       <li><a target="_blank" href="http://www.sistemasagiles.com.ar/">Sistemas Ágiles</a> ("Agile Systems") (Argentina)</li>
-      <li><a target="_blank" href="http://www.definescope.com/en/services/consulting/">DefineScope</a> (Portugal)</li>
       <li><a target="_blank" href="http://www.tasko.it/">Tasko</a> (Italy)</li>
       <li><a target="_blank" href="http://www.geekondemand.it/"> GeekOnDemand</a> (Italy)</li>
       <li><a target="_blank" href="http://stifix.com"> Stifix</a> (Indonesia)</li>

applications/examples/views/default/who.html

@@ -82,6 +82,7 @@
 </li><li>Keith Yang (openid)
 </li><li><a href="http://dev.s-cubism.com/web2py_plugins">Kenji Hosoda</a> (plugins)
 </li><li>Kyle Smith (javascript)
+</li><li><a href="https://github.com/leonelcamara">Leonel Câmara</a>
 </li><li><a href="http://blog.donews.com/limodou/">Limodou</a> (winservice)
 </li><li><a href="https://github.com/lucasdavila">Lucas D'Ávila</a>
 </li><li>Marc Abramowitz (tests and travis continuous integration) 
@@ -99,6 +100,7 @@
 </li><li>Michael Willis (shell)
 </li><li>Michele Comitini (facebook)
 </li><li>Michael Toomim (scheduler)
+</li><li>Narendra Bhati (security)
 </li><li>Nathan Freeze (admin design, IS_STRONG, DAL features, <a href="http://web2pyslices.com">web2pyslices.com</a>)
 </li><li>Niall Sweeny (MSSQL support)
 </li><li>Niccolo Polo (epydoc)

applications/examples/views/layout.html

@@ -1,22 +1,17 @@
+<!DOCTYPE html>
 <html>
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="apple-mobile-web-app-capable" content="yes" />
-    <link href="{{=URL('static','css/stupid.css')}}" rel="stylesheet" type="text/css"/>
     <link href="{{=URL('static','css/calendar.css')}}" rel="stylesheet" type="text/css"/>
     <link href="{{=URL('static','css/web2py.css')}}" rel="stylesheet" type="text/css"/>
+    <link href="{{=URL('static','css/stupid.css')}}" rel="stylesheet" type="text/css"/>
+    <link href="{{=URL('static','css/examples.css')}}" rel="stylesheet" type="text/css"/>
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css">
-    <style>
-      th, td {color: black!important}
-      tbody tr:hover {background-color:transparent}
-      tbody tr {border-bottom: none}
-      p {text-align: left}
-      p, li { line-height: 1.6em}
-      pre {background-color: black!important;border-radius:5px; color:white; padding:10px}
-      a.btn.btn180 {padding:20px; font-size:1.2em; width:200px!important}      
-    </style>
+    <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
+    <link rel="apple-touch-icon" href="{{=URL('static','images/favicon.png')}}">
     {{
     left_sidebar_enabled = globals().get('left_sidebar_enabled', False)
     right_sidebar_enabled = globals().get('right_sidebar_enabled', False)
@@ -29,7 +24,7 @@
     <header class="black padded">      
       <div class="container middle max900">
         <div class="fill middle">                    
-          <label class="ham padded fa fa-bars" for="menu"></label>
+          <label class="ham" for="menu"><i class="fa fa-bars padded"></i></label>
           <div class="burger accordion">
             <input type="checkbox" id="menu"/>
             {{=MENU(response.menu,_class='menu')}}

applications/welcome/controllers/default.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 # this file is released under public domain and you can use without limitations
 
-#########################################################################
-## This is a sample controller
-## - index is the default action of any application
-## - user is required for authentication and authorization
-## - download is for downloading files uploaded in the db (does streaming)
-#########################################################################
+# -------------------------------------------------------------------------
+# This is a sample controller
+# - index is the default action of any application
+# - user is required for authentication and authorization
+# - download is for downloading files uploaded in the db (does streaming)
+# -------------------------------------------------------------------------
+
 
 def index():
     """

applications/welcome/models/db.py

@@ -1,60 +1,84 @@
 # -*- coding: utf-8 -*-
 
-#########################################################################
-## This scaffolding model makes your app work on Google App Engine too
-## File is released under public domain and you can use without limitations
-#########################################################################
+# -------------------------------------------------------------------------
+# This scaffolding model makes your app work on Google App Engine too
+# File is released under public domain and you can use without limitations
+# -------------------------------------------------------------------------
 
 if request.global_settings.web2py_version < "2.14.1":
     raise HTTP(500, "Requires web2py 2.13.3 or newer")
 
-## if SSL/HTTPS is properly configured and you want all HTTP requests to
-## be redirected to HTTPS, uncomment the line below:
+# -------------------------------------------------------------------------
+# if SSL/HTTPS is properly configured and you want all HTTP requests to
+# be redirected to HTTPS, uncomment the line below:
+# -------------------------------------------------------------------------
 # request.requires_https()
 
-## app configuration made easy. Look inside private/appconfig.ini
+# -------------------------------------------------------------------------
+# app configuration made easy. Look inside private/appconfig.ini
+# -------------------------------------------------------------------------
 from gluon.contrib.appconfig import AppConfig
-## once in production, remove reload=True to gain full speed
+
+# -------------------------------------------------------------------------
+# once in production, remove reload=True to gain full speed
+# -------------------------------------------------------------------------
 myconf = AppConfig(reload=True)
 
 if not request.env.web2py_runtime_gae:
-    ## if NOT running on Google App Engine use SQLite or other DB
+    # ---------------------------------------------------------------------
+    # if NOT running on Google App Engine use SQLite or other DB
+    # ---------------------------------------------------------------------
     db = DAL(myconf.get('db.uri'),
              pool_size=myconf.get('db.pool_size'),
              migrate_enabled=myconf.get('db.migrate'),
              check_reserved=['all'])
 else:
-    ## connect to Google BigTable (optional 'google:datastore://namespace')
+    # ---------------------------------------------------------------------
+    # connect to Google BigTable (optional 'google:datastore://namespace')
+    # ---------------------------------------------------------------------
     db = DAL('google:datastore+ndb')
-    ## store sessions and tickets there
+    # ---------------------------------------------------------------------
+    # store sessions and tickets there
+    # ---------------------------------------------------------------------
     session.connect(request, response, db=db)
-    ## or store session in Memcache, Redis, etc.
-    ## from gluon.contrib.memdb import MEMDB
-    ## from google.appengine.api.memcache import Client
-    ## session.connect(request, response, db = MEMDB(Client()))
+    # ---------------------------------------------------------------------
+    # or store session in Memcache, Redis, etc.
+    # from gluon.contrib.memdb import MEMDB
+    # from google.appengine.api.memcache import Client
+    # session.connect(request, response, db = MEMDB(Client()))
+    # ---------------------------------------------------------------------
 
-## by default give a view/generic.extension to all actions from localhost
-## none otherwise. a pattern can be 'controller/function.extension'
+# -------------------------------------------------------------------------
+# by default give a view/generic.extension to all actions from localhost
+# none otherwise. a pattern can be 'controller/function.extension'
+# -------------------------------------------------------------------------
 response.generic_patterns = ['*'] if request.is_local else []
-## choose a style for forms
+# -------------------------------------------------------------------------
+# choose a style for forms
+# -------------------------------------------------------------------------
 response.formstyle = myconf.get('forms.formstyle')  # or 'bootstrap3_stacked' or 'bootstrap2' or other
 response.form_label_separator = myconf.get('forms.separator') or ''
 
-
-## (optional) optimize handling of static files
+# -------------------------------------------------------------------------
+# (optional) optimize handling of static files
+# -------------------------------------------------------------------------
 # response.optimize_css = 'concat,minify,inline'
 # response.optimize_js = 'concat,minify,inline'
-## (optional) static assets folder versioning
+
+# -------------------------------------------------------------------------
+# (optional) static assets folder versioning
+# -------------------------------------------------------------------------
 # response.static_version = '0.0.0'
-#########################################################################
-## Here is sample code if you need for
-## - email capabilities
-## - authentication (registration, login, logout, ... )
-## - authorization (role based authorization)
-## - services (xml, csv, json, xmlrpc, jsonrpc, amf, rss)
-## - old style crud actions
-## (more options discussed in gluon/tools.py)
-#########################################################################
+
+# -------------------------------------------------------------------------
+# Here is sample code if you need for
+# - email capabilities
+# - authentication (registration, login, logout, ... )
+# - authorization (role based authorization)
+# - services (xml, csv, json, xmlrpc, jsonrpc, amf, rss)
+# - old style crud actions
+# (more options discussed in gluon/tools.py)
+# -------------------------------------------------------------------------
 
 from gluon.tools import Auth, Service, PluginManager
 
@@ -63,10 +87,14 @@ auth = Auth(db, host_names=myconf.get('host.names'))
 service = Service()
 plugins = PluginManager()
 
-## create all tables needed by auth if not custom tables
+# -------------------------------------------------------------------------
+# create all tables needed by auth if not custom tables
+# -------------------------------------------------------------------------
 auth.define_tables(username=False, signature=False)
 
-## configure email
+# -------------------------------------------------------------------------
+# configure email
+# -------------------------------------------------------------------------
 mail = auth.settings.mailer
 mail.settings.server = 'logging' if request.is_local else myconf.get('smtp.server')
 mail.settings.sender = myconf.get('smtp.sender')
@@ -74,27 +102,31 @@ mail.settings.login = myconf.get('smtp.login')
 mail.settings.tls = myconf.get('smtp.tls') or False
 mail.settings.ssl = myconf.get('smtp.ssl') or False
 
-## configure auth policy
+# -------------------------------------------------------------------------
+# configure auth policy
+# -------------------------------------------------------------------------
 auth.settings.registration_requires_verification = False
 auth.settings.registration_requires_approval = False
 auth.settings.reset_password_requires_verification = True
 
-#########################################################################
-## Define your tables below (or better in another model file) for example
-##
-## >>> db.define_table('mytable',Field('myfield','string'))
-##
-## Fields can be 'string','text','password','integer','double','boolean'
-##       'date','time','datetime','blob','upload', 'reference TABLENAME'
-## There is an implicit 'id integer autoincrement' field
-## Consult manual for more options, validators, etc.
-##
-## More API examples for controllers:
-##
-## >>> db.mytable.insert(myfield='value')
-## >>> rows=db(db.mytable.myfield=='value').select(db.mytable.ALL)
-## >>> for row in rows: print row.id, row.myfield
-#########################################################################
+# -------------------------------------------------------------------------
+# Define your tables below (or better in another model file) for example
+#
+# >>> db.define_table('mytable', Field('myfield', 'string'))
+#
+# Fields can be 'string','text','password','integer','double','boolean'
+#       'date','time','datetime','blob','upload', 'reference TABLENAME'
+# There is an implicit 'id integer autoincrement' field
+# Consult manual for more options, validators, etc.
+#
+# More API examples for controllers:
+#
+# >>> db.mytable.insert(myfield='value')
+# >>> rows = db(db.mytable.myfield == 'value').select(db.mytable.ALL)
+# >>> for row in rows: print row.id, row.myfield
+# -------------------------------------------------------------------------
 
-## after defining tables, uncomment below to enable auditing
+# -------------------------------------------------------------------------
+# after defining tables, uncomment below to enable auditing
+# -------------------------------------------------------------------------
 # auth.enable_record_versioning(db)

applications/welcome/models/menu.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 # this file is released under public domain and you can use without limitations
 
-#########################################################################
-## Customize your APP title, subtitle and menus here
-#########################################################################
+# ----------------------------------------------------------------------------------------------------------------------
+# Customize your APP title, subtitle and menus here
+# ----------------------------------------------------------------------------------------------------------------------
 
 response.logo = A(B('web', SPAN(2), 'py'), XML('™ '),
                   _class="navbar-brand", _href="http://www.web2py.com/",
@@ -11,18 +11,22 @@ response.logo = A(B('web',SPAN(2),'py'),XML('™ '),
 response.title = request.application.replace('_', ' ').title()
 response.subtitle = ''
 
-## read more at http://dev.w3.org/html5/markup/meta.name.html
+# ----------------------------------------------------------------------------------------------------------------------
+# read more at http://dev.w3.org/html5/markup/meta.name.html
+# ----------------------------------------------------------------------------------------------------------------------
 response.meta.author = myconf.get('app.author')
 response.meta.description = myconf.get('app.description')
 response.meta.keywords = myconf.get('app.keywords')
 response.meta.generator = myconf.get('app.generator')
 
-## your http://google.com/analytics id
+# ----------------------------------------------------------------------------------------------------------------------
+# your http://google.com/analytics id
+# ----------------------------------------------------------------------------------------------------------------------
 response.google_analytics_id = None
 
-#########################################################################
-## this is the main application menu add/remove items as required
-#########################################################################
+# ----------------------------------------------------------------------------------------------------------------------
+# this is the main application menu add/remove items as required
+# ----------------------------------------------------------------------------------------------------------------------
 
 response.menu = [
     (T('Home'), False, URL('default', 'index'), [])
@@ -30,15 +34,20 @@ response.menu = [
 
 DEVELOPMENT_MENU = True
 
-#########################################################################
-## provide shortcuts for development. remove in production
-#########################################################################
+
+# ----------------------------------------------------------------------------------------------------------------------
+# provide shortcuts for development. remove in production
+# ----------------------------------------------------------------------------------------------------------------------
 
 def _():
+    # ------------------------------------------------------------------------------------------------------------------
     # shortcuts
+    # ------------------------------------------------------------------------------------------------------------------
     app = request.application
     ctr = request.controller
+    # ------------------------------------------------------------------------------------------------------------------
     # useful links to internal and external resources
+    # ------------------------------------------------------------------------------------------------------------------
     response.menu += [
         (T('My Sites'), False, URL('admin', 'default', 'site')),
         (T('This App'), False, '#', [
@@ -133,6 +142,10 @@ def _():
              'http://webchat.freenode.net/?channels=web2py'),
         ]),
     ]
-if DEVELOPMENT_MENU: _()
 
-if "auth" in locals(): auth.wikimenu() 
+
+if DEVELOPMENT_MENU:
+    _()
+
+if "auth" in locals():
+    auth.wikimenu()

applications/welcome/routes.example.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+# ----------------------------------------------------------------------------------------------------------------------
 #  This is an app-specific example router
 #
 #  This simple router is used for setting languages from app/languages directory
@@ -8,31 +9,33 @@
 #  a default_language
 #
 # See <web2py-root-dir>/examples/routes.parametric.example.py for parameter's detail
-#-------------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------------------------------------------------
 # To enable this route file you must do the steps:
-#
 # 1. rename <web2py-root-dir>/examples/routes.parametric.example.py to routes.py
-# 2. rename this APP/routes.example.py to APP/routes.py
-#    (where APP - is your application directory)
-# 3. restart web2py (or reload routes in web2py admin interfase)
+# 2. rename this APP/routes.example.py to APP/routes.py (where APP - is your application directory)
+# 3. restart web2py (or reload routes in web2py admin interface)
 #
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
+# ----------------------------------------------------------------------------------------------------------------------
 
 from fileutils import abspath
 from languages import read_possible_languages
 
 possible_languages = read_possible_languages(abspath('applications', app))
-#NOTE! app - is an application based router's parameter with name of an
-#            application. E.g.'welcome'
+# ----------------------------------------------------------------------------------------------------------------------
+# NOTE! app - is an application based router's parameter with name of an application. E.g.'welcome'
+# ----------------------------------------------------------------------------------------------------------------------
 
 routers = {
     app: dict(
         default_language=possible_languages['default'][0],
-        languages = [lang for lang in possible_languages
-                           if lang != 'default']
+        languages=[lang for lang in possible_languages if lang != 'default']
     )
 }
 
-#NOTE! To change language in your application using these rules add this line
-#in one of your models files:
+# ----------------------------------------------------------------------------------------------------------------------
+# NOTE! To change language in your application using these rules add this line in one of your models files:
+# ----------------------------------------------------------------------------------------------------------------------
 #   if request.uri_language: T.force(request.uri_language)

fabfile.py

@@ -115,8 +115,9 @@ def deploy(appname=None, all=False):
     """fab -H username@host deploy:appname,all"""
     appname = appname or os.path.split(os.getcwd())[-1]
     appfolder = applications+'/'+appname
-    if os.path.exists('_update.zip'):
-        os.unlink('_update.zip')
+    zipfile = os.path.join(appfolder, '_update.zip')
+    if os.path.exists(zipfile):
+        os.unlink(zipfile)
 
     backup = mkdir_or_backup(appname)
             

gluon/admin.py

@@ -148,7 +148,7 @@ def app_compile(app, request, skip_failed_views=False):
         failed_views = compile_application(folder, skip_failed_views)
         return failed_views
     except (Exception, RestrictedError):
-        tb = traceback.format_exc(sys.exc_info)
+        tb = traceback.format_exc()
         remove_compiled_application(folder)
         return tb
 
@@ -167,7 +167,7 @@ def app_create(app, request, force=False, key=None, info=False):
             os.mkdir(path)
         except:
             if info:
-                return False, traceback.format_exc(sys.exc_info)
+                return False, traceback.format_exc()
             else:
                 return False
     elif not force:
@@ -197,7 +197,7 @@ def app_create(app, request, force=False, key=None, info=False):
     except:
         rmtree(path)
         if info:
-            return False, traceback.format_exc(sys.exc_info)
+            return False, traceback.format_exc()
         else:
             return False
 

gluon/cache.py

@@ -54,6 +54,7 @@ try:
 except ImportError:
     HAVE_PSUTIL = False
 
+
 def remove_oldest_entries(storage, percentage=90):
     # compute current memory usage (%)
     old_mem = psutil.virtual_memory().percent
@@ -66,7 +67,8 @@ def remove_oldest_entries(storage, percentage=90):
         # comute used memory again
         new_mem = psutil.virtual_memory().percent
         # if the used memory did not decrease stop
-        if new_mem >= old_mem: break
+        if new_mem >= old_mem:
+            break
         # net new measurement for memory usage and loop
         old_mem = new_mem
 
@@ -78,6 +80,7 @@ __all__ = ['Cache', 'lazy_cache']
 
 DEFAULT_TIME_EXPIRE = 300
 
+
 class CacheAbstract(object):
     """
     Abstract class for cache implementations.
@@ -182,13 +185,14 @@ class CacheInRam(CacheAbstract):
         self.request = request
         self.storage = OrderedDict() if HAVE_PSUTIL else {}
         self.app = request.application if request else ''
+
     def initialize(self):
         if self.initialized:
             return
         else:
             self.initialized = True
         self.locker.acquire()
-        if not self.app in self.meta_storage:
+        if self.app not in self.meta_storage:
             self.storage = self.meta_storage[self.app] = \
                 OrderedDict() if HAVE_PSUTIL else {}
             self.stats[self.app] = {'hit_total': 0, 'misses': 0}
@@ -205,7 +209,7 @@ class CacheInRam(CacheAbstract):
         else:
             self._clear(storage, regex)
 
-        if not self.app in self.stats:
+        if self.app not in self.stats:
             self.stats[self.app] = {'hit_total': 0, 'misses': 0}
 
         self.locker.release()
@@ -251,7 +255,7 @@ class CacheInRam(CacheAbstract):
         self.locker.acquire()
         self.storage[key] = (now, value)
         self.stats[self.app]['misses'] += 1
-        if HAVE_PSUTIL and self.max_ram_utilization!=None and random.random()<0.10:
+        if HAVE_PSUTIL and self.max_ram_utilization is not None and random.random() < 0.10:
             remove_oldest_entries(self.storage, percentage=self.max_ram_utilization)
         self.locker.release()
         return value
@@ -292,14 +296,15 @@ class CacheOnDisk(CacheAbstract):
             self.folder = folder
             self.key_filter_in = lambda key: key
             self.key_filter_out = lambda key: key
-            self.file_lock_time_wait = file_lock_time_wait # How long we should wait before retrying to lock a file held by another process
+            self.file_lock_time_wait = file_lock_time_wait
+            # How long we should wait before retrying to lock a file held by another process
             # We still need a mutex for each file as portalocker only blocks other processes
             self.file_locks = defaultdict(thread.allocate_lock)
 
-
             # Make sure we use valid filenames.
             if sys.platform == "win32":
                 import base64
+
                 def key_filter_in_windows(key):
                     """
                     Windows doesn't allow \ / : * ? "< > | in filenames.
@@ -316,7 +321,6 @@ class CacheOnDisk(CacheAbstract):
                 self.key_filter_in = key_filter_in_windows
                 self.key_filter_out = key_filter_out_windows
 
-
         def wait_portalock(self, val_file):
             """
             Wait for the process file lock.
@@ -328,15 +332,12 @@ class CacheOnDisk(CacheAbstract):
                 except:
                     time.sleep(self.file_lock_time_wait)
 
-
         def acquire(self, key):
             self.file_locks[key].acquire()
 
-
         def release(self, key):
             self.file_locks[key].release()
 
-
         def __setitem__(self, key, value):
             key = self.key_filter_in(key)
             val_file = recfile.open(key, mode='wb', path=self.folder)
@@ -344,7 +345,6 @@ class CacheOnDisk(CacheAbstract):
             pickle.dump(value, val_file, pickle.HIGHEST_PROTOCOL)
             val_file.close()
 
-
         def __getitem__(self, key):
             key = self.key_filter_in(key)
             try:
@@ -357,12 +357,10 @@ class CacheOnDisk(CacheAbstract):
             val_file.close()
             return value
 
-
         def __contains__(self, key):
             key = self.key_filter_in(key)
             return (key in self.file_locks) or recfile.exists(key, path=self.folder)
 
-
         def __delitem__(self, key):
             key = self.key_filter_in(key)
             try:
@@ -370,13 +368,11 @@ class CacheOnDisk(CacheAbstract):
             except IOError:
                 raise KeyError
 
-
         def __iter__(self):
             for dirpath, dirnames, filenames in os.walk(self.folder):
                 for filename in filenames:
                     yield self.key_filter_out(filename)
 
-
         def safe_apply(self, key, function, default_value=None):
             """
             Safely apply a function to the value of a key in storage and set
@@ -403,25 +399,21 @@ class CacheOnDisk(CacheAbstract):
             val_file.close()
             return new_value
 
-
         def keys(self):
             return list(self.__iter__())
 
-
         def get(self, key, default=None):
             try:
                 return self[key]
             except KeyError:
                 return default
 
-
     def __init__(self, request=None, folder=None):
         self.initialized = False
         self.request = request
         self.folder = folder
         self.storage = None
 
-
     def initialize(self):
         if self.initialized:
             return
@@ -440,7 +432,6 @@ class CacheOnDisk(CacheAbstract):
 
         self.storage = CacheOnDisk.PersistentStorage(folder)
 
-
     def __call__(self, key, f,
                  time_expire=DEFAULT_TIME_EXPIRE):
         self.initialize()
@@ -487,7 +478,6 @@ class CacheOnDisk(CacheAbstract):
         self.storage.release(key)
         return value
 
-
     def clear(self, regex=None):
         self.initialize()
         storage = self.storage
@@ -504,7 +494,6 @@ class CacheOnDisk(CacheAbstract):
                 pass
             storage.release(key)
 
-
     def increment(self, key, value=1):
         self.initialize()
         self.storage.acquire(key)
@@ -513,7 +502,6 @@ class CacheOnDisk(CacheAbstract):
         return value
 
 
-
 class CacheAction(object):
     def __init__(self, func, key, time_expire, cache, cache_model):
         self.__name__ = func.__name__
@@ -602,6 +590,7 @@ class Cache(object):
         """
         from gluon import current
         from gluon.http import HTTP
+
         def wrap(func):
             def wrapped_f():
                 if current.request.env.request_method != 'GET':
@@ -621,7 +610,8 @@ class Cache(object):
                     cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire)
                     if not session_ and public_:
                         cache_control += ', public'
-                        expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)).strftime('%a, %d %b %Y %H:%M:%S GMT')
+                        expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)
+                                   ).strftime('%a, %d %b %Y %H:%M:%S GMT')
                     else:
                         cache_control += ', private'
                         expires = 'Fri, 01 Jan 1990 00:00:00 GMT'
@@ -675,11 +665,9 @@ class Cache(object):
                     if str(status)[0] in '123':
                         send_headers = True
                 if send_headers:
-                    headers = {
-                        'Pragma' : None,
+                    headers = {'Pragma': None,
                                'Expires': expires,
-                        'Cache-Control' : cache_control
-                        }
+                               'Cache-Control': cache_control}
                     current.response.headers.update(headers)
                 if cache_model and not send_headers:
                     # we cached already the value, but the status is not valid
@@ -740,8 +728,7 @@ class Cache(object):
         allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix')
         it will add prefix to all the cache keys used.
         """
-        return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\
-            cache_model(prefix + key, f, time_expire)
+        return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix: cache_model(prefix + key, f, time_expire)
 
 
 def lazy_cache(key=None, time_expire=None, cache_model='ram'):

gluon/compileapp.py

@@ -676,8 +676,8 @@ def run_view_in(environment):
     else:
         filename = pjoin(folder, 'views', view)
         if os.path.exists(path): # compiled views
-            x = view.replace('/', '.')
-            files = ['views.%s.pyc' % x]
+            x = view.replace('/', '_')
+            files = ['views_%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(path, files[0]))
             # Don't use a generic view if the non-compiled view exists.
             if is_compiled or (not is_compiled and not os.path.exists(filename)):

gluon/contrib/imageutils.py

@@ -27,7 +27,7 @@ from gluon import current
 
 class RESIZE(object):
 
-    def __init__(self, nx=160, ny=80, quality=100, padding = False
+    def __init__(self, nx=160, ny=80, quality=100, padding = False,
                  error_message=' image resize'):
         (self.nx, self.ny, self.quality, self.error_message, self.padding) = (
             nx, ny, quality, error_message, padding)

gluon/contrib/login_methods/oneall_account.py

@@ -51,7 +51,7 @@ class OneallAccount(object):
             reg_id=profile.get('identity_token','')
             username=profile.get('preferredUsername',email)
             first_name=name.get('givenName', dname.split(' ')[0])
-            last_name=profile.get('familyName', dname.split(' ')[1] if(len(dname.split(' ')) > 1) else None)
+            last_name=profile.get('familyName', dname.split(' ')[1] if(dname.count(' ') > 0) else None)
             return dict(registration_id=reg_id,username=username,email=email,
                         first_name=first_name,last_name=last_name)
         self.mappings.default = defaultmapping

gluon/contrib/markdown/markdown2.py

@@ -53,8 +53,9 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * header-ids: Adds "id" attributes to headers. The id value is a slug of
   the header text.
 * html-classes: Takes a dict mapping html tag names (lowercase) to a
-  string to use for a "class" tag attribute. Currently only supports
-  "pre" and "code" tags. Add an issue if you require this for other tags.
+  string to use for a "class" tag attribute. Currently only supports "img",
+  "table", "pre" and "code" tags. Add an issue if you require this for other
+  tags.
 * markdown-in-html: Allow the use of `markdown="1"` in a block HTML tag to
   have markdown processing be done on its contents. Similar to
   <http://michelf.com/projects/php-markdown/extra/#markdown-attr> but with
@@ -70,9 +71,14 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * smarty-pants: Replaces ' and " with curly quotation marks or curly
   apostrophes.  Replaces --, ---, ..., and . . . with en dashes, em dashes,
   and ellipses.
+* spoiler: A special kind of blockquote commonly hidden behind a
+  click on SO. Syntax per <http://meta.stackexchange.com/a/72878>.
 * toc: The returned HTML string gets a new "toc_html" attribute which is
   a Table of Contents for the document. (experimental)
 * xml: Passes one-liner processing instructions and namespaced XML tags.
+* tables: Tables using the same format as GFM
+  <https://help.github.com/articles/github-flavored-markdown#tables> and
+  PHP-Markdown Extra <https://michelf.ca/projects/php-markdown/extra/#table>.
 * wiki-tables: Google Code Wiki-style tables. See
   <http://code.google.com/p/support/wiki/WikiSyntax#Tables>.
 """
@@ -82,13 +88,11 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 #   not yet sure if there implications with this. Compare 'pydoc sre'
 #   and 'perldoc perlre'.
 
-__version_info__ = (2, 2, 4)
+__version_info__ = (2, 3, 1)
 __version__ = '.'.join(map(str, __version_info__))
 __author__ = "Trent Mick"
 
-import os
 import sys
-from pprint import pprint
 import re
 import logging
 try:
@@ -102,13 +106,7 @@ import codecs
 
 #---- Python version compat
 
-try:
-    from urllib.parse import quote # python3
-except ImportError:
-    from urllib import quote # python2
-
 if sys.version_info[:2] < (2,4):
-    from sets import Set as set
     def reversed(sequence):
         for i in sequence[::-1]:
             yield i
@@ -804,6 +802,8 @@ class Markdown(object):
             text = self._prepare_pyshell_blocks(text)
         if "wiki-tables" in self.extras:
             text = self._do_wiki_tables(text)
+        if "tables" in self.extras:
+            text = self._do_tables(text)
 
         text = self._do_code_blocks(text)
 
@@ -844,6 +844,79 @@ class Markdown(object):
 
         return _pyshell_block_re.sub(self._pyshell_block_sub, text)
 
+    def _table_sub(self, match):
+        trim_space_re = '^[ \t\n]+|[ \t\n]+$'
+        trim_bar_re = '^\||\|$'
+
+        head, underline, body = match.groups()
+
+        # Determine aligns for columns.
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", underline)).split('|')]
+        align_from_col_idx = {}
+        for col_idx, col in enumerate(cols):
+            if col[0] == ':' and col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="center"'
+            elif col[0] == ':':
+                align_from_col_idx[col_idx] = ' align="left"'
+            elif col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="right"'
+
+        # thead
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<thead>', '<tr>']
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", head)).split('|')]
+        for col_idx, col in enumerate(cols):
+            hlines.append('  <th%s>%s</th>' % (
+                align_from_col_idx.get(col_idx, ''),
+                self._run_span_gamut(col)
+            ))
+        hlines.append('</tr>')
+        hlines.append('</thead>')
+
+        # tbody
+        hlines.append('<tbody>')
+        for line in body.strip('\n').split('\n'):
+            hlines.append('<tr>')
+            cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", line)).split('|')]
+            for col_idx, col in enumerate(cols):
+                hlines.append('  <td%s>%s</td>' % (
+                    align_from_col_idx.get(col_idx, ''),
+                    self._run_span_gamut(col)
+                ))
+            hlines.append('</tr>')
+        hlines.append('</tbody>')
+        hlines.append('</table>')
+
+        return '\n'.join(hlines) + '\n'
+
+    def _do_tables(self, text):
+        """Copying PHP-Markdown and GFM table syntax. Some regex borrowed from
+        https://github.com/michelf/php-markdown/blob/lib/Michelf/Markdown.php#L2538
+        """
+        less_than_tab = self.tab_width - 1
+        table_re = re.compile(r'''
+                (?:(?<=\n\n)|\A\n?)             # leading blank line
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (.*[|].*)  \n                   # $1: header row (at least one pipe)
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (                               # $2: underline row
+                    # underline row with leading bar
+                    (?:  \|\ *:?-+:?\ *  )+  \|?  \n
+                    |
+                    # or, underline row without leading bar
+                    (?:  \ *:?-+:?\ *\|  )+  (?:  \ *:?-+:?\ *  )?  \n
+                )
+
+                (                               # $3: data rows
+                    (?:
+                        ^[ ]{0,%d}(?!\ )         # ensure line begins with 0 to less_than_tab spaces
+                        .*\|.*  \n
+                    )+
+                )
+            ''' % (less_than_tab, less_than_tab, less_than_tab), re.M | re.X)
+        return table_re.sub(self._table_sub, text)
+
     def _wiki_table_sub(self, match):
         ttext = match.group(0).strip()
         #print 'wiki table: %r' % match.group(0)
@@ -853,7 +926,7 @@ class Markdown(object):
             row = [c.strip() for c in re.split(r'(?<!\\)\|\|', line)]
             rows.append(row)
         #pprint(rows)
-        hlines = ['<table>', '<tbody>']
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<tbody>']
         for row in rows:
             hrow = ['<tr>']
             for cell in row:
@@ -899,6 +972,9 @@ class Markdown(object):
 
         text = self._encode_amps_and_angles(text)
 
+        if "strike" in self.extras:
+            text = self._do_strike(text)
+
         text = self._do_italics_and_bold(text)
 
         if "smarty-pants" in self.extras:
@@ -1206,7 +1282,6 @@ class Markdown(object):
                                  .replace('_', self._escape_table['_'])
                         title = self.titles.get(link_id)
                         if title:
-                            before = title
                             title = _xml_escape_attr(title) \
                                 .replace('*', self._escape_table['*']) \
                                 .replace('_', self._escape_table['_'])
@@ -1418,7 +1493,6 @@ class Markdown(object):
     def _list_item_sub(self, match):
         item = match.group(4)
         leading_line = match.group(1)
-        leading_space = match.group(2)
         if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
             item = self._run_block_gamut(self._outdent(item))
         else:
@@ -1654,6 +1728,11 @@ class Markdown(object):
         self._escape_table[text] = hashed
         return hashed
 
+    _strike_re = re.compile(r"~~(?=\S)(.+?)(?<=\S)~~", re.S)
+    def _do_strike(self, text):
+        text = self._strike_re.sub(r"<strike>\1</strike>", text)
+        return text
+
     _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
     _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
     _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
@@ -1714,37 +1793,52 @@ class Markdown(object):
         text = text.replace(". . .", "&#8230;")
         return text
 
-    _block_quote_re = re.compile(r'''
+    _block_quote_base = r'''
         (                           # Wrap whole match in \1
           (
-            ^[ \t]*>[ \t]?          # '>' at the start of a line
+            ^[ \t]*>%s[ \t]?        # '>' at the start of a line
               .+\n                  # rest of the first line
             (.+\n)*                 # subsequent consecutive lines
             \n*                     # blanks
           )+
         )
-        ''', re.M | re.X)
+    '''
+    _block_quote_re = re.compile(_block_quote_base % '', re.M | re.X)
+    _block_quote_re_spoiler = re.compile(_block_quote_base % '[ \t]*?!?', re.M | re.X)
     _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M);
-
+    _bq_one_level_re_spoiler = re.compile('^[ \t]*>[ \t]*?![ \t]?', re.M);
+    _bq_all_lines_spoilers = re.compile(r'\A(?:^[ \t]*>[ \t]*?!.*[\n\r]*)+\Z', re.M)
     _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
     def _dedent_two_spaces_sub(self, match):
         return re.sub(r'(?m)^  ', '', match.group(1))
 
     def _block_quote_sub(self, match):
         bq = match.group(1)
-        bq = self._bq_one_level_re.sub('', bq)  # trim one level of quoting
-        bq = self._ws_only_line_re.sub('', bq)  # trim whitespace-only lines
+        is_spoiler = 'spoiler' in self.extras and self._bq_all_lines_spoilers.match(bq)
+        # trim one level of quoting
+        if is_spoiler:
+            bq = self._bq_one_level_re_spoiler.sub('', bq)
+        else:
+            bq = self._bq_one_level_re.sub('', bq)
+        # trim whitespace-only lines
+        bq = self._ws_only_line_re.sub('', bq)
         bq = self._run_block_gamut(bq)          # recurse
 
         bq = re.sub('(?m)^', '  ', bq)
         # These leading spaces screw with <pre> content, so we need to fix that:
         bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
 
-        return "<blockquote>\n%s\n</blockquote>\n\n" % bq
+        if is_spoiler:
+            return '<blockquote class="spoiler">\n%s\n</blockquote>\n\n' % bq
+        else:
+            return '<blockquote>\n%s\n</blockquote>\n\n' % bq
 
     def _do_block_quotes(self, text):
         if '>' not in text:
             return text
+        if 'spoiler' in self.extras:
+            return self._block_quote_re_spoiler.sub(self._block_quote_sub, text)
+        else:
             return self._block_quote_re.sub(self._block_quote_sub, text)
 
     def _form_paragraphs(self, text):
@@ -2053,7 +2147,6 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False):
     if DEBUG:
         print("dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
               % (tabsize, skip_first_line))
-    indents = []
     margin = None
     for i, line in enumerate(lines):
         if i == 0 and skip_first_line: continue

gluon/contrib/markmin/markmin2html.py

@@ -7,6 +7,7 @@ import re
 import urllib
 from cgi import escape
 from string import maketrans
+
 try:
     from ast import parse as ast_parse
     import ast
@@ -542,7 +543,9 @@ regex_URL=re.compile(r'@/(?P<a>\w*)/(?P<c>\w*)/(?P<f>\w*(\.\w+)?)(/(?P<args>[\w\
 regex_env2 = re.compile(r'@\{(?P<a>[\w\-\.]+?)(\:(?P<b>.*?))?\}')
 regex_expand_meta = re.compile('(' + META + '|' + DISABLED_META + '|````)')
 regex_dd = re.compile(r'\$\$(?P<latex>.*?)\$\$')
-regex_code = re.compile('('+META+'|'+DISABLED_META+r'|````)|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',re.S)
+regex_code = re.compile(
+    '(' + META + '|' + DISABLED_META + r'|````)|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',
+    re.S)
 regex_strong = re.compile(r'\*\*(?P<t>[^\s*]+( +[^\s*]+)*)\*\*')
 regex_del = re.compile(r'~~(?P<t>[^\s*]+( +[^\s*]+)*)~~')
 regex_em = re.compile(r"''(?P<t>([^\s']| |'(?!'))+)''")
@@ -554,7 +557,9 @@ regex_proto = re.compile(r'(?<!["\w>/=])(?P<p>\w+):(?P<k>\w+://[\w\d\-+=?%&/:.]+
 regex_auto = re.compile(r'(?<!["\w>/=])(?P<k>\w+://[\w\d\-+_=?%&/:.,;#]+\w|[\w\-.]+@[\w\-.]+)', re.M)
 regex_link = re.compile(r'(' + LINK + r')|\[\[(?P<s>.+?)\]\]', re.S)
 regex_link_level2 = re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?(?:\s+(?P<p>popup))?\s*$', re.S)
-regex_media_level2=re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?\s+(?P<p>img|IMG|left|right|center|video|audio|blockleft|blockright)(?:\s+(?P<w>\d+px))?\s*$',re.S)
+regex_media_level2 = re.compile(
+    r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?\s+(?P<p>img|IMG|left|right|center|video|audio|blockleft|blockright)(?:\s+(?P<w>\d+px))?\s*$',
+    re.S)
 
 regex_markmin_escape = re.compile(r"(\\*)(['`:*~\\[\]{}@\$+\-.#\n])")
 regex_backslash = re.compile(r"\\(['`:*~\\[\]{}@\$+\-.#\n])")
@@ -562,9 +567,11 @@ ttab_in  = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x1
 ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')
 
+
 def make_dict(b):
     return '{%s}' % regex_quote.sub("'\g<name>':", b)
 
+
 def safe_eval(node_or_string, env):
     """
     Safely evaluate an expression node or a string containing a Python
@@ -578,6 +585,7 @@ def safe_eval(node_or_string, env):
         node_or_string = ast_parse(node_or_string, mode='eval')
     if isinstance(node_or_string, ast.Expression):
         node_or_string = node_or_string.body
+
     def _convert(node):
         if isinstance(node, ast.Str):
             return node.s
@@ -606,24 +614,30 @@ def safe_eval(node_or_string, env):
             else:
                 return left - right
         raise ValueError('malformed string')
+
     return _convert(node_or_string)
 
+
 def markmin_escape(text):
     """ insert \\ before markmin control characters: '`:*~[]{}@$ """
     return regex_markmin_escape.sub(
         lambda m: '\\' + m.group(0).replace('\\', '\\\\'), text)
 
+
 def replace_autolinks(text, autolinks):
     return regex_auto.sub(lambda m: autolinks(m.group('k')), text)
 
+
 def replace_at_urls(text, url):
     # this is experimental @{function/args}
     def u1(match, url=url):
         a, c, f, args = match.group('a', 'c', 'f', 'args')
         return url(a=a or None, c=c or None, f=f or None,
                    args=(args or '').split('/'), scheme=True, host=True)
+
     return regex_URL.sub(u1, text)
 
+
 def replace_components(text, env):
     # not perfect but acceptable
     def u2(match, env=env):
@@ -639,16 +653,18 @@ def replace_components(text,env):
             except Exception, e:
                 f = 'ERROR: %s' % e
             return str(f)
+
     text = regex_env2.sub(u2, text)
     return text
 
+
 def autolinks_simple(url):
     """
     it automatically converts the url to link,
     image, video or audio tag
     """
     u_url = url.lower()
-    if '@' in url and not '://' in url:
+    if '@' in url and '://' not in url:
         return '<a href="mailto:%s">%s</a>' % (url, url)
     elif u_url.endswith(('.jpg', '.jpeg', '.gif', '.png')):
         return '<img src="%s" controls />' % url
@@ -658,6 +674,7 @@ def autolinks_simple(url):
         return '<audio src="%s" controls></audio>' % url
     return '<a href="%s">%s</a>' % (url, url)
 
+
 def protolinks_simple(proto, url):
     """
     it converts url to html-string using appropriate proto-prefix:
@@ -675,9 +692,11 @@ def protolinks_simple(proto, url):
         return '<img style="width:100px" src="http://chart.apis.google.com/chart?cht=qr&chs=100x100&chl=%s&choe=UTF-8&chld=H" alt="QR Code" title="QR Code" />' % url
     return proto + ':' + url
 
+
 def email_simple(email):
     return '<a href="mailto:%s">%s</a>' % (email, email)
 
+
 def render(text,
            extra={},
            allowed={},
@@ -925,8 +944,10 @@ def render(text,
     >>> render("anchor with name 'NEWLINE': [[NEWLINE [newline] ]]")
     '<p>anchor with name \\'NEWLINE\\': <span class="anchor" id="markmin_NEWLINE">newline</span></p>'
     """
-    if autolinks=="default": autolinks = autolinks_simple
-    if protolinks=="default": protolinks = protolinks_simple
+    if autolinks == "default":
+        autolinks = autolinks_simple
+    if protolinks == "default":
+        protolinks = protolinks_simple
     pp = '\n' if pretty_print else ''
     if isinstance(text, unicode):
         text = text.encode('utf8')
@@ -945,6 +966,7 @@ def render(text,
     # store them into segments they will be treated as code
     #############################################################
     segments = []
+
     def mark_code(m):
         g = m.group(0)
         if g in (META, DISABLED_META):
@@ -956,10 +978,12 @@ def render(text,
         else:
             c = m.group('c') or ''
             p = m.group('p') or ''
-            if 'code' in allowed and not c in allowed['code']: c = ''
+            if 'code' in allowed and c not in allowed['code']:
+                c = ''
             code = m.group('t').replace('!`!', '`')
             segments.append((code, c, p, m.group(0)))
         return META
+
     text = regex_code.sub(mark_code, text)
 
     #############################################################
@@ -967,10 +991,12 @@ def render(text,
     # store them into links they will be treated as link
     #############################################################
     links = []
+
     def mark_link(m):
         links.append(None if m.group() == LINK
                      else m.group('s'))
         return LINK
+
     text = regex_link.sub(mark_link, text)
     text = escape(text)
 
@@ -1264,9 +1290,11 @@ def render(text,
                 ltags = []
                 tlev = []
                 lev = 0
-                if br and mtag == 'p': out.append(br)
+                if br and mtag == 'p':
+                    out.append(br)
                 if mtag != 'q' and s != META:
-                   if pend: etags=[pend]
+                    if pend:
+                        etags = [pend]
                     out.append(pbeg)
                     mtag = 'p'
                 else:
@@ -1336,7 +1364,7 @@ def render(text,
         t = t or ''
         a = escape(a) if a else ''
         if k:
-            if '#' in k and not ':' in k.split('#')[0]: 
+            if '#' in k and ':' not in k.split('#')[0]:
                 # wikipage, not external url
                 k = k.replace('#', '#' + id_prefix)
             k = escape(k)
@@ -1358,7 +1386,7 @@ def render(text,
     parts = text.split(LINK)
     text = parts[0]
     for i, s in enumerate(links):
-        if s == None:
+        if s is None:
             html = LINK
         else:
             html = regex_media_level2.sub(sub_media, s)
@@ -1374,19 +1402,20 @@ def render(text,
     #############################################################
     def expand_meta(m):
         code, b, p, s = segments.pop(0)
-        if code==None or m.group() == DISABLED_META:
+        if code is None or m.group() == DISABLED_META:
             return escape(s)
         if b in extra:
-            if code[:1]=='\n': code=code[1:]
-            if code[-1:]=='\n': code=code[:-1]
+            if code[:1] == '\n':
+                code = code[1:]
+            if code[-1:] == '\n':
+                code = code[:-1]
             if p:
                 return str(extra[b](code, p))
             else:
                 return str(extra[b](code))
         elif b == 'cite':
-            return '['+','.join('<a href="#%s" class="%s">%s</a>' \
-                                   % (id_prefix+d,b,d) \
-                                   for d in escape(code).split(','))+']'
+            return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
+                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
         elif b == 'latex':
             return LATEX % urllib.quote(code)
         elif b in html_colors:
@@ -1407,6 +1436,7 @@ def render(text,
         if beg and end:
             return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
         return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+
     text = regex_expand_meta.sub(expand_meta, text)
 
     if environment:
@@ -1423,10 +1453,12 @@ def markmin2html(text, extra={}, allowed={}, sep='p',
                   class_prefix=class_prefix, id_prefix=id_prefix,
                   pretty_print=pretty_print)
 
+
 def run_doctests():
     import doctest
     doctest.testmod()
 
+
 if __name__ == '__main__':
     import sys
     import doctest
@@ -1467,6 +1499,7 @@ if __name__ == '__main__':
                           body=markmin2html(__doc__, pretty_print=True))
     elif sys.argv[1:2] == ['-t']:
         from timeit import Timer
+
         loops = 1000
         ts = Timer("markmin2html(__doc__)", "from markmin2html import markmin2html")
         print 'timeit "markmin2html(__doc__)":'
@@ -1502,4 +1535,3 @@ if __name__ == '__main__':
         print "       file.markmin  [file.css] - process file.markmin + built in file.css (optional)"
         print "       file.markmin  [@path_to/css] - process file.markmin + link path_to/css (optional)"
         run_doctests()
-        

gluon/contrib/markmin/markmin2latex.py

@@ -39,16 +39,20 @@ regex_link = re.compile('\[\[(?P<t>[^\]]*?) +(?P<k>\S+)\]\]')
 regex_auto = re.compile('(?<!["\w])(?P<k>\w+://[\w\.\-\?&%\:]+)', re.M)
 regex_commas = re.compile('[ ]+(?P<t>[,;\.])')
 regex_noindent = re.compile('\n\n(?P<t>[a-z])')
+
+
 # regex_quote_left = re.compile('"(?=\w)')
 # regex_quote_right = re.compile('(?=\w\.)"')
 
 def latex_escape(text, pound=True):
     text = text.replace('\\', '{\\textbackslash}')
-    for c in '^_&$%{}': text=text.replace(c,'\\'+c)
+    for c in '^_&$%{}':
+        text = text.replace(c, '\\' + c)
     text = text.replace('\\{\\textbackslash\\}', '{\\textbackslash}')
     if pound: text = text.replace('#', '\\#')
     return text
 
+
 def render(text,
            extra={},
            allowed={},
@@ -65,26 +69,29 @@ def render(text,
     text = regex_newlines.sub('\n', text)
     while True:
         item = regex_code.search(text, i)
-        if not item: break
+        if not item:
+            break
         if item.group() == META:
             segments.append((None, None))
             text = text[:item.start()] + META + text[item.end():]
         else:
             c = item.group('c') or ''
-            if 'code' in allowed and not c in allowed['code']: c = ''
+            if 'code' in allowed and c not in allowed['code']:
+                c = ''
             code = item.group('t').replace('!`!', '`')
             segments.append((code, c))
             text = text[:item.start()] + META + text[item.end():]
         i = item.start() + 3
 
-
     #############################################################
     # do h1,h2,h3,h4,h5,h6,b,i,ol,ul and normalize spaces
     #############################################################
 
     title = regex_title.search(text)
-    if not title: title='Title'
-    else: title=title.group('t')
+    if not title:
+        title = 'Title'
+    else:
+        title = title.group('t')
 
     text = latex_escape(text, pound=False)
 
@@ -106,9 +113,11 @@ def render(text,
     #############################################################
     while True:
         item = regex_table.search(text)
-        if not item: break
+        if not item:
+            break
         c = item.group('c') or ''
-        if 'table' in allowed and not c in allowed['table']: c = ''
+        if 'table' in allowed and c not in allowed['table']:
+            c = ''
         content = item.group('t')
         if ' | ' in content:
             rows = content.replace('\n', '\\\\\n').replace(' | ', ' & ')
@@ -116,7 +125,8 @@ def render(text,
             cols = row0.count(' & ') + 1
             cal = '{' + ''.join('l' for j in range(cols)) + '}'
             tabular = '\\begin{center}\n{\\begin{tabular}' + cal + '\\hline\n' + row0 + '\\\\ \\hline\n' + row2 + ' \\\\ \\hline\n\\end{tabular}}\n\\end{center}'
-            if row2.count('\n')>20: tabular='\\newpage\n'+tabular
+            if row2.count('\n') > 20:
+                tabular = '\\newpage\n' + tabular
             text = text[:item.start()] + tabular + text[item.end():]
         else:
             text = text[:item.start()] + '\\begin{quote}' + content + '\\end{quote}' + text[item.end():]
@@ -127,8 +137,10 @@ def render(text,
 
     def sub(x):
         f = image_mapper(x.group('k'))
-        if not f: return None
-        return '\n\\begin{center}\\includegraphics[width=8cm]{%s}\\end{center}\n' % (f)
+        if not f:
+            return None
+        return '\n\\begin{center}\\includegraphics[width=8cm]{%s}\\end{center}\n' % f
+
     text = regex_image_width.sub(sub, text)
     text = regex_image.sub(sub, text)
 
@@ -140,7 +152,8 @@ def render(text,
     regex = re.compile('\\\\_\w*\.(eps|png|jpg|gif)')
     while True:
         match = regex.search(text)
-        if not match: break
+        if not match:
+            break
         text = text[:match.start()] + text[match.start() + 1:]
     # text = regex_quote_left.sub('``',text)
     # text = regex_quote_right.sub("''",text)
@@ -157,7 +170,7 @@ def render(text,
     text = parts[0]
     authors = []
     for i, (code, b) in enumerate(segments):
-        if code==None:
+        if code is None:
             html = META
         else:
             if b == 'hidden':
@@ -185,19 +198,25 @@ def render(text,
                 key = b[6:]
                 html = '\\begin{%s}%s\\end{%s}' % (key, code, key)
             elif b in extra:
-                if code[:1]=='\n': code=code[1:]
-                if code[-1:]=='\n': code=code[:-1]
+                if code[:1] == '\n':
+                    code = code[1:]
+                if code[-1:] == '\n':
+                    code = code[:-1]
                 html = extra[b](code)
             elif code[:1] == '\n' or code[:-1] == '\n':
-                if code[:1]=='\n': code=code[1:]
-                if code[-1:]=='\n': code=code[:-1]
+                if code[:1] == '\n':
+                    code = code[1:]
+                if code[-1:] == '\n':
+                    code = code[:-1]
                 if code.startswith('<') or code.startswith('{{') or code.startswith('http'):
                     html = '\\begin{lstlisting}[keywords={}]\n%s\n\\end{lstlisting}' % code
                 else:
                     html = '\\begin{lstlisting}\n%s\n\\end{lstlisting}' % code
             else:
-                if code[:1]=='\n': code=code[1:]
-                if code[-1:]=='\n': code=code[:-1]
+                if code[:1] == '\n':
+                    code = code[1:]
+                if code[-1:] == '\n':
+                    code = code[:-1]
                 html = '{\\ft %s}' % latex_escape(code)
         try:
             text = text + html + parts[i + 1]
@@ -207,6 +226,7 @@ def render(text,
     text = text.replace(' ~\\cite', '~\\cite')
     return text, title, authors
 
+
 WRAPPER = """
 \\documentclass[12pt]{article}
 \\usepackage{hyperref}
@@ -239,12 +259,14 @@ WRAPPER = """
 \\end{document}
 """
 
+
 def markmin2latex(data, image_mapper=lambda x: x, extra={},
                   wrapper=WRAPPER):
     body, title, authors = render(data, extra=extra, image_mapper=image_mapper)
     author = '\n\\and\n'.join(a.replace('\n', '\\\\\n\\footnotesize ') for a in authors)
     return wrapper % dict(title=title, author=author, body=body)
 
+
 if __name__ == '__main__':
     parser = OptionParser()
     parser.add_option("-i", "--info", dest="info",
@@ -261,6 +283,7 @@ if __name__ == '__main__':
     (options, args) = parser.parse_args()
     if options.info:
         import markmin2html
+
         markmin2latex(markmin2html.__doc__)
     elif options.test:
         doctest.testmod()
@@ -287,5 +310,3 @@ if __name__ == '__main__':
                                wrapper=wrapper,
                                chapters=options.chapters)
         print output
-
-

gluon/contrib/markmin/markmin2pdf.py

@@ -13,9 +13,10 @@ from markmin2latex import markmin2latex
 
 __all__ = ['markmin2pdf']
 
-def removeall(path):
 
+def removeall(path):
     ERROR_STR = """Error removing %(path)s, %(error)s """
+
     def rmgeneric(path, __func__):
         try:
             __func__(path)
@@ -107,6 +108,7 @@ if __name__ == '__main__':
     import sys
     import doctest
     import markmin2html
+
     if sys.argv[1:2] == ['-h']:
         data, warnings, errors = markmin2pdf(markmin2html.__doc__)
         if errors:
@@ -127,4 +129,3 @@ if __name__ == '__main__':
             print data
     else:
         doctest.testmod()
-

gluon/contrib/redis_scheduler.py

@@ -9,6 +9,17 @@ Scheduler with redis backend
 ---------------------------------
 """
 
+import os
+import time
+import socket
+import datetime
+import logging
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+from gluon.scheduler import *
+from gluon.scheduler import _decode_dict
+from gluon.contrib.redis_utils import RWatchError
+
 USAGE = """
 ## Example
 
@@ -35,11 +46,6 @@ mysched = RScheduler(db, dict(demo1=demo1,demo2=demo2), ...., redis_conn=rconn)
 
 """
 
-import os
-import time
-import socket
-import datetime
-import logging
 
 path = os.getcwd()
 
@@ -47,20 +53,19 @@ if 'WEB2PY_PATH' not in os.environ:
     os.environ['WEB2PY_PATH'] = path
 
 try:
-    from gluon.contrib.simplejson import loads, dumps
-except:
+    # try external module
     from simplejson import loads, dumps
+except ImportError:
+    try:
+        # try stdlib (Python >= 2.6)
+        from json import loads, dumps
+    except:
+        # fallback to pure-Python module
+        from gluon.contrib.simplejson import loads, dumps
 
 IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
-logger = logging.getLogger('web2py.rscheduler.%s' % IDENTIFIER)
-
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-from gluon.scheduler import *
-from gluon.scheduler import _decode_dict
-from gluon.contrib.redis_utils import RWatchError
-
+logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
 POLLING = 'POLLING'
 
@@ -111,8 +116,7 @@ class RScheduler(Scheduler):
         self._application = current.request.application or 'appname'
 
     def _nkey(self, key):
-        """Helper to restrict all keys to a namespace
-        and track them"""
+        """Helper to restrict all keys to a namespace and track them."""
         prefix = 'w2p:rsched:%s' % self._application
         allkeys = '%s:allkeys' % prefix
         newkey = "%s:%s" % (prefix, key)
@@ -120,10 +124,7 @@ class RScheduler(Scheduler):
         return newkey
 
     def prune_all(self):
-        """
-        Just to be fair and implement a method
-        that does housekeeping
-        """
+        """Global housekeeping."""
         all_keys = self._nkey('allkeys')
         with self.r_server.pipeline() as pipe:
             while True:
@@ -148,8 +149,9 @@ class RScheduler(Scheduler):
 
     def send_heartbeat(self, counter):
         """
-        workers coordination has evolved into something is not that
-        easy. Here we try to do what we need in a single transaction,
+        Workers coordination in redis.
+        It has evolved into something is not that easy.
+        Here we try to do what we need in a single transaction,
         and retry that transaction if something goes wrong
         """
         with self.r_server.pipeline() as pipe:
@@ -167,7 +169,9 @@ class RScheduler(Scheduler):
 
     def inner_send_heartbeat(self, counter, pipe):
         """
-        Does a few things:
+        Do a few things in the "maintenance" thread.
+
+        Specifically:
         - registers the workers
         - accepts commands sent to workers (KILL, TERMINATE, PICK, DISABLED, etc)
         - adjusts sleep
@@ -269,6 +273,8 @@ class RScheduler(Scheduler):
 
     def being_a_ticker(self, pipe):
         """
+        Elects a ticker.
+
         This is slightly more convoluted than the original
         but if far more efficient
         """
@@ -311,7 +317,9 @@ class RScheduler(Scheduler):
 
     def assign_tasks(self, db):
         """
-        The real beauty. We don't need to ASSIGN tasks, we just put
+        The real beauty.
+
+        We don't need to ASSIGN tasks, we just put
         them into the relevant queue
         """
         st, sd = db.scheduler_task, db.scheduler_task_deps
@@ -375,9 +383,6 @@ class RScheduler(Scheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -437,6 +442,7 @@ class RScheduler(Scheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def pop_task(self, db):
+        """Lift a task off a queue."""
         r_server = self.r_server
         st = self.db.scheduler_task
         task = None
@@ -533,7 +539,9 @@ class RScheduler(Scheduler):
 
     def report_task(self, task, task_report):
         """
-        Needs overwriting only because we need to pop from the
+        Override.
+
+        Needs it only because we need to pop from the
         running tasks
         """
         r_server = self.r_server
@@ -558,12 +566,12 @@ class RScheduler(Scheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
-                      and task.next_run_time > task.stop_time
-                      and True or False)
-        status = (task.run_again and is_expired and EXPIRED
-                  or task.run_again and not is_expired
-                  and QUEUED or COMPLETED)
+        is_expired = (task.stop_time and
+                      task.next_run_time > task.stop_time and
+                      True or False)
+        status = (task.run_again and is_expired and EXPIRED or
+                  task.run_again and not is_expired and
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             # assigned calculations
             d = dict(status=status,
@@ -579,10 +587,10 @@ class RScheduler(Scheduler):
             st_mapping = {'FAILED': 'FAILED',
                           'TIMEOUT': 'TIMEOUT',
                           'STOPPED': 'FAILED'}[task_report.status]
-            status = (task.retry_failed
-                      and task.times_failed < task.retry_failed
-                      and QUEUED or task.retry_failed == -1
-                      and QUEUED or st_mapping)
+            status = (task.retry_failed and
+                      task.times_failed < task.retry_failed and
+                      QUEUED or task.retry_failed == -1 and
+                      QUEUED or st_mapping)
             db(st.id == task.task_id).update(
                 times_failed=st.times_failed + 1,
                 next_run_time=task.next_run_time,
@@ -596,7 +604,7 @@ class RScheduler(Scheduler):
         r_server.hdel(running_dict, task.task_id)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -620,8 +628,8 @@ class RScheduler(Scheduler):
                 time.sleep(0.5)
 
     def get_workers(self, only_ticker=False):
-        """ Returns a dict holding worker_name : {**columns}
-        representing all "registered" workers
+        """Return a dict holding worker_name : {**columns}
+        representing all "registered" workers.
         only_ticker returns only the worker running as a TICKER,
         if there is any
         """

gluon/dal.py

@@ -79,7 +79,7 @@ def _default_validators(db, field):
     if (field.notnull or field.unique) and not field_type in excluded_fields:
         requires.insert(0, validators.IS_NOT_EMPTY())
     elif not field.notnull and not field.unique and requires:
-        requires[0] = validators.IS_EMPTY_OR(requires[0])
+        requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field in ('string', 'text', 'password') else None)
     return requires
 
 from gluon.serializers import custom_json, xml

gluon/html.py

@@ -308,7 +308,7 @@ def URL(a=None,
         else:
             function = f
 
-        # if the url gets a static resource, don't force extention
+        # if the url gets a static resource, don't force extension
         if controller == 'static':
             extension = None
             # add static version to url
@@ -330,11 +330,9 @@ def URL(a=None,
     if args:
         if url_encode:
             if encode_embedded_slash:
-                other = '/' + '/'.join([urllib.quote(str(
-                    x), '') for x in args])
+                other = '/' + '/'.join([urllib.quote(str(x), '') for x in args])
             else:
-                other = args and urllib.quote(
-                    '/' + '/'.join([str(x) for x in args]))
+                other = args and urllib.quote('/' + '/'.join([str(x) for x in args]))
         else:
             other = args and ('/' + '/'.join([str(x) for x in args]))
     else:
@@ -955,7 +953,6 @@ class DIV(XmlComponent):
         # get the xml for the inner components
         co = join([xmlescape(component) for component in
                    self.components])
-
         return (fa, co)
 
     def xml(self):
@@ -1059,7 +1056,7 @@ class DIV(XmlComponent):
 
         >>> a = DIV(DIV(SPAN('x', _class='abc'), DIV(SPAN('y', _class='abc'), SPAN('z', _class='abc'))))
         >>> b = a.elements('span.abc', replace=P('x', _class='xyz'))
-        >>> print a
+        >>> print a  # We should .xml() here instead of print
         <div><div><p class="xyz">x</p><div><p class="xyz">x</p><p class="xyz">x</p></div></div></div>
 
         "replace" can be a callable, which will be passed the original element and
@@ -2805,7 +2802,7 @@ class MARKMIN(XmlComponent):
         self.extra = extra or {}
         self.allowed = allowed or {}
         self.sep = sep
-        self.url = URL if url == True else url
+        self.url = URL if url is True else url
         self.environment = environment
         self.latex = latex
         self.autolinks = autolinks

gluon/languages.py

@@ -1002,17 +1002,19 @@ def update_all_languages(application_path):
             findT(application_path, language[:-3])
 
 
-def update_from_langfile(target, source):
+def update_from_langfile(target, source, force_update=False):
     """this will update untranslated messages in target from source (where both are language files)
     this can be used as first step when creating language file for new but very similar language
         or if you want update your app from welcome app of newer web2py version
         or in non-standard scenarios when you work on target and from any reason you have partial translation in source
+    Args:
+        force_update: if False existing translations remain unchanged, if True existing translations will update from source
     """
     src = read_dict(source)
     sentences = read_dict(target)
     for key in sentences:
         val = sentences[key]
-        if not val or val == key:
+        if not val or val == key or force_update:
             new_val = src.get(key)
             if new_val and new_val != val:
                 sentences[key] = new_val

gluon/scheduler.py

@@ -9,6 +9,26 @@ Background processes made simple
 ---------------------------------
 """
 
+
+import os
+import time
+import multiprocessing
+import sys
+import threading
+import traceback
+import signal
+import socket
+import datetime
+import logging
+import optparse
+import tempfile
+import types
+import Queue
+from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
+from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+
 USAGE = """
 ## Example
 
@@ -67,20 +87,6 @@ sudo restart web2py-scheduler
 sudo status web2py-scheduler
 """
 
-import os
-import time
-import multiprocessing
-import sys
-import threading
-import traceback
-import signal
-import socket
-import datetime
-import logging
-import optparse
-import types
-import Queue
-
 path = os.getcwd()
 
 if 'WEB2PY_PATH' not in os.environ:
@@ -101,12 +107,6 @@ IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
 logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
-from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
-from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-
-
 QUEUED = 'QUEUED'
 ASSIGNED = 'ASSIGNED'
 RUNNING = 'RUNNING'
@@ -168,24 +168,25 @@ class TaskReport(object):
 
 
 class JobGraph(object):
-    """Experimental: with JobGraph you can specify
-    dependencies amongs tasks"""
+    """Experimental: dependencies amongs tasks"""
 
     def __init__(self, db, job_name):
         self.job_name = job_name or 'job_0'
         self.db = db
 
     def add_deps(self, task_parent, task_child):
-        """Creates a dependency between task_parent and task_child"""
+        """Create a dependency between task_parent and task_child."""
         self.db.scheduler_task_deps.insert(task_parent=task_parent,
                                            task_child=task_child,
                                            job_name=self.job_name)
 
-    def validate(self, job_name):
-        """Validates if all tasks job_name can be completed, i.e. there
-        are no mutual dependencies among tasks.
+    def validate(self, job_name=None):
+        """Validate if all tasks job_name can be completed.
+
+        Checks if there are no mutual dependencies among tasks.
         Commits at the end if successfull, or it rollbacks the entire
-        transaction. Handle with care!"""
+        transaction. Handle with care!
+        """
         db = self.db
         sd = db.scheduler_task_deps
         if job_name:
@@ -223,14 +224,6 @@ class JobGraph(object):
             db.rollback()
             return None
 
-
-def demo_function(*argv, **kwargs):
-    """ test function """
-    for i in range(argv[0]):
-        print 'click', i
-        time.sleep(1)
-    return 'done'
-
 # the two functions below deal with simplejson decoding as unicode, esp for the dict decode
 # and subsequent usage as function Keyword arguments unicode variable names won't work!
 # borrowed from http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-unicode-ones-from-json-in-python
@@ -261,11 +254,12 @@ def _decode_dict(dct):
 
 
 def executor(queue, task, out):
-    """The function used to execute tasks in the background process"""
+    """The function used to execute tasks in the background process."""
     logger.debug('    task started')
 
     class LogOutput(object):
-        """Facility to log output at intervals"""
+        """Facility to log output at intervals."""
+
         def __init__(self, out_queue):
             self.out_queue = out_queue
             self.stdout = sys.stdout
@@ -280,7 +274,11 @@ def executor(queue, task, out):
         def write(self, data):
             self.out_queue.put(data)
 
-    W2P_TASK = Storage({'id': task.task_id, 'uuid': task.uuid})
+    W2P_TASK = Storage({
+                       'id': task.task_id,
+                       'uuid': task.uuid,
+                       'run_id': task.run_id
+                       })
     stdout = LogOutput(out)
     try:
         if task.app:
@@ -318,6 +316,11 @@ def executor(queue, task, out):
             result = eval(task.function)(
                 *loads(task.args, object_hook=_decode_dict),
                 **loads(task.vars, object_hook=_decode_dict))
+        if len(result) >= 1024:
+            fd, temp_path = tempfile.mkstemp(suffix='.w2p_sched')
+            with os.fdopen(fd, 'w') as f:
+                f.write(result)
+            result = 'w2p_special:%s' % temp_path
         queue.put(TaskReport('COMPLETED', result=result))
     except BaseException, e:
         tb = traceback.format_exc()
@@ -335,7 +338,7 @@ class MetaScheduler(threading.Thread):
         self.empty_runs = 0
 
     def async(self, task):
-        """Starts the background process
+        """Start the background process.
 
         Args:
             task : a `Task` object
@@ -410,6 +413,12 @@ class MetaScheduler(threading.Thread):
             else:
                 logger.debug('  task completed or failed')
                 tr = queue.get()
+        result = tr.result
+        if result and result.startswith('w2p_special'):
+            temp_path = result.replace('w2p_special:', '', 1)
+            with open(temp_path) as f:
+                tr.result = f.read()
+            os.unlink(temp_path)
         tr.output = task_output
         return tr
 
@@ -444,50 +453,23 @@ class MetaScheduler(threading.Thread):
         self.start()
 
     def send_heartbeat(self, counter):
-        print 'thum'
-        time.sleep(1)
+        raise NotImplementedError
 
     def pop_task(self):
         """Fetches a task ready to be executed"""
-        return Task(
-            app=None,
-            function='demo_function',
-            timeout=7,
-            args='[2]',
-            vars='{}')
+        raise NotImplementedError
 
     def report_task(self, task, task_report):
         """Creates a task report"""
-        print 'reporting task'
-        pass
+        raise NotImplementedError
 
     def sleep(self):
-        pass
+        raise NotImplementedError
 
     def loop(self):
         """Main loop, fetching tasks and starting executor's background
         processes"""
-        try:
-            self.start_heartbeats()
-            while True and self.have_heartbeat:
-                logger.debug('looping...')
-                task = self.pop_task()
-                if task:
-                    self.empty_runs = 0
-                    self.report_task(task, self.async(task))
-                else:
-                    self.empty_runs += 1
-                    logger.debug('sleeping...')
-                    if self.max_empty_runs != 0:
-                        logger.debug('empty runs %s/%s',
-                                     self.empty_runs, self.max_empty_runs)
-                        if self.empty_runs >= self.max_empty_runs:
-                            logger.info(
-                                'empty runs limit reached, killing myself')
-                            self.die()
-                    self.sleep()
-        except KeyboardInterrupt:
-            self.die()
+        raise NotImplementedError
 
 
 TASK_STATUS = (QUEUED, RUNNING, COMPLETED, FAILED, TIMEOUT, STOPPED, EXPIRED)
@@ -594,11 +576,11 @@ class Scheduler(MetaScheduler):
         return True
 
     def now(self):
-        """Shortcut that fetches current time based on UTC preferences"""
+        """Shortcut that fetches current time based on UTC preferences."""
         return self.utc_time and datetime.datetime.utcnow() or datetime.datetime.now()
 
     def set_requirements(self, scheduler_task):
-        """Called to set defaults for lazy_tables connections"""
+        """Called to set defaults for lazy_tables connections."""
         from gluon import current
         if hasattr(current, 'request'):
             scheduler_task.application_name.default = '%s/%s' % (
@@ -606,7 +588,7 @@ class Scheduler(MetaScheduler):
             )
 
     def define_tables(self, db, migrate):
-        """Defines Scheduler tables structure"""
+        """Define Scheduler tables structure."""
         from pydal.base import DEFAULT
         logger.debug('defining tables (migrate=%s)', migrate)
         now = self.now
@@ -693,14 +675,14 @@ class Scheduler(MetaScheduler):
 
     @staticmethod
     def total_seconds(td):
-        # backport for py2.6
+        """Backport for py2.6."""
         if hasattr(td, 'total_seconds'):
             return td.total_seconds()
         else:
             return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6
 
     def loop(self, worker_name=None):
-        """Main loop
+        """Main loop.
 
         This works basically as a neverending loop that:
 
@@ -752,7 +734,8 @@ class Scheduler(MetaScheduler):
             self.die()
 
     def wrapped_assign_tasks(self, db):
-        """Commodity function to call `assign_tasks` and trap exceptions
+        """Commodity function to call `assign_tasks` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `assign_task` after 0.5 seconds
         """
@@ -773,7 +756,8 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -793,7 +777,7 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def pop_task(self, db):
-        """Grabs a task ready to be executed from the queue"""
+        """Grab a task ready to be executed from the queue."""
         now = self.now()
         st = self.db.scheduler_task
         if self.is_a_ticker and self.do_assign_tasks:
@@ -874,7 +858,8 @@ class Scheduler(MetaScheduler):
             uuid=task.uuid)
 
     def wrapped_report_task(self, task, task_report):
-        """Commodity function to call `report_task` and trap exceptions
+        """Commodity function to call `report_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -891,8 +876,10 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def report_task(self, task, task_report):
-        """Takes care of storing the result according to preferences
-        and deals with logic for repeating tasks"""
+        """Take care of storing the result according to preferences.
+
+        Deals with logic for repeating tasks.
+        """
         db = self.db
         now = self.now()
         st = db.scheduler_task
@@ -914,12 +901,12 @@ class Scheduler(MetaScheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
-                      and task.next_run_time > task.stop_time
-                      and True or False)
-        status = (task.run_again and is_expired and EXPIRED
-                  or task.run_again and not is_expired
-                  and QUEUED or COMPLETED)
+        is_expired = (task.stop_time and
+                      task.next_run_time > task.stop_time and
+                      True or False)
+        status = (task.run_again and is_expired and EXPIRED or
+                  task.run_again and not is_expired and
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             d = dict(status=status,
                      next_run_time=task.next_run_time,
@@ -945,27 +932,26 @@ class Scheduler(MetaScheduler):
         logger.info('task completed (%s)', task_report.status)
 
     def update_dependencies(self, db, task_id):
+        """Unblock execution paths for Jobs."""
         db(db.scheduler_task_deps.task_child == task_id).update(can_visit=True)
 
     def adj_hibernation(self):
-        """Used to increase the "sleep" interval for DISABLED workers"""
+        """Used to increase the "sleep" interval for DISABLED workers."""
         if self.w_stats.status == DISABLED:
             wk_st = self.w_stats.sleep
             hibernation = wk_st + HEARTBEAT if wk_st < MAXHIBERNATION else MAXHIBERNATION
             self.w_stats.sleep = hibernation
 
     def send_heartbeat(self, counter):
-        """This function is vital for proper coordination among available
-        workers.
-        It:
+        """Coordination among available workers.
 
+        It:
         - sends the heartbeat
         - elects a ticker among available workers (the only process that
             effectively dispatch tasks to workers)
         - deals with worker's statuses
         - does "housecleaning" for dead workers
         - triggers tasks assignment to workers
-
         """
         if not self.db_thread:
             logger.debug('thread building own DAL object')
@@ -1053,7 +1039,8 @@ class Scheduler(MetaScheduler):
         self.sleep()
 
     def being_a_ticker(self):
-        """Elects a TICKER process that assigns tasks to available workers.
+        """Elect a TICKER process that assigns tasks to available workers.
+
         Does its best to elect a worker that is not busy processing other tasks
         to allow a proper distribution of tasks among all active workers ASAP
         """
@@ -1087,7 +1074,7 @@ class Scheduler(MetaScheduler):
             return False
 
     def assign_tasks(self, db):
-        """Assigns task to workers, that can then pop them from the queue
+        """Assign task to workers, that can then pop them from the queue.
 
         Deals with group_name(s) logic, in order to assign linearly tasks
         to available workers for those groups
@@ -1137,9 +1124,6 @@ class Scheduler(MetaScheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -1151,8 +1135,8 @@ class Scheduler(MetaScheduler):
         # intelligence (like esteeming how many tasks will a worker complete
         # before the ticker reassign them around, but the gain is quite small
         # 50 is a sweet spot also for fast tasks, with sane heartbeat values
-        # NB: ticker reassign tasks every 5 cycles, so if a worker completes its
-        # 50 tasks in less than heartbeat*5 seconds,
+        # NB: ticker reassign tasks every 5 cycles, so if a worker completes
+        # its 50 tasks in less than heartbeat*5 seconds,
         # it won't pick new tasks until heartbeat*5 seconds pass.
 
         # If a worker is currently elaborating a long task, its tasks needs to
@@ -1183,8 +1167,6 @@ class Scheduler(MetaScheduler):
                         status=ASSIGNED,
                         assigned_worker_name=assigned_wn
                     )
-                    if not task.task_name:
-                        d['task_name'] = task.function_name
                     db(
                         (st.id == task.id) &
                         (st.status.belongs((QUEUED, ASSIGNED)))
@@ -1204,14 +1186,13 @@ class Scheduler(MetaScheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def sleep(self):
-        """Calculates the number of seconds to sleep according to worker's
-        status and `heartbeat` parameter"""
+        """Calculate the number of seconds to sleep."""
         time.sleep(self.w_stats.sleep)
         # should only sleep until next available task
 
     def set_worker_status(self, group_names=None, action=ACTIVE,
                           exclude=None, limit=None, worker_name=None):
-        """Internal function to set worker's status"""
+        """Internal function to set worker's status."""
         ws = self.db.scheduler_worker
         if not group_names:
             group_names = self.group_names
@@ -1235,10 +1216,12 @@ class Scheduler(MetaScheduler):
                 self.db(ws.id.belongs(workers)).update(status=action)
 
     def disable(self, group_names=None, limit=None, worker_name=None):
-        """Sets DISABLED on the workers processing `group_names` tasks.
+        """Set DISABLED on the workers processing `group_names` tasks.
+
         A DISABLED worker will be kept alive but it won't be able to process
         any waiting tasks, essentially putting it to sleep.
-        By default, all group_names of Scheduler's instantation are selected"""
+        By default, all group_names of Scheduler's instantation are selected
+        """
         self.set_worker_status(
             group_names=group_names,
             action=DISABLED,
@@ -1283,8 +1266,9 @@ class Scheduler(MetaScheduler):
             pvars: "raw" kwargs to be passed to the function. Automatically
                 jsonified
             kwargs: all the parameters available (basically, every
-                `scheduler_task` column). If args and vars are here, they should
-                be jsonified already, and they will override pargs and pvars
+                `scheduler_task` column). If args and vars are here, they
+                should be jsonified already, and they will override pargs
+                and pvars
 
         Returns:
             a dict just as a normal validate_and_insert(), plus a uuid key

gluon/sqlhtml.py

@@ -1897,33 +1897,39 @@ class SQLFORM(FORM):
                 else:
                     field_type = field.type
 
-                operators = SELECT(*[OPTION(T(option), _value=option) for option in options], _class='form-control')
+                operators = SELECT(*[OPTION(T(option), _value=option) for option in options], 
+                                    _class='form-control')
                 _id = "%s_%s" % (value_id, name)
                 if field_type in ['boolean', 'double', 'time', 'integer']:
                     widget_ = SQLFORM.widgets[field_type]
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control')
+                    value_input = widget_.widget(field, field.default, _id=_id, 
+                                                 _class=widget_._class + ' form-control')
                 elif field_type == 'date':
                     iso_format = {'_data-w2p_date_format': '%Y-%m-%d'}
                     widget_ = SQLFORM.widgets.date
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control', **iso_format)
+                    value_input = widget_.widget(field, field.default, _id=_id, 
+                                                 _class=widget_._class + ' form-control', 
+                                                 **iso_format)
                 elif field_type == 'datetime':
                     iso_format = {'_data-w2p_datetime_format': '%Y-%m-%d %H:%M:%S'}
                     widget_ = SQLFORM.widgets.datetime
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control', **iso_format)
-                elif (field_type.startswith('reference ') or
-                      field_type.startswith('list:reference ')) and \
-                      hasattr(field.requires, 'options') or \
-                      hasattr(field.requires, 'options'):
+                    value_input = widget_.widget(field, field.default, _id=_id, 
+                                                 _class=widget_._class + ' form-control', 
+                                                 **iso_format)
+                elif hasattr(field.requires, 'options'):
                     value_input = SELECT(
                         *[OPTION(v, _value=k)
                           for k, v in field.requires.options()],
                          _class='form-control',
                          **dict(_id=_id))
-                elif field_type.startswith('reference ') or \
-                     field_type.startswith('list:integer') or \
-                     field_type.startswith('list:reference '):
+                elif (field_type.startswith('integer') or
+                      field_type.startswith('reference ') or
+                      field_type.startswith('list:integer') or
+                      field_type.startswith('list:reference ')):
                     widget_ = SQLFORM.widgets.integer
-                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control')
+                    value_input = widget_.widget(
+                        field, field.default, _id=_id, 
+                        _class=widget_._class + ' form-control')
                 else:
                     value_input = INPUT(
                         _type='text', _id=_id,
@@ -3037,7 +3043,16 @@ class SQLFORM(FORM):
             query = query & constraints[table._tablename]
         if isinstance(links, dict):
             links = links.get(table._tablename, [])
-        for key in 'columns,orderby,searchable,sortable,paginate,deletable,editable,details,selectable,create,fields'.split(','):
+        for key in ('fields', 'field_id', 'left', 'headers', 'orderby', 'groupby', 'searchable', 
+                    'sortable', 'paginate', 'deletable', 'editable', 'details', 'selectable',
+                    'create', 'csv', 'links', 'links_in_grid', 'upload', 'maxtextlengths',
+                    'maxtextlength', 'onvalidation', 'onfailure', 'oncreate', 'onupdate',
+                    'ondelete', 'sorter_icons', 'ui', 'showbuttontext', '_class', 'formname',
+                    'search_widget', 'advanced_search', 'ignore_rw', 'formstyle', 'exportclasses',
+                    'formargs', 'createargs', 'editargs', 'viewargs', 'selectable_submit_button',
+                    'buttons_placement', 'links_placement', 'noconfirm', 'cache_count', 'client_side_delete',
+                    'ignore_common_filters', 'auto_pagination', 'use_cursor'
+                   ):
             if isinstance(kwargs.get(key, None), dict):
                 if table._tablename in kwargs[key]:
                     kwargs[key] = kwargs[key][table._tablename]

gluon/tests/__init__.py

@@ -21,6 +21,8 @@ from test_contribs import *
 from test_web import *
 from test_dal import *
 from test_tools import *
+from test_appadmin import *
+from test_scheduler import *
 
 if sys.version[:3] == '2.7':
     from test_old_doctests import *

gluon/tests/test_appadmin.py

@@ -0,0 +1,175 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.sqlhtml
+"""
+import os
+import sys
+if sys.version < "2.7":
+    import unittest2 as unittest
+else:
+    import unittest
+
+from fix_path import fix_sys_path
+
+fix_sys_path(__file__)
+
+
+from compileapp import run_controller_in, run_view_in
+from languages import translator
+from gluon.storage import Storage, List
+import gluon.fileutils
+from gluon.dal import DAL, Field, Table
+from gluon.http import HTTP
+
+DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
+
+try:
+    import json
+except ImportError:
+    from gluon.contrib import simplejson as json
+
+
+def fake_check_credentials(foo):
+    return True
+
+
+class TestAppAdmin(unittest.TestCase):
+
+    def setUp(self):
+        from gluon.globals import Request, Response, Session, current
+        from gluon.html import A, DIV, FORM, MENU, TABLE, TR, INPUT, URL, XML
+        from gluon.validators import IS_NOT_EMPTY
+        from compileapp import LOAD
+        from gluon.http import HTTP, redirect
+        from gluon.tools import Auth
+        from gluon.sql import SQLDB
+        from gluon.sqlhtml import SQLTABLE, SQLFORM
+        self.original_check_credentials = gluon.fileutils.check_credentials
+        gluon.fileutils.check_credentials = fake_check_credentials
+        request = Request(env={})
+        request.application = 'welcome'
+        request.controller = 'appadmin'
+        request.function = self._testMethodName.split('_')[1]
+        request.folder = 'applications/welcome'
+        request.env.http_host = '127.0.0.1:8000'
+        request.env.remote_addr = '127.0.0.1'
+        response = Response()
+        session = Session()
+        T = translator('', 'en')
+        session.connect(request, response)
+        current.request = request
+        current.response = response
+        current.session = session
+        current.T = T
+        db = DAL(DEFAULT_URI, check_reserved=['all'])
+        auth = Auth(db)
+        auth.define_tables(username=True, signature=False)
+        db.define_table('t0', Field('tt'), auth.signature)
+        # Create a user
+        db.auth_user.insert(first_name='Bart',
+                            last_name='Simpson',
+                            username='user1',
+                            email='user1@test.com',
+                            password='password_123',
+                            registration_key=None,
+                            registration_id=None)
+        self.env = locals()
+
+    def tearDown(self):
+        gluon.fileutils.check_credentials = self.original_check_credentials
+
+    def run_function(self):
+        return run_controller_in(self.env['request'].controller, self.env['request'].function, self.env)
+
+    def run_view(self):
+        return run_view_in(self.env)
+
+    def test_index(self):
+        result = self.run_function()
+        self.assertTrue('db' in result['databases'])
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_select(self):
+        request = self.env['request']
+        request.args = List(['db'])
+        request.env.query_string = 'query=db.auth_user.id>0'
+        result = self.run_function()
+        self.assertTrue('table' in result and 'query' in result)
+        self.assertTrue(result['table'] == 'auth_user')
+        self.assertTrue(result['query'] == 'db.auth_user.id>0')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        result = self.run_function()
+        self.assertTrue('table' in result)
+        self.assertTrue('form' in result)
+        self.assertTrue(str(result['table']) is 'auth_user')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        data['first_name'] = 'Lisa'
+        data['last_name'] = 'Simpson'
+        data['username'] = 'lisasimpson'
+        data['password'] = 'password_123'
+        data['email'] = 'lisa@example.com'
+        request._vars = data
+        result = self.run_function()
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+        db = self.env['db']
+        lisa_record = db(db.auth_user.username == 'lisasimpson').select().first()
+        self.assertIsNotNone(lisa_record)
+        del data['_formkey']
+        del data['_formname']
+        del data['password']
+        for key in data:
+            self.assertEqual(data[key], lisa_record[key])
+
+    def test_update_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user', '1'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        for element in form.elements('input'):
+            data[element['_name']] = element['_value']
+        data['email'] = 'user1@example.com'
+        data['id'] = '1'
+        request._vars = data
+        self.assertRaises(HTTP, self.run_function)
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_cache.py

@@ -37,10 +37,11 @@ def tearDownModule():
         pass
 
 
-
 class TestCache(unittest.TestCase):
 
-    def testCacheInRam(self):
+    # TODO: test_CacheAbstract(self):
+
+    def test_CacheInRam(self):
 
         # defaults to mode='http'
         cache = CacheInRam()
@@ -67,8 +68,7 @@ class TestCache(unittest.TestCase):
         cache.increment('b')
         self.assertEqual(cache('b', lambda: 'x', 100), 1)
 
-
-    def testCacheOnDisk(self):
+    def test_CacheOnDisk(self):
 
         # defaults to mode='http'
         s = Storage({'application': 'admin',
@@ -97,7 +97,13 @@ class TestCache(unittest.TestCase):
         cache.increment('b')
         self.assertEqual(cache('b', lambda: 'x', 100), 1)
 
-    def testCacheWithPrefix(self):
+    # TODO: def test_CacheAction(self):
+
+    # TODO: def test_Cache(self):
+
+    # TODO: def test_lazy_cache(self):
+
+    def test_CacheWithPrefix(self):
         s = Storage({'application': 'admin',
                      'folder': 'applications/admin'})
         cache = Cache(s)
@@ -106,7 +112,7 @@ class TestCache(unittest.TestCase):
         self.assertEqual(prefix('a', lambda: 2, 100), 1)
         self.assertEqual(cache.ram('prefixa', lambda: 2, 100), 1)
 
-    def testRegex(self):
+    def test_Regex(self):
         cache = CacheInRam()
         self.assertEqual(cache('a1', lambda: 1, 0), 1)
         self.assertEqual(cache('a2', lambda: 2, 100), 2)
@@ -114,7 +120,7 @@ class TestCache(unittest.TestCase):
         self.assertEqual(cache('a1', lambda: 2, 0), 2)
         self.assertEqual(cache('a2', lambda: 3, 100), 3)
 
-    def testDALcache(self):
+    def test_DALcache(self):
         s = Storage({'application': 'admin',
                      'folder': 'applications/admin'})
         cache = Cache(s)

gluon/tests/test_dal.py

@@ -106,6 +106,7 @@ class TestDALAdapters(unittest.TestCase):
     def test_mysql(self):
         if os.environ.get('APPVEYOR'):
             return
+        if os.environ.get('TRAVIS'):
             os.environ["DB"] = "mysql://root:@localhost/pydal"
             result = self._run_tests()
             self.assertTrue(result)
@@ -113,6 +114,7 @@ class TestDALAdapters(unittest.TestCase):
     def test_pg8000(self):
         if os.environ.get('APPVEYOR'):
             return
+        if os.environ.get('TRAVIS'):
             os.environ["DB"] = "postgres:pg8000://postgres:@localhost/pydal"
             result = self._run_tests()
             self.assertTrue(result)

gluon/tests/test_html.py

@@ -14,11 +14,14 @@ from html import *
 from html import verifyURL
 from html import truncate_string
 from storage import Storage
+from html import XML_pickle, XML_unpickle
+from html import TAG_pickler, TAG_unpickler
 
 
 class TestBareHelpers(unittest.TestCase):
 
-    # TODO: def test_xmlescape(self):
+    # xmlescape() = covered by other tests
+
     # TODO: def test_call_as_list(self):
 
     def test_truncate_string(self):
@@ -26,6 +29,8 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(truncate_string('Lorem ipsum dolor sit amet, consectetur adipiscing elit, '
                                          'sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.',
                                          length=30), 'Lorem ipsum dolor sit amet,...')
+        self.assertEqual(truncate_string('Short text shorter than the length parameter.', length=100),
+                         'Short text shorter than the length parameter.')
         # French text
         self.assertEqual(truncate_string('Un texte en français avec des accents et des caractères bizarre.', length=30),
                          'Un texte en français avec d...')
@@ -93,6 +98,11 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
         rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'p': (1, 3), 'q': 2}, hmac_key='key', hash_vars='p')
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
+        # test url_encode
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=False)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li\xc3\xa9=2&ma\xc3\xaf=1&ma\xc3\xaf=3')
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=True)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li%C3%A9=2&ma%C3%AF=1&ma%C3%AF=3')
         # test CRLF detection
         self.assertRaises(SyntaxError, URL, *['a\n', 'c', 'f'])
         self.assertRaises(SyntaxError, URL, *['a\r', 'c', 'f'])
@@ -168,10 +178,13 @@ class TestBareHelpers(unittest.TestCase):
         # bug check for the sanitizer for closing no-close tags
         self.assertEqual(XML('<p>Test</p><br/><p>Test</p><br/>', sanitize=True),
                          XML('<p>Test</p><br /><p>Test</p><br />'))
+        # basic flatten test
+        self.assertEqual(XML('<p>Test</p>').flatten(), '<p>Test</p>')
+        self.assertEqual(XML('<p>Test</p>').flatten(render=lambda text, tag, attr: text), '<p>Test</p>')
 
-    # TODO: def test_XML_unpickle(self):
-
-    # TODO: def test_XML_pickle(self):
+    def test_XML_pickle_unpickle(self):
+        # weird test
+        self.assertEqual(XML_unpickle(XML_pickle('data to be pickle')[1][0]), 'data to be pickle')
 
     def test_DIV(self):
         # Empty DIV()
@@ -199,7 +212,31 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(a.xml(), '<div class="abc"><span>a</span><div>b</div></div>')
         self.assertEqual([el.xml() for el in s.siblings()], ['<div>b</div>'])
         self.assertEqual(s.sibling().xml(), '<div>b</div>')
+        # siblings with wrong args
         self.assertEqual(s.siblings('a'), [])
+        # siblings with good args
+        self.assertEqual(s.siblings('div')[0].xml(), '<div>b</div>')
+        # Check for siblings with wrong kargs and value
+        self.assertEqual(s.siblings(a='d'), [])
+        # Check for siblings with good kargs and value
+        # Can't figure this one out what is a right value here??
+        # Commented for now...
+        # self.assertEqual(s.siblings(div='<div>b</div>'), ???)
+        # No other sibling should return None
+        self.assertEqual(DIV(P('First element')).element('p').sibling(), None)
+        # --------------------------------------------------------------------------------------------------------------
+        # This use unicode to hit xmlescape() line :
+        #     """
+        #     elif isinstance(data, unicode):
+        #         data = data.encode('utf8', 'xmlcharrefreplace')
+        #     """
+        self.assertEqual(DIV(u'Texte en français avec des caractères accentués...').xml(),
+                         '<div>Texte en fran\xc3\xa7ais avec des caract\xc3\xa8res accentu\xc3\xa9s...</div>')
+        # --------------------------------------------------------------------------------------------------------------
+        self.assertEqual(DIV('Test with an ID', _id='id-of-the-element').xml(),
+                         '<div id="id-of-the-element">Test with an ID</div>')
+        self.assertEqual(DIV().element('p'), None)
+
         # Corner case for raise coverage of one line
         # I think such assert fail cause of python 2.6
         # Work under python 2.7
@@ -207,6 +244,9 @@ class TestBareHelpers(unittest.TestCase):
         #     DIV(BR('<>')).xml()
         # self.assertEqual(cm.exception[0], '<br/> tags cannot have components')
 
+        # test .get('attrib')
+        self.assertEqual(DIV('<p>Test</p>', _class="class_test").get('_class'), 'class_test')
+
     def test_CAT(self):
         # Empty CAT()
         self.assertEqual(CAT().xml(), '')
@@ -215,9 +255,10 @@ class TestBareHelpers(unittest.TestCase):
         # CAT(' ')
         self.assertEqual(CAT(' ').xml(), ' ')
 
-    # TODO: def test_TAG_unpickler(self):
-
-    # TODO: def test_TAG_pickler(self):
+    def test_TAG_pickler_unpickler(self):
+        # weird test
+        self.assertEqual(TAG_unpickler(TAG_pickler(TAG.div('data to be pickle'))[1][0]).xml(),
+                         '<div>data to be pickle</div>')
 
     def test_TAG(self):
         self.assertEqual(TAG.first(TAG.second('test'), _key=3).xml(),
@@ -225,6 +266,9 @@ class TestBareHelpers(unittest.TestCase):
         # ending in underscore "triggers" <input /> style
         self.assertEqual(TAG.first_(TAG.second('test'), _key=3).xml(),
                          '<first key="3" />')
+        # unicode test for TAG
+        self.assertEqual(TAG.div(u'Texte en français avec des caractères accentués...').xml(),
+                         '<div>Texte en fran\xc3\xa7ais avec des caract\xc3\xa8res accentu\xc3\xa9s...</div>')
 
     def test_HTML(self):
         self.assertEqual(HTML('<>', _a='1', _b='2').xml(),
@@ -259,6 +303,8 @@ class TestBareHelpers(unittest.TestCase):
                          '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd">\n<html a="1" b="2" lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">&lt;&gt;</html>')
         self.assertEqual(XHTML('<>', _a='1', _b='2', doctype='xmlns').xml(),
                          'xmlns\n<html a="1" b="2" lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">&lt;&gt;</html>')
+        self.assertEqual(XHTML('<>', _a='1', _b='2', _xmlns='xmlns').xml(),
+                         '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n<html a="1" b="2" lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">&lt;&gt;</html>')
 
     def test_HEAD(self):
         self.assertEqual(HEAD('<>', _a='1', _b='2').xml(),
@@ -290,6 +336,8 @@ class TestBareHelpers(unittest.TestCase):
     def test_STYLE(self):
         self.assertEqual(STYLE('<>', _a='1', _b='2').xml(),
                          '<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
+        # Try to hit : return DIV.xml(self)
+        self.assertEqual(STYLE().xml(), '<style></style>')
 
     def test_IMG(self):
         self.assertEqual(IMG(_a='1', _b='2').xml(),
@@ -351,18 +399,12 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(HR(_a='1', _b='2').xml(), '<hr a="1" b="2" />')
 
     def test_A(self):
-        self.assertEqual(
-            A('<>', _a='1', _b='2').xml(),
-            '<a a="1" b="2">&lt;&gt;</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>'
-            )
+        self.assertEqual(A('<>', _a='1', _b='2').xml(),
+                         '<a a="1" b="2">&lt;&gt;</a>')
+        self.assertEqual(A('a', cid='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
+        self.assertEqual(A('a', callback='b', _id='c').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>')
         # Callback with no id trigger web2py_uuid() call
         from html import web2pyHTMLParser
         a = A('a', callback='b').xml()
@@ -370,38 +412,22 @@ class TestBareHelpers(unittest.TestCase):
             uuid_generated = tag.attributes['_id']
         self.assertEqual(a,
                          '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="{id}">a</a>'.format(id=uuid_generated))
-        self.assertEqual(
-            A('a', delete='tr').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>'
-            )
-        self.assertEqual(
-            A('a', _id='b', target='<self>').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', component='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', _id='b', callback='c', noconfirm=True).xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b', _disable_with='processing...').xml(),
-            '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
-            '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
+        self.assertEqual(A('a', delete='tr').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>')
+        self.assertEqual(A('a', _id='b', target='<self>').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>')
+        self.assertEqual(A('a', component='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>')
+        self.assertEqual(A('a', _id='b', callback='c', noconfirm=True).xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>')
+        self.assertEqual(A('a', cid='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
+        self.assertEqual(A('a', cid='b', _disable_with='processing...').xml(),
+                         '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>')
+        self.assertEqual(A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
+        self.assertEqual(A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
+                         '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
 
     def test_BUTTON(self):
         self.assertEqual(BUTTON('test', _type='button').xml(),
@@ -515,8 +541,9 @@ class TestBareHelpers(unittest.TestCase):
                          '<iframe a="1" b="2">&lt;&gt;</iframe>')
 
     def test_INPUT(self):
-        self.assertEqual(INPUT(_a='1', _b='2').xml(),
-                         '<input a="1" b="2" type="text" />')
+        self.assertEqual(INPUT(_a='1', _b='2').xml(), '<input a="1" b="2" type="text" />')
+        # list value
+        self.assertEqual(INPUT(_value=[1, 2, 3]).xml(), '<input type="text" value="[1, 2, 3]" />')
 
     def test_TEXTAREA(self):
         self.assertEqual(TEXTAREA('<>', _a='1', _b='2').xml(),
@@ -560,6 +587,12 @@ class TestBareHelpers(unittest.TestCase):
                                 OPTION('option 2', _value='2'),
                                 _multiple='multiple').xml(),
                          '<select multiple="multiple"><option selected="selected" value="1">option 1</option><option value="2">option 2</option></select>')
+        # More then one select with mutilple
+        self.assertEqual(SELECT(OPTION('option 1', _value='1', _selected='selected'),
+                                OPTION('option 2', _value='2', _selected='selected'),
+                                _multiple='multiple').xml(),
+                         '<select multiple="multiple"><option selected="selected" value="1">option 1</option><option selected="selected" value="2">option 2</option></select>'
+                         )
         # OPTGROUP
         self.assertEqual(SELECT(OPTGROUP(OPTION('option 1', _value='1'),
                                          OPTION('option 2', _value='2'),
@@ -574,6 +607,11 @@ class TestBareHelpers(unittest.TestCase):
         # String value
         self.assertEqual(SELECT('Option 1', 'Option 2').xml(),
                          '<select><option value="Option 1">Option 1</option><option value="Option 2">Option 2</option></select>')
+        # list as a value
+        self.assertEqual(SELECT(OPTION('option 1', _value=[1, 2, 3]),
+                                OPTION('option 2', _value=[4, 5, 6], _selected='selected'),
+                                _multiple='multiple').xml(),
+                         '<select multiple="multiple"><option value="[1, 2, 3]">option 1</option><option selected="selected" value="[4, 5, 6]">option 2</option></select>')
 
     def test_FIELDSET(self):
         self.assertEqual(FIELDSET('<>', _a='1', _b='2').xml(),
@@ -586,17 +624,40 @@ class TestBareHelpers(unittest.TestCase):
     def test_FORM(self):
         self.assertEqual(FORM('<>', _a='1', _b='2').xml(),
                          '<form a="1" action="#" b="2" enctype="multipart/form-data" method="post">&lt;&gt;</form>')
+        # These 2 crash AppVeyor and Travis with: "ImportError: No YAML serializer available"
+        # self.assertEqual(FORM('<>', _a='1', _b='2').as_yaml(),
+        #                  "accepted: null\nattributes: {_a: '1', _action: '#', _b: '2', _enctype: multipart/form-data, _method: post}\ncomponents: [<>]\nerrors: {}\nlatest: {}\nparent: null\nvars: {}\n")
+        # self.assertEqual(FORM('<>', _a='1', _b='2').as_xml(),
+        #                  '<?xml version="1.0" encoding="UTF-8"?><document><errors></errors><vars></vars><parent>None</parent><attributes><_enctype>multipart/form-data</_enctype><_action>#</_action><_b>2</_b><_a>1</_a><_method>post</_method></attributes><components><item>&amp;lt;&amp;gt;</item></components><accepted>None</accepted><latest></latest></document>')
 
     def test_BEAUTIFY(self):
         self.assertEqual(BEAUTIFY(['a', 'b', {'hello': 'world'}]).xml(),
                          '<div><table><tr><td><div>a</div></td></tr><tr><td><div>b</div></td></tr><tr><td><div><table><tr><td style="font-weight:bold;vertical-align:top;">hello</td><td style="vertical-align:top;">:</td><td><div>world</div></td></tr></table></div></td></tr></table></div>')
+        # unicode
+        self.assertEqual(BEAUTIFY([P(u'àéèûôç'), 'a', 'b', {'hello': 'world'}]).xml(),
+                         '<div><table><tr><td><div><p>\xc3\xa0\xc3\xa9\xc3\xa8\xc3\xbb\xc3\xb4\xc3\xa7</p></div></td></tr><tr><td><div>a</div></td></tr><tr><td><div>b</div></td></tr><tr><td><div><table><tr><td style="font-weight:bold;vertical-align:top;">hello</td><td style="vertical-align:top;">:</td><td><div>world</div></td></tr></table></div></td></tr></table></div>')
 
     def test_MENU(self):
         self.assertEqual(MENU([('Home', False, '/welcome/default/index', [])]).xml(),
                          '<ul class="web2py-menu web2py-menu-vertical"><li class="web2py-menu-first"><a href="/welcome/default/index">Home</a></li></ul>')
+        # Multiples entries menu
+        self.assertEqual(MENU([('Home', False, '/welcome/default/index', []),
+                               ('Item 1', False, '/welcome/default/func_one', []),
+                               ('Item 2', False, '/welcome/default/func_two', []),
+                               ('Item 3', False, '/welcome/default/func_three', []),
+                               ('Item 4', False, '/welcome/default/func_four', [])]).xml(),
+                         '<ul class="web2py-menu web2py-menu-vertical"><li class="web2py-menu-first"><a href="/welcome/default/index">Home</a></li><li><a href="/welcome/default/func_one">Item 1</a></li><li><a href="/welcome/default/func_two">Item 2</a></li><li><a href="/welcome/default/func_three">Item 3</a></li><li class="web2py-menu-last"><a href="/welcome/default/func_four">Item 4</a></li></ul>'
+                         )
         # mobile=True
         self.assertEqual(MENU([('Home', False, '/welcome/default/index', [])], mobile=True).xml(),
                          '<select class="web2py-menu web2py-menu-vertical" onchange="window.location=this.value"><option value="/welcome/default/index">Home</option></select>')
+        # Multiples entries menu for mobile
+        self.assertEqual(MENU([('Home', False, '/welcome/default/index', []),
+                               ('Item 1', False, '/welcome/default/func_one', []),
+                               ('Item 2', False, '/welcome/default/func_two', []),
+                               ('Item 3', False, '/welcome/default/func_three', []),
+                               ('Item 4', False, '/welcome/default/func_four', [])], mobile=True).xml(),
+                         '<select class="web2py-menu web2py-menu-vertical" onchange="window.location=this.value"><option value="/welcome/default/index">Home</option><option value="/welcome/default/func_one">Item 1</option><option value="/welcome/default/func_two">Item 2</option><option value="/welcome/default/func_three">Item 3</option><option value="/welcome/default/func_four">Item 4</option></select>')
 
     # TODO: def test_embed64(self):
 

gluon/tests/test_recfile.py

@@ -23,8 +23,8 @@ class TestRecfile(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree('tests')
 
-    def testgeneration(self):
-        for k in range(20):
+    def test_generation(self):
+        for k in range(10):
             teststring = 'test%s' % k
             filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
             with recfile.open(filename, "w") as g:
@@ -35,6 +35,40 @@ class TestRecfile(unittest.TestCase):
             recfile.remove(filename)
             is_there = recfile.exists(filename)
             self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = str(uuid.uuid4()) + '.test'
+            with recfile.open(filename, "w", path='tests') as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r", path='tests').read(), teststring)
+            is_there = recfile.exists(filename, path='tests')
+            self.assertTrue(is_there)
+            recfile.remove(filename, path='tests')
+            is_there = recfile.exists(filename, path='tests')
+            self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = os.path.join('tests', str(uuid.uuid4()), str(uuid.uuid4()) + '.test')
+            with recfile.open(filename, "w") as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r").read(), teststring)
+            is_there = recfile.exists(filename)
+            self.assertTrue(is_there)
+            recfile.remove(filename)
+            is_there = recfile.exists(filename)
+            self.assertFalse(is_there)
+
+    def test_existing(self):
+        filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
+        with open(filename, 'w') as g:
+            g.write('this file exists')
+        self.assertTrue(recfile.exists(filename))
+        self.assertTrue(hasattr(recfile.open(filename, "r"), 'read'))
+        recfile.remove(filename, path='tests')
+        self.assertFalse(recfile.exists(filename))
+        self.assertRaises(IOError, recfile.remove, filename)
+        self.assertRaises(IOError, recfile.open, filename, "r")
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_scheduler.py

@@ -0,0 +1,526 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.scheduler
+"""
+import os
+import unittest
+import glob
+import datetime
+import sys
+
+from fix_path import fix_sys_path
+
+
+fix_sys_path(__file__)
+
+from gluon.storage import Storage
+from gluon.languages import translator
+from gluon.scheduler import JobGraph, Scheduler
+from gluon.dal import DAL
+
+
+class BaseTestScheduler(unittest.TestCase):
+    def setUp(self):
+        self.db = None
+        self.cleanfolder()
+        from gluon import current
+        s = Storage({'application': 'welcome',
+                     'folder': 'applications/welcome',
+                     'controller': 'default'})
+        current.request = s
+        T = translator('', 'en')
+        current.T = T
+        self.db = DAL('sqlite://dummy2.db', check_reserved=['all'])
+
+    def cleanfolder(self):
+        if self.db:
+            self.db.close()
+        try:
+            os.unlink('dummy2.db')
+        except:
+            pass
+        tfiles = glob.glob('*_scheduler*.table')
+        for a in tfiles:
+            os.unlink(a)
+
+    def tearDown(self):
+        self.cleanfolder()
+        try:
+            self.inner_teardown()
+        except:
+            pass
+
+
+class TestsForJobGraph(BaseTestScheduler):
+
+    def testJobGraph(self):
+        s = Scheduler(self.db)
+        myjob = JobGraph(self.db, 'job_1')
+        fname = 'foo'
+        # We have a few items to wear, and there's an "order" to respect...
+        # Items are: watch, jacket, shirt, tie, pants, undershorts, belt, shoes, socks
+        # Now, we can't put on the tie without wearing the shirt first, etc...
+        watch = s.queue_task(fname, task_name='watch')
+        jacket = s.queue_task(fname, task_name='jacket')
+        shirt = s.queue_task(fname, task_name='shirt')
+        tie = s.queue_task(fname, task_name='tie')
+        pants = s.queue_task(fname, task_name='pants')
+        undershorts = s.queue_task(fname, task_name='undershorts')
+        belt = s.queue_task(fname, task_name='belt')
+        shoes = s.queue_task(fname, task_name='shoes')
+        socks = s.queue_task(fname, task_name='socks')
+        # before the tie, comes the shirt
+        myjob.add_deps(tie.id, shirt.id)
+        # before the belt too comes the shirt
+        myjob.add_deps(belt.id, shirt.id)
+        # before the jacket, comes the tie
+        myjob.add_deps(jacket.id, tie.id)
+        # before the belt, come the pants
+        myjob.add_deps(belt.id, pants.id)
+        # before the shoes, comes the pants
+        myjob.add_deps(shoes.id, pants.id)
+        # before the pants, comes the undershorts
+        myjob.add_deps(pants.id, undershorts.id)
+        # before the shoes, comes the undershorts
+        myjob.add_deps(shoes.id, undershorts.id)
+        # before the jacket, comes the belt
+        myjob.add_deps(jacket.id, belt.id)
+        # before the shoes, comes the socks
+        myjob.add_deps(shoes.id, socks.id)
+
+        ## results in the following topological sort
+        # 9,3,6 --> 4,5 --> 8,7 --> 2
+        # socks, shirt, undershorts
+        # tie, pants
+        # shoes, belt
+        # jacket
+        known_toposort = [
+            set([socks.id, shirt.id, undershorts.id]),
+            set([tie.id, pants.id]),
+            set([shoes.id, belt.id]),
+            set([jacket.id])
+        ]
+        toposort = myjob.validate('job_1')
+        self.assertEqual(toposort, known_toposort)
+        # add a cyclic dependency, jacket to undershorts
+        myjob.add_deps(undershorts.id, jacket.id)
+        # no exceptions raised, but result None
+        self.assertEqual(myjob.validate('job_1'), None)
+
+    def testJobGraphFailing(self):
+        s = Scheduler(self.db)
+        myjob = JobGraph(self.db, 'job_1')
+        fname = 'foo'
+        # We have a few items to wear, and there's an "order" to respect...
+        # Items are: watch, jacket, shirt, tie, pants, undershorts, belt, shoes, socks
+        # Now, we can't put on the tie without wearing the shirt first, etc...
+        watch = s.queue_task(fname, task_name='watch')
+        jacket = s.queue_task(fname, task_name='jacket')
+        shirt = s.queue_task(fname, task_name='shirt')
+        tie = s.queue_task(fname, task_name='tie')
+        pants = s.queue_task(fname, task_name='pants')
+        undershorts = s.queue_task(fname, task_name='undershorts')
+        belt = s.queue_task(fname, task_name='belt')
+        shoes = s.queue_task(fname, task_name='shoes')
+        socks = s.queue_task(fname, task_name='socks')
+        # before the tie, comes the shirt
+        myjob.add_deps(tie.id, shirt.id)
+        # before the belt too comes the shirt
+        myjob.add_deps(belt.id, shirt.id)
+        # before the jacket, comes the tie
+        myjob.add_deps(jacket.id, tie.id)
+        # before the belt, come the pants
+        myjob.add_deps(belt.id, pants.id)
+        # before the shoes, comes the pants
+        myjob.add_deps(shoes.id, pants.id)
+        # before the pants, comes the undershorts
+        myjob.add_deps(pants.id, undershorts.id)
+        # before the shoes, comes the undershorts
+        myjob.add_deps(shoes.id, undershorts.id)
+        # before the jacket, comes the belt
+        myjob.add_deps(jacket.id, belt.id)
+        # before the shoes, comes the socks
+        myjob.add_deps(shoes.id, socks.id)
+        # add a cyclic dependency, jacket to undershorts
+        myjob.add_deps(undershorts.id, jacket.id)
+        # no exceptions raised, but result None
+        self.assertEqual(myjob.validate('job_1'), None)
+        # and no deps added
+        deps_inserted = self.db(self.db.scheduler_task_deps.id>0).count()
+        self.assertEqual(deps_inserted, 0)
+
+    def testJobGraphDifferentJobs(self):
+        s = Scheduler(self.db)
+        myjob1 = JobGraph(self.db, 'job_1')
+        myjob2 = JobGraph(self.db, 'job_2')
+        fname = 'foo'
+        # We have a few items to wear, and there's an "order" to respect...
+        # Items are: watch, jacket, shirt, tie, pants, undershorts, belt, shoes, socks
+        # Now, we can't put on the tie without wearing the shirt first, etc...
+        watch = s.queue_task(fname, task_name='watch')
+        jacket = s.queue_task(fname, task_name='jacket')
+        shirt = s.queue_task(fname, task_name='shirt')
+        tie = s.queue_task(fname, task_name='tie')
+        pants = s.queue_task(fname, task_name='pants')
+        undershorts = s.queue_task(fname, task_name='undershorts')
+        belt = s.queue_task(fname, task_name='belt')
+        shoes = s.queue_task(fname, task_name='shoes')
+        socks = s.queue_task(fname, task_name='socks')
+        # before the tie, comes the shirt
+        myjob1.add_deps(tie.id, shirt.id)
+        # before the belt too comes the shirt
+        myjob1.add_deps(belt.id, shirt.id)
+        # before the jacket, comes the tie
+        myjob1.add_deps(jacket.id, tie.id)
+        # before the belt, come the pants
+        myjob1.add_deps(belt.id, pants.id)
+        # before the shoes, comes the pants
+        myjob2.add_deps(shoes.id, pants.id)
+        # before the pants, comes the undershorts
+        myjob2.add_deps(pants.id, undershorts.id)
+        # before the shoes, comes the undershorts
+        myjob2.add_deps(shoes.id, undershorts.id)
+        # before the jacket, comes the belt
+        myjob2.add_deps(jacket.id, belt.id)
+        # before the shoes, comes the socks
+        myjob2.add_deps(shoes.id, socks.id)
+        # every job by itself can be completed
+        self.assertNotEqual(myjob1.validate('job_1'), None)
+        self.assertNotEqual(myjob1.validate('job_2'), None)
+        # and, implicitly, every queued task can be too
+        self.assertNotEqual(myjob1.validate(), None)
+        # add a cyclic dependency, jacket to undershorts
+        myjob2.add_deps(undershorts.id, jacket.id)
+        # every job can still be completed by itself
+        self.assertNotEqual(myjob1.validate('job_1'), None)
+        self.assertNotEqual(myjob1.validate('job_2'), None)
+        # but trying to see if every task will ever be completed fails
+        self.assertEqual(myjob2.validate(), None)
+
+
+class TestsForSchedulerAPIs(BaseTestScheduler):
+
+    def testQueue_Task(self):
+
+        def isnotqueued(result):
+            self.assertEqual(result.id, None)
+            self.assertEqual(result.uuid, None)
+            self.assertEqual(len(result.errors.keys()) > 0, True)
+
+        def isqueued(result):
+            self.assertNotEqual(result.id, None)
+            self.assertNotEqual(result.uuid, None)
+            self.assertEqual(len(result.errors.keys()), 0)
+
+        s = Scheduler(self.db)
+        fname = 'foo'
+        watch = s.queue_task(fname, task_name='watch')
+        # queuing a task returns id, errors, uuid
+        self.assertEqual(set(watch.keys()), set(['id', 'uuid', 'errors']))
+        # queueing nothing isn't allowed
+        self.assertRaises(TypeError, s.queue_task, *[])
+        # passing pargs and pvars wrongly
+        # # pargs as dict
+        isnotqueued(s.queue_task(fname, dict(a=1), dict(b=1)))
+        # # pvars as list
+        isnotqueued(s.queue_task(fname, ['foo', 'bar'], ['foo', 'bar']))
+        # two tasks with the same uuid won't be there
+        isqueued(s.queue_task(fname, uuid='a'))
+        isnotqueued(s.queue_task(fname, uuid='a'))
+        # # #FIXME add here every parameter
+
+    def testTask_Status(self):
+        s = Scheduler(self.db)
+        fname = 'foo'
+        watch = s.queue_task(fname, task_name='watch')
+        # fetch status by id
+        by_id = s.task_status(watch.id)
+        # fetch status by uuid
+        by_uuid = s.task_status(watch.uuid)
+        # fetch status by query
+        by_query = s.task_status(self.db.scheduler_task.function_name == 'foo')
+        self.assertEqual(by_id, by_uuid)
+        self.assertEqual(by_id, by_query)
+        # fetch status by anything else throws
+        self.assertRaises(SyntaxError, s.task_status, *[[1, 2]])
+        # adding output returns the joined set, plus "result"
+        rtn = s.task_status(watch.id, output=True)
+        self.assertEqual(set(rtn.keys()), set(['scheduler_run', 'scheduler_task', 'result']))
+
+
+class testForSchedulerRunnerBase(BaseTestScheduler):
+
+    def inner_teardown(self):
+        from gluon import current
+        fdest = os.path.join(current.request.folder, 'models', 'scheduler.py')
+        os.unlink(fdest)
+        additional_files = [
+            os.path.join(current.request.folder, 'private', 'demo8.pholder')
+        ]
+        for f in additional_files:
+            try:
+                os.unlink(f)
+            except:
+                pass
+
+    def writefunction(self, content, initlines=None):
+        from gluon import current
+        fdest = os.path.join(current.request.folder, 'models', 'scheduler.py')
+        if initlines is None:
+            initlines = """
+import os
+import time
+from gluon.scheduler import Scheduler
+db_dal = os.path.abspath(os.path.join(request.folder, '..', '..', 'dummy2.db'))
+sched_dal = DAL('sqlite://%s' % db_dal, folder=os.path.dirname(db_dal))
+sched = Scheduler(sched_dal, max_empty_runs=15, migrate=False, heartbeat=1)
+            """
+        with open(fdest, 'w') as q:
+            q.write(initlines)
+            q.write(content)
+
+    def exec_sched(self):
+        import subprocess
+        call_args = [sys.executable, 'web2py.py', '--no-banner', '-D', '20','-K', 'welcome']
+        ret = subprocess.call(call_args, env=dict(os.environ))
+        return ret
+
+    def fetch_results(self, sched, task):
+        info = sched.task_status(task.id)
+        task_runs = self.db(self.db.scheduler_run.task_id == task.id).select()
+        return info, task_runs
+
+    def exec_asserts(self, stmts, tag):
+        for stmt in stmts:
+            self.assertEqual(stmt[1], True, msg="%s - %s" % (tag, stmt[0]))
+
+
+class TestsForSchedulerRunner(testForSchedulerRunnerBase):
+
+    def testRepeats_and_Expired_and_Prio(self):
+        s = Scheduler(self.db)
+        repeats = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), repeats=2, period=5)
+        a_while_ago = datetime.datetime.now() - datetime.timedelta(seconds=60)
+        expired = s.queue_task('demo4', stop_time=a_while_ago)
+        prio1 = s.queue_task('demo1', ['scheduled_first'])
+        prio2 = s.queue_task('demo1', ['scheduled_second'], next_run_time=a_while_ago)
+        self.db.commit()
+        self.writefunction(r"""
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # repeats check
+        task, task_run = self.fetch_results(s, repeats)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are COMPLETED ", (task_run[0].status == task_run[1].status == 'COMPLETED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'REPEATS')
+
+        # expired check
+        task, task_run = self.fetch_results(s, expired)
+        res = [
+            ("task status expired", task.status == 'EXPIRED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task didn't run at all", len(task_run) == 0)
+        ]
+        self.exec_asserts(res, 'EXPIRATION')
+
+        # prio check
+        task1 = s.task_status(prio1.id, output=True)
+        task2 = s.task_status(prio2.id, output=True)
+        res = [
+            ("tasks status completed", task1.scheduler_task.status == task2.scheduler_task.status == 'COMPLETED'),
+            ("priority2 was executed before priority1" , task1.scheduler_run.id > task2.scheduler_run.id)
+        ]
+        self.exec_asserts(res, 'PRIORITY')
+
+    def testNoReturn_and_Timeout_and_Progress(self):
+        s = Scheduler(self.db)
+        noret1 = s.queue_task('demo5')
+        noret2 = s.queue_task('demo3')
+        timeout1 = s.queue_task('demo4', timeout=5)
+        timeout2 = s.queue_task('demo4')
+        progress = s.queue_task('demo6', sync_output=2)
+        self.db.commit()
+        self.writefunction(r"""
+def demo3():
+    time.sleep(15)
+    print 1/0
+    return None
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
+
+def demo5():
+    time.sleep(15)
+    print "I'm printing something"
+    rtn = dict(a=1, b=2)
+
+def demo6():
+    time.sleep(5)
+    print '50%'
+    time.sleep(5)
+    print '!clear!100%'
+    return 1
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # noreturn check
+        task1, task_run1 = self.fetch_results(s, noret1)
+        task2, task_run2 = self.fetch_results(s, noret2)
+        res = [
+            ("tasks no_returns1 completed", task1.status == 'COMPLETED'),
+            ("tasks no_returns2 failed", task2.status == 'FAILED'),
+            ("no_returns1 doesn't have a scheduler_run record", len(task_run1) == 0),
+            ("no_returns2 has a scheduler_run record FAILED", (len(task_run2) == 1 and task_run2[0].status == 'FAILED')),
+        ]
+        self.exec_asserts(res, 'NO_RETURN')
+
+        # timeout check
+        task1 = s.task_status(timeout1.id, output=True)
+        task2 = s.task_status(timeout2.id, output=True)
+        res = [
+            ("tasks timeouts1 timeoutted", task1.scheduler_task.status == 'TIMEOUT'),
+            ("tasks timeouts2 completed", task2.scheduler_task.status == 'COMPLETED')
+        ]
+        self.exec_asserts(res, 'TIMEOUT')
+
+        # progress check
+        task1 = s.task_status(progress.id, output=True)
+        res = [
+            ("tasks percentages completed", task1.scheduler_task.status == 'COMPLETED'),
+            ("output contains only 100%", task1.scheduler_run.run_output.strip() == "100%")
+        ]
+        self.exec_asserts(res, 'PROGRESS')
+
+    def testDrift_and_env_and_immediate(self):
+        s = Scheduler(self.db)
+        immediate = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), immediate=True)
+        env = s.queue_task('demo7')
+        drift = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), period=93, prevent_drift=True)
+        self.db.commit()
+        self.writefunction(r"""
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+import random
+def demo7():
+    time.sleep(random.randint(1,5))
+    print W2P_TASK, request.now
+    return W2P_TASK.id, W2P_TASK.uuid, W2P_TASK.run_id
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # immediate check, can only check that nothing breaks
+        task1 = s.task_status(immediate.id)
+        res = [
+            ("tasks status completed", task1.status == 'COMPLETED'),
+        ]
+        self.exec_asserts(res, 'IMMEDIATE')
+
+        # drift check
+        task, task_run = self.fetch_results(s, drift)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("next_run_time is exactly start_time + period", (task.next_run_time == task.start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'DRIFT')
+
+        # env check
+        task1 = s.task_status(env.id, output=True)
+        res = [
+            ("task %s returned W2P_TASK correctly" % (task1.scheduler_task.id),  task1.result == [task1.scheduler_task.id, task1.scheduler_task.uuid, task1.scheduler_run.id]),
+        ]
+        self.exec_asserts(res, 'ENV')
+
+
+    def testRetryFailed(self):
+        s = Scheduler(self.db)
+        failed = s.queue_task('demo2', retry_failed=1, period=1)
+        failed_consecutive = s.queue_task('demo8', retry_failed=2, repeats=2, period=1)
+        self.db.commit()
+        self.writefunction(r"""
+def demo2():
+    1/0
+
+def demo8():
+    placeholder = os.path.join(request.folder, 'private', 'demo8.pholder')
+    with open(placeholder, 'a') as g:
+        g.write('\nplaceholder for demo8 created')
+    num_of_lines = 0
+    with open(placeholder) as f:
+        num_of_lines = len([a for a in f.read().split('\n') if a])
+    print 'number of lines', num_of_lines
+    if num_of_lines <= 2:
+       1/0
+    else:
+        os.unlink(placeholder)
+    return 1
+""")
+        ret = self.exec_sched()
+        # process finished just fine
+        self.assertEqual(ret, 0)
+        # failed - checks
+        task, task_run = self.fetch_results(s, failed)
+        res = [
+            ("task status failed", task.status == 'FAILED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task times_failed is 2", task.times_failed == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are FAILED", (task_run[0].status == task_run[1].status == 'FAILED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'FAILED')
+
+        # failed consecutive - checks
+        task, task_run = self.fetch_results(s, failed_consecutive)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task times_failed is 0", task.times_failed == 0),
+            ("task ran 6 times", len(task_run) == 6),
+            ("scheduler_run records for COMPLETED is 2", len([run.status for run in task_run if run.status == 'COMPLETED']) == 2),
+            ("scheduler_run records for FAILED is 4", len([run.status for run in task_run if run.status == 'FAILED']) == 4),
+        ]
+        self.exec_asserts(res, 'FAILED_CONSECUTIVE')
+
+    def testHugeResult(self):
+        s = Scheduler(self.db)
+        huge_result = s.queue_task('demo10', retry_failed=1, period=1)
+        self.db.commit()
+        self.writefunction(r"""
+def demo10():
+    res = 'a' * 99999
+    return dict(res=res)
+""")
+        ret = self.exec_sched()
+        # process finished just fine
+        self.assertEqual(ret, 0)
+        # huge_result - checks
+        task = s.task_status(huge_result.id, output=True)
+        res = [
+            ("task status completed", task.scheduler_task.status == 'COMPLETED'),
+            ("task times_run is 1", task.scheduler_task.times_run == 1),
+            ("result is the correct one", task.result == dict(res='a' * 99999))
+        ]
+        self.exec_asserts(res, 'HUGE_RESULT')
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_sqlhtml.py

@@ -0,0 +1,439 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.sqlhtml
+"""
+import os
+import sys
+if sys.version < "2.7":
+    import unittest2 as unittest
+else:
+    import unittest
+
+from fix_path import fix_sys_path
+
+fix_sys_path(__file__)
+
+from sqlhtml import safe_int, SQLFORM, SQLTABLE
+
+DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
+
+from gluon.dal import DAL, Field
+from pydal.objects import Table
+from tools import Auth, Mail
+from gluon.globals import Request, Response, Session
+from storage import Storage
+from languages import translator
+from gluon.http import HTTP
+from validators import *
+
+# TODO: Create these test...
+
+# class Test_add_class(unittest.TestCase):
+#     def test_add_class(self):
+#         pass
+
+
+# class Test_represent(unittest.TestCase):
+#     def test_represent(self):
+#         pass
+
+
+# class TestCacheRepresenter(unittest.TestCase):
+#     def test___call__(self):
+#         pass
+
+#     def test___init__(self):
+#         pass
+
+
+class Test_safe_int(unittest.TestCase):
+    def test_safe_int(self):
+        # safe int
+        self.assertEqual(safe_int(1), 1)
+        # not safe int
+        self.assertEqual(safe_int('1x'), 0)
+
+
+# class Test_safe_float(unittest.TestCase):
+#     def test_safe_float(self):
+#         pass
+
+
+# class Test_show_if(unittest.TestCase):
+#     def test_show_if(self):
+#         pass
+
+
+# class TestFormWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestStringWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestIntegerWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestDoubleWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestDecimalWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestDateWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestDatetimeWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestTextWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestJSONWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestBooleanWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestListWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestMultipleOptionsWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestRadioWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestCheckboxesWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestPasswordWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestUploadWidget(unittest.TestCase):
+#     def test__attributes(self):
+#         pass
+
+#     def test_represent(self):
+#         pass
+
+#     def test_widget(self):
+#         pass
+
+
+# class TestAutocompleteWidget(unittest.TestCase):
+#     def test___call__(self):
+#         pass
+
+#     def test___init__(self):
+#         pass
+
+#     def test_callback(self):
+#         pass
+
+
+# class Test_formstyle_table3cols(unittest.TestCase):
+#     def test_formstyle_table3cols(self):
+#         pass
+
+
+# class Test_formstyle_table2cols(unittest.TestCase):
+#     def test_formstyle_table2cols(self):
+#         pass
+
+
+# class Test_formstyle_divs(unittest.TestCase):
+#     def test_formstyle_divs(self):
+#         pass
+
+
+# class Test_formstyle_inline(unittest.TestCase):
+#     def test_formstyle_inline(self):
+#         pass
+
+
+# class Test_formstyle_ul(unittest.TestCase):
+#     def test_formstyle_ul(self):
+#         pass
+
+
+# class Test_formstyle_bootstrap(unittest.TestCase):
+#     def test_formstyle_bootstrap(self):
+#         pass
+
+
+# class Test_formstyle_bootstrap3_stacked(unittest.TestCase):
+#     def test_formstyle_bootstrap3_stacked(self):
+#         pass
+
+
+# class Test_formstyle_bootstrap3_inline_factory(unittest.TestCase):
+#     def test_formstyle_bootstrap3_inline_factory(self):
+#         pass
+
+
+class TestSQLFORM(unittest.TestCase):
+
+    def setUp(self):
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        T = translator('', 'en')
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        current.T = T
+        self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+        self.auth = Auth(self.db)
+        self.auth.define_tables(username=True, signature=False)
+        self.db.define_table('t0', Field('tt'), self.auth.signature)
+        self.auth.enable_record_versioning(self.db)
+        # Create a user
+        self.db.auth_user.insert(first_name='Bart',
+                                 last_name='Simpson',
+                                 username='user1',
+                                 email='user1@test.com',
+                                 password='password_123',
+                                 registration_key=None,
+                                 registration_id=None)
+
+        self.db.commit()
+
+    def test_SQLFORM(self):
+        form = SQLFORM(self.db.auth_user)
+        self.assertEqual(form.xml(), '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="auth_user_first_name__row"><td class="w2p_fl"><label class="" for="auth_user_first_name" id="auth_user_first_name__label">First name: </label></td><td class="w2p_fw"><input class="string" id="auth_user_first_name" name="first_name" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="auth_user_last_name__row"><td class="w2p_fl"><label class="" for="auth_user_last_name" id="auth_user_last_name__label">Last name: </label></td><td class="w2p_fw"><input class="string" id="auth_user_last_name" name="last_name" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="auth_user_email__row"><td class="w2p_fl"><label class="" for="auth_user_email" id="auth_user_email__label">E-mail: </label></td><td class="w2p_fw"><input class="string" id="auth_user_email" name="email" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="auth_user_username__row"><td class="w2p_fl"><label class="" for="auth_user_username" id="auth_user_username__label">Username: </label></td><td class="w2p_fw"><input class="string" id="auth_user_username" name="username" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="auth_user_password__row"><td class="w2p_fl"><label class="" for="auth_user_password" id="auth_user_password__label">Password: </label></td><td class="w2p_fw"><input class="password" id="auth_user_password" name="password" type="password" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
+
+    # def test_assert_status(self):
+    #     pass
+
+    #  def test_createform(self):
+    #     pass
+
+    #  def test_accepts(self):
+    #     pass
+
+    #  def test_dictform(self):
+    #     pass
+
+    #  def test_smartdictform(self):
+    #     pass
+
+    def test_factory(self):
+        factory_form = SQLFORM.factory(Field('field_one', 'string', IS_NOT_EMPTY()),
+                                       Field('field_two', 'string'))
+        self.assertEqual(factory_form.xml(), '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="no_table_field_one__row"><td class="w2p_fl"><label class="" for="no_table_field_one" id="no_table_field_one__label">Field One: </label></td><td class="w2p_fw"><input class="string" id="no_table_field_one" name="field_one" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="no_table_field_two__row"><td class="w2p_fl"><label class="" for="no_table_field_two" id="no_table_field_two__label">Field Two: </label></td><td class="w2p_fw"><input class="string" id="no_table_field_two" name="field_two" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
+
+    #  def test_build_query(self):
+    #     pass
+
+    #  def test_search_menu(self):
+    #     pass
+
+    def test_grid(self):
+        grid_form = SQLFORM.grid(self.db.auth_user)
+        self.assertEqual(grid_form.xml(), '<div class="web2py_grid "><div class="web2py_console  "><form action="/a/c/f" enctype="multipart/form-data" method="GET"><input class="form-control" id="w2p_keywords" name="keywords" onfocus="jQuery(&#x27;#w2p_query_fields&#x27;).change();jQuery(&#x27;#w2p_query_panel&#x27;).slideDown();" type="text" value="" /><input class="btn btn-default" type="submit" value="Search" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_keywords&#x27;).val(&#x27;&#x27;);" type="submit" value="Clear" /></form><div id="w2p_query_panel" style="display:none;"><select class="form-control" id="w2p_query_fields" onchange="jQuery(&#x27;.w2p_query_row&#x27;).hide();jQuery(&#x27;#w2p_field_&#x27;+jQuery(&#x27;#w2p_query_fields&#x27;).val().replace(&#x27;.&#x27;,&#x27;-&#x27;)).show();" style="float:left"><option value="auth_user.id">Id</option><option value="auth_user.first_name">First name</option><option value="auth_user.last_name">Last name</option><option value="auth_user.email">E-mail</option><option value="auth_user.username">Username</option></select><div class="w2p_query_row" id="w2p_field_auth_user-id" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="in">in</option><option value="not in">not in</option></select><input class="id form-control" id="w2p_value_auth_user-id" type="text" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;new&#x27;,&#x27;auth_user.id&#x27;)" title="Start building a new search" type="button" value="New Search" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;and&#x27;,&#x27;auth_user.id&#x27;)" title="Add this to the search as an AND term" type="button" value="+ And" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;or&#x27;,&#x27;auth_user.id&#x27;)" title="Add this to the search as an OR term" type="button" value="+ Or" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_query_panel&#x27;).slideUp()" type="button" value="Close" /></div><div class="w2p_query_row" id="w2p_field_auth_user-first_name" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="starts with">starts with</option><option value="contains">contains</option><option value="in">in</option><option value="not in">not in</option></select><input class="string form-control" id="w2p_value_auth_user-first_name" type="text" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;new&#x27;,&#x27;auth_user.first_name&#x27;)" title="Start building a new search" type="button" value="New Search" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;and&#x27;,&#x27;auth_user.first_name&#x27;)" title="Add this to the search as an AND term" type="button" value="+ And" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;or&#x27;,&#x27;auth_user.first_name&#x27;)" title="Add this to the search as an OR term" type="button" value="+ Or" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_query_panel&#x27;).slideUp()" type="button" value="Close" /></div><div class="w2p_query_row" id="w2p_field_auth_user-last_name" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="starts with">starts with</option><option value="contains">contains</option><option value="in">in</option><option value="not in">not in</option></select><input class="string form-control" id="w2p_value_auth_user-last_name" type="text" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;new&#x27;,&#x27;auth_user.last_name&#x27;)" title="Start building a new search" type="button" value="New Search" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;and&#x27;,&#x27;auth_user.last_name&#x27;)" title="Add this to the search as an AND term" type="button" value="+ And" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;or&#x27;,&#x27;auth_user.last_name&#x27;)" title="Add this to the search as an OR term" type="button" value="+ Or" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_query_panel&#x27;).slideUp()" type="button" value="Close" /></div><div class="w2p_query_row" id="w2p_field_auth_user-email" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="starts with">starts with</option><Line truncated
+
+    def test_smartgrid(self):
+        smartgrid_form = SQLFORM.smartgrid(self.db.auth_user)
+        self.assertEqual(smartgrid_form.xml(), '<div class="web2py_grid "><div class="web2py_breadcrumbs"><ul class=""><li class="active w2p_grid_breadcrumb_elem"><a href="/a/c/f/auth_user">Auth users</a></li></ul></div><div class="web2py_console  "><form action="/a/c/f/auth_user" enctype="multipart/form-data" method="GET"><input class="form-control" id="w2p_keywords" name="keywords" onfocus="jQuery(&#x27;#w2p_query_fields&#x27;).change();jQuery(&#x27;#w2p_query_panel&#x27;).slideDown();" type="text" value="" /><input class="btn btn-default" type="submit" value="Search" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_keywords&#x27;).val(&#x27;&#x27;);" type="submit" value="Clear" /></form><div id="w2p_query_panel" style="display:none;"><select class="form-control" id="w2p_query_fields" onchange="jQuery(&#x27;.w2p_query_row&#x27;).hide();jQuery(&#x27;#w2p_field_&#x27;+jQuery(&#x27;#w2p_query_fields&#x27;).val().replace(&#x27;.&#x27;,&#x27;-&#x27;)).show();" style="float:left"><option value="auth_user.id">Id</option><option value="auth_user.first_name">First name</option><option value="auth_user.last_name">Last name</option><option value="auth_user.email">E-mail</option><option value="auth_user.username">Username</option></select><div class="w2p_query_row" id="w2p_field_auth_user-id" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="in">in</option><option value="not in">not in</option></select><input class="id form-control" id="w2p_value_auth_user-id" type="text" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;new&#x27;,&#x27;auth_user.id&#x27;)" title="Start building a new search" type="button" value="New Search" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;and&#x27;,&#x27;auth_user.id&#x27;)" title="Add this to the search as an AND term" type="button" value="+ And" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;or&#x27;,&#x27;auth_user.id&#x27;)" title="Add this to the search as an OR term" type="button" value="+ Or" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_query_panel&#x27;).slideUp()" type="button" value="Close" /></div><div class="w2p_query_row" id="w2p_field_auth_user-first_name" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="starts with">starts with</option><option value="contains">contains</option><option value="in">in</option><option value="not in">not in</option></select><input class="string form-control" id="w2p_value_auth_user-first_name" type="text" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;new&#x27;,&#x27;auth_user.first_name&#x27;)" title="Start building a new search" type="button" value="New Search" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;and&#x27;,&#x27;auth_user.first_name&#x27;)" title="Add this to the search as an AND term" type="button" value="+ And" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;or&#x27;,&#x27;auth_user.first_name&#x27;)" title="Add this to the search as an OR term" type="button" value="+ Or" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_query_panel&#x27;).slideUp()" type="button" value="Close" /></div><div class="w2p_query_row" id="w2p_field_auth_user-last_name" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</option><option value="&gt;">&gt;</option><option value="&lt;=">&lt;=</option><option value="&gt;=">&gt;=</option><option value="starts with">starts with</option><option value="contains">contains</option><option value="in">in</option><option value="not in">not in</option></select><input class="string form-control" id="w2p_value_auth_user-last_name" type="text" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;new&#x27;,&#x27;auth_user.last_name&#x27;)" title="Start building a new search" type="button" value="New Search" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;and&#x27;,&#x27;auth_user.last_name&#x27;)" title="Add this to the search as an AND term" type="button" value="+ And" /><input class="btn btn-default" onclick="w2p_build_query(&#x27;or&#x27;,&#x27;auth_user.last_name&#x27;)" title="Add this to the search as an OR term" type="button" value="+ Or" /><input class="btn btn-default" onclick="jQuery(&#x27;#w2p_query_panel&#x27;).slideUp()" type="button" value="Close" /></div><div class="w2p_query_row" id="w2p_field_auth_user-email" style="display:none"><select class="form-control"><option value="=">=</option><option value="!=">!=</option><option value="&lt;">&lt;</oLine truncated
+
+
+class TestSQLTABLE(unittest.TestCase):
+    def setUp(self):
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        T = translator('', 'en')
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        current.T = T
+        self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+        self.auth = Auth(self.db)
+        self.auth.define_tables(username=True, signature=False)
+        self.db.define_table('t0', Field('tt'), self.auth.signature)
+        self.auth.enable_record_versioning(self.db)
+        # Create a user
+        self.db.auth_user.insert(first_name='Bart',
+                                 last_name='Simpson',
+                                 username='user1',
+                                 email='user1@test.com',
+                                 password='password_123',
+                                 registration_key=None,
+                                 registration_id=None)
+
+        self.db.commit()
+
+    def test_SQLTABLE(self):
+        rows = self.db(self.db.auth_user.id > 0).select(self.db.auth_user.ALL)
+        sqltable = SQLTABLE(rows)
+        self.assertEqual(sqltable.xml(), '<table><thead><tr><th>auth_user.id</th><th>auth_user.first_name</th><th>auth_user.last_name</th><th>auth_user.email</th><th>auth_user.username</th><th>auth_user.password</th><th>auth_user.registration_key</th><th>auth_user.reset_password_key</th><th>auth_user.registration_id</th></tr></thead><tbody><tr class="w2p_odd odd"><td>1</td><td>Bart</td><td>Simpson</td><td>user1@test.com</td><td>user1</td><td>password_123</td><td>None</td><td></td><td>None</td></tr></tbody></table>')
+
+
+# class TestExportClass(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+# class TestExporterTSV(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+# class TestExporterCSV(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+# class TestExporterCSV_hidden(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+# class TestExporterHTML(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+# class TestExporterXML(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+# class TestExporterJSON(unittest.TestCase):
+#     def test___init__(self):
+#         pass
+
+#     def test_export(self):
+#         pass
+
+#     def test_represented(self):
+#         pass
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_template.py

@@ -9,10 +9,11 @@ from fix_path import fix_sys_path
 
 fix_sys_path(__file__)
 
+import template
 from template import render
 
 
-class TestVirtualFields(unittest.TestCase):
+class TestTemplate(unittest.TestCase):
 
     def testRun(self):
         self.assertEqual(render(content='{{for i in range(n):}}{{=i}}{{pass}}',
@@ -61,6 +62,80 @@ class TestVirtualFields(unittest.TestCase):
         self.assertRaises(
             SyntaxError, render, content='{{pass\n=list((1,2,\n3))}}')
 
+    def testWithDummyFileSystem(self):
+        from os.path import join as pjoin
+        import contextlib
+        from StringIO import StringIO
+        from gluon.restricted import RestrictedError
+
+        @contextlib.contextmanager
+        def monkey_patch(module, fn_name, patch):
+            try:
+                unpatch = getattr(module, fn_name)
+            except AttributeError:
+                unpatch = None
+            setattr(module, fn_name, patch)
+            try:
+                yield
+            finally:
+                if unpatch is None:
+                    delattr(module, fn_name)
+                else:
+                    setattr(module, fn_name, unpatch)
+
+        def dummy_open(path, mode):
+            if path == pjoin('views', 'layout.html'):
+                return StringIO("{{block left_sidebar}}left{{end}}"
+                                "{{include}}"
+                                "{{block right_sidebar}}right{{end}}")
+            elif path == pjoin('views', 'layoutbrackets.html'):
+                return StringIO("[[block left_sidebar]]left[[end]]"
+                                "[[include]]"
+                                "[[block right_sidebar]]right[[end]]")
+            elif path == pjoin('views', 'default', 'index.html'):
+                return StringIO("{{extend 'layout.html'}}"
+                                "{{block left_sidebar}}{{super}} {{end}}"
+                                "to"
+                                "{{block right_sidebar}} {{super}}{{end}}")
+            elif path == pjoin('views', 'default', 'indexbrackets.html'):
+                return StringIO("[[extend 'layoutbrackets.html']]"
+                                "[[block left_sidebar]][[super]] [[end]]"
+                                "to"
+                                "[[block right_sidebar]] [[super]][[end]]")
+            elif path == pjoin('views', 'default', 'missing.html'):
+                return StringIO("{{extend 'wut'}}"
+                                "{{block left_sidebar}}{{super}} {{end}}"
+                                "to"
+                                "{{block right_sidebar}} {{super}}{{end}}")
+            elif path == pjoin('views', 'default', 'noescape.html'):
+                return StringIO("""{{=NOESCAPE('<script></script>')}}""")
+            raise IOError
+
+        with monkey_patch(template, 'open', dummy_open):
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'index.html'),
+                       path='views'),
+                'left to right')
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'indexbrackets.html'),
+                       path='views', delimiters=('[[', ']]')),
+                'left to right')
+            self.assertRaises(
+                RestrictedError,
+                render,
+                filename=pjoin('views', 'default', 'missing.html'),
+                path='views')
+            response = template.DummyResponse()
+            response.delimiters = ('[[', ']]')
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'indexbrackets.html'),
+                       path='views', context={'response': response}),
+                'left to right')
+            self.assertEqual(
+                render(filename=pjoin('views', 'default', 'noescape.html'),
+                       context={'NOESCAPE': template.NOESCAPE}),
+                '<script></script>')
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_tools.py

@@ -7,6 +7,7 @@
 import os
 import sys
 import smtplib
+import datetime
 if sys.version < "2.7":
     import unittest2 as unittest
 else:
@@ -20,7 +21,7 @@ DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
 from gluon.dal import DAL, Field
 from pydal.objects import Table
-from tools import Auth, Mail
+from tools import Auth, Mail, Recaptcha, Recaptcha2, prettydate
 from gluon.globals import Request, Response, Session
 from storage import Storage
 from languages import translator
@@ -29,57 +30,6 @@ from gluon.http import HTTP
 python_version = sys.version[:3]
 IS_IMAP = "imap" in DEFAULT_URI
 
-@unittest.skipIf(IS_IMAP, "TODO: Imap raises 'Connection refused'")
-class TestAuth(unittest.TestCase):
-
-    def testRun(self):
-        # setup
-        request = Request(env={})
-        request.application = 'a'
-        request.controller = 'c'
-        request.function = 'f'
-        request.folder = 'applications/admin'
-        response = Response()
-        session = Session()
-        T = translator('', 'en')
-        session.connect(request, response)
-        from gluon.globals import current
-        current.request = request
-        current.response = response
-        current.session = session
-        current.T = T
-        db = DAL(DEFAULT_URI, check_reserved=['all'])
-        auth = Auth(db)
-        auth.define_tables(username=True, signature=False)
-        self.assertTrue('auth_user' in db)
-        self.assertTrue('auth_group' in db)
-        self.assertTrue('auth_membership' in db)
-        self.assertTrue('auth_permission' in db)
-        self.assertTrue('auth_event' in db)
-        db.define_table('t0', Field('tt'), auth.signature)
-        auth.enable_record_versioning(db)
-        self.assertTrue('t0_archive' in db)
-        for f in ['login', 'register', 'retrieve_password',
-                  'retrieve_username']:
-            html_form = getattr(auth, f)().xml()
-            self.assertTrue('name="_formkey"' in html_form)
-
-        for f in ['logout', 'verify_email', 'reset_password',
-                  'change_password', 'profile', 'groups']:
-            self.assertRaisesRegexp(HTTP, "303*", getattr(auth, f))
-
-        self.assertRaisesRegexp(HTTP, "401*", auth.impersonate)
-
-        try:
-            for t in ['t0_archive', 't0', 'auth_cas', 'auth_event',
-                      'auth_membership', 'auth_permission', 'auth_group',
-                      'auth_user']:
-                db[t].drop()
-        except SyntaxError as e:
-            # GAE doesn't support drop
-            pass
-        return
-
 
 class TestMail(unittest.TestCase):
     """
@@ -87,10 +37,19 @@ class TestMail(unittest.TestCase):
     """
 
     class Message(object):
+
         def __init__(self, sender, to, payload):
             self.sender = sender
             self.to = to
             self.payload = payload
+            self._parsed_payload = None
+
+        @property
+        def parsed_payload(self):
+            if self._parsed_payload is None:
+                import email
+                self._parsed_payload = email.message_from_string(self.payload)
+            return self._parsed_payload
 
     class DummySMTP(object):
         """
@@ -125,7 +84,6 @@ class TestMail(unittest.TestCase):
         def starttls(self):
             self.tls = True
 
-
     def setUp(self):
         self.original_SMTP = smtplib.SMTP
         self.original_SMTP_SSL = smtplib.SMTP_SSL
@@ -189,6 +147,19 @@ class TestMail(unittest.TestCase):
         message = TestMail.DummySMTP.inbox.pop()
         self.assertTrue('Content-Type: text/html' in message.payload)
 
+    def test_alternative(self):
+        mail = Mail()
+        mail.settings.server = 'smtp.example.com:25'
+        mail.settings.sender = 'you@example.com'
+        self.assertTrue(mail.send(to=['somebody@example.com'],
+                                  message=('Text only', '<html><pre>HTML Only</pre></html>')))
+        message = TestMail.DummySMTP.inbox.pop()
+        self.assertTrue(message.parsed_payload.is_multipart())
+        self.assertTrue(message.parsed_payload.get_content_type() == 'multipart/alternative')
+        parts = message.parsed_payload.get_payload()
+        self.assertTrue('Text only' in parts[0].as_string())
+        self.assertTrue('<html><pre>HTML Only</pre></html>' in parts[1].as_string())
+
     def test_ssl(self):
         mail = Mail()
         mail.settings.server = 'smtp.example.com:25'
@@ -213,9 +184,855 @@ class TestMail(unittest.TestCase):
                                   message='world'))
         TestMail.DummySMTP.inbox.pop()
 
+    def test_attachment(self):
+        module_file = os.path.abspath(__file__)
+        mail = Mail()
+        mail.settings.server = 'smtp.example.com:25'
+        mail.settings.sender = 'you@example.com'
+        self.assertTrue(mail.send(to=['somebody@example.com'],
+                                  subject='hello',
+                                  message='world',
+                                  attachments=Mail.Attachment(module_file)))
+        message = TestMail.DummySMTP.inbox.pop()
+        attachment = message.parsed_payload.get_payload(1).get_payload(decode=True)
+        with open(module_file, 'rb') as mf:
+            self.assertEqual(attachment.decode('utf-8'), mf.read().decode('utf-8'))
+        # Test missing attachment name error
+        stream = open(module_file)
+        self.assertRaises(Exception, lambda *args, **kwargs: Mail.Attachment(*args, **kwargs), stream)
+        stream.close()
+        # Test you can define content-id and content type
+        self.assertTrue(mail.send(to=['somebody@example.com'],
+                                  subject='hello',
+                                  message='world',
+                                  attachments=Mail.Attachment(module_file, content_id='trololo', content_type='tra/lala')))
+        message = TestMail.DummySMTP.inbox.pop()
+        self.assertTrue('Content-Type: tra/lala' in message.payload)
+        self.assertTrue('Content-Id: <trololo>' in message.payload)
 
 
+# class TestRecaptcha(unittest.TestCase):
+#     def test_Recaptcha(self):
+#         from html import FORM
+#         form = FORM(Recaptcha(public_key='public_key', private_key='private_key'))
+#         self.assertEqual(form.xml(),
+#                          '<form action="#" enctype="multipart/form-data" method="post"><div id="recaptcha"><script><!--\nvar RecaptchaOptions = {};\n//--></script><script src="http://www.google.com/recaptcha/api/challenge?k=public_key" type="text/javascript"></script><noscript><iframe frameborder="0" height="300" src="http://www.google.com/recaptcha/api/noscript?k=public_key" width="500"></iframe><br /><input name="recaptcha_response_field" type="hidden" value="manual_challenge" /></noscript></div></form>')
+#
+#
+# class TestRecaptcha2(unittest.TestCase):
+#     def test_Recaptcha2(self):
+#         from html import FORM
+#         form = FORM(Recaptcha2(public_key='public_key', private_key='private_key'))
+#         rtn = '<form action="#" enctype="multipart/form-data" method="post"><div><script async="" defer="" src="https://www.google.com/recaptcha/api.js"></script><div class="g-recaptcha" data-sitekey="public_key"></div><noscript>\n<div style="width: 302px; height: 352px;">\n<div style="width: 302px; height: 352px; position: relative;">\n  <div style="width: 302px; height: 352px; position: absolute;">\n    <iframe src="https://www.google.com/recaptcha/api/fallback?k=public_key"\n            frameborder="0" scrolling="no"\n            style="width: 302px; height:352px; border-style: none;">\n    </iframe>\n  </div>\n  <div style="width: 250px; height: 80px; position: absolute; border-style: none;\n              bottom: 21px; left: 25px; margin: 0px; padding: 0px; right: 25px;">\n    <textarea id="g-recaptcha-response" name="g-recaptcha-response"\n              class="g-recaptcha-response"\n              style="width: 250px; height: 80px; border: 1px solid #c1c1c1;\n                     margin: 0px; padding: 0px; resize: none;" value="">\n    </textarea>\n  </div>\n</div>\n</div></noscript></div></form>'
+#         self.assertEqual(form.xml(), rtn)
 
+# TODO: class TestAuthJWT(unittest.TestCase):
+
+
+@unittest.skipIf(IS_IMAP, "TODO: Imap raises 'Connection refused'")
+# class TestAuth(unittest.TestCase):
+#
+#     def setUp(self):
+#         request = Request(env={})
+#         request.application = 'a'
+#         request.controller = 'c'
+#         request.function = 'f'
+#         request.folder = 'applications/admin'
+#         response = Response()
+#         session = Session()
+#         T = translator('', 'en')
+#         session.connect(request, response)
+#         from gluon.globals import current
+#         current.request = request
+#         current.response = response
+#         current.session = session
+#         current.T = T
+#         self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+#         self.auth = Auth(self.db)
+#         self.auth.define_tables(username=True, signature=False)
+#         self.db.define_table('t0', Field('tt'), self.auth.signature)
+#         self.auth.enable_record_versioning(self.db)
+#         # Create a user
+#         self.auth.get_or_create_user(dict(first_name='Bart',
+#                                           last_name='Simpson',
+#                                           username='bart',
+#                                           email='bart@simpson.com',
+#                                           password='bart_password',
+#                                           registration_key='bart',
+#                                           registration_id=''
+#                                           ))
+#         # self.auth.settings.registration_requires_verification = False
+#         # self.auth.settings.registration_requires_approval = False
+#
+#     def test_assert_setup(self):
+#         self.assertEqual(self.db(self.db.auth_user.username == 'bart').select().first()['username'], 'bart')
+#         self.assertTrue('auth_user' in self.db)
+#         self.assertTrue('auth_group' in self.db)
+#         self.assertTrue('auth_membership' in self.db)
+#         self.assertTrue('auth_permission' in self.db)
+#         self.assertTrue('auth_event' in self.db)
+#
+#     def test_enable_record_versioning(self):
+#         self.assertTrue('t0_archive' in self.db)
+#
+#     def test_basic_blank_forms(self):
+#         for f in ['login', 'retrieve_password',
+#                   'retrieve_username',
+#                   # 'register'  # register complain about : client_side=self.settings.client_side
+#                   ]:
+#             html_form = getattr(self.auth, f)().xml()
+#             self.assertTrue('name="_formkey"' in html_form)
+#
+#         # NOTE: Not sure it is the proper way to logout_bare() as there is not methods for that and auth.logout() failed
+#         self.auth.logout_bare()
+#         # self.assertTrue(self.auth.is_logged_in())
+#
+#         for f in ['logout', 'verify_email', 'reset_password',
+#                   'change_password', 'profile', 'groups']:
+#             self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
+#
+#         self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate)
+#
+#         try:
+#             for t in ['t0_archive', 't0', 'auth_cas', 'auth_event',
+#                       'auth_membership', 'auth_permission', 'auth_group',
+#                       'auth_user']:
+#                 self.db[t].drop()
+#         except SyntaxError as e:
+#             # GAE doesn't support drop
+#             pass
+#         return
+#
+#     def test_get_or_create_user(self):
+#         self.db.auth_user.insert(email='user1@test.com', username='user1', password='password_123')
+#         self.db.commit()
+#         # True case
+#         self.assertEqual(self.auth.get_or_create_user({'email': 'user1@test.com',
+#                                                        'username': 'user1',
+#                                                        'password': 'password_123'
+#                                                        })['username'], 'user1')
+#         # user2 doesn't exist yet and get created
+#         self.assertEqual(self.auth.get_or_create_user({'email': 'user2@test.com',
+#                                                        'username': 'user2'})['username'], 'user2')
+#         # user3 for corner case
+#         self.assertEqual(self.auth.get_or_create_user({'first_name': 'Omer',
+#                                                        'last_name': 'Simpson',
+#                                                        'email': 'user3@test.com',
+#                                                        'registration_id': 'user3',
+#                                                        'username': 'user3'})['username'], 'user3')
+#         # False case
+#         self.assertEqual(self.auth.get_or_create_user({'email': ''}), None)
+#         self.db.auth_user.truncate()
+#         self.db.commit()
+#
+#     def test_login_bare(self):
+#         # The following test case should succeed but failed as I never received the user record but False
+#         self.auth.login_bare(username='bart@simpson.com', password='bart_password')
+#         self.assertTrue(self.auth.is_logged_in())
+#         # Failing login because bad_password
+#         self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+#         self.db.auth_user.truncate()
+#
+#     def test_register_bare(self):
+#         # corner case empty register call register_bare without args
+#         self.assertRaises(ValueError, self.auth.register_bare)
+#         # failing register_bare user already exist
+#         self.assertEqual(self.auth.register_bare(username='bart', password='wrong_password'), False)
+#         # successful register_bare
+#         self.assertEqual(self.auth.register_bare(username='user2',
+#                                                  email='user2@test.com',
+#                                                  password='password_123')['username'], 'user2')
+#         # raise ValueError
+#         self.assertRaises(ValueError, self.auth.register_bare,
+#                           **dict(wrong_field_name='user3', password='password_123'))
+#         # raise ValueError wrong email
+#         self.assertRaises(ValueError, self.auth.register_bare,
+#                           **dict(email='user4@', password='password_123'))
+#         self.db.auth_user.truncate()
+#         self.db.commit()
+#
+#     def test_bulk_register(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         self.auth.settings.bulk_register_enabled = True
+#         bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
+#         self.assertTrue('name="_formkey"' in bulk_register_form)
+#
+#     def test_change_password(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         change_password_form = getattr(self.auth, 'change_password')().xml()
+#         self.assertTrue('name="_formkey"' in change_password_form)
+#
+#     def test_profile(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         profile_form = getattr(self.auth, 'profile')().xml()
+#         self.assertTrue('name="_formkey"' in profile_form)
+#
+#     # def test_impersonate(self):
+#     #     # Create a user to be impersonated
+#     #     self.auth.get_or_create_user(dict(first_name='Omer',
+#     #                                       last_name='Simpson',
+#     #                                       username='omer',
+#     #                                       email='omer@test.com',
+#     #                                       password='password_omer',
+#     #                                       registration_key='',
+#     #                                       registration_id=''))
+#     #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+#     #     self.auth.add_group('impersonate')
+#     #     self.auth.add_membership(user_id=1,
+#     #                              group_id=self.db(self.db.auth_user.username == 'bart'
+#     #                                               ).select(self.db.auth_user.id).first().id)
+#     #     self.auth.add_permission(group_id=self.db(self.db.auth_group.role == 'impersonate'
+#     #                                               ).select(self.db.auth_group.id).first().id,
+#     #                              name='impersonate',
+#     #                              table_name='auth_user',
+#     #                              record_id=0)
+#     #     # Bart login
+#     #     self.auth.login_bare(username='bart', password='bart_password')
+#     #     self.assertTrue(self.auth.is_logged_in())
+#     #     # Bart impersonate Omer
+#     #     omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
+#     #     impersonate_form = self.auth.impersonate(user_id=omer_id)
+#     #     self.assertTrue(self.auth.is_impersonating())
+#     #     self.assertEqual(impersonate_form, 'test')
+#
+#     # def test_impersonate(self):
+#     #     request = Request(env={})
+#     #     request.application = 'a'
+#     #     request.controller = 'c'
+#     #     request.function = 'f'
+#     #     request.folder = 'applications/admin'
+#     #     response = Response()
+#     #     session = Session()
+#     #     T = translator('', 'en')
+#     #     session.connect(request, response)
+#     #     from gluon.globals import current
+#     #     current.request = request
+#     #     current.response = response
+#     #     current.session = session
+#     #     current.T = T
+#     #     db = DAL(DEFAULT_URI, check_reserved=['all'])
+#     #     auth = Auth(db)
+#     #     auth.define_tables(username=True, signature=False)
+#     #     db.define_table('t0', Field('tt'), auth.signature)
+#     #     auth.enable_record_versioning(db)
+#     #     # Create a user
+#     #     auth.get_or_create_user(dict(first_name='Bart',
+#     #                                       last_name='Simpson',
+#     #                                       username='bart',
+#     #                                       email='bart@simpson.com',
+#     #                                       password='bart_password',
+#     #                                       registration_key='bart',
+#     #                                       registration_id=''
+#     #                                       ))
+#     #     # Create a user to be impersonated
+#     #     auth.get_or_create_user(dict(first_name='Omer',
+#     #                                       last_name='Simpson',
+#     #                                       username='omer',
+#     #                                       email='omer@test.com',
+#     #                                       password='password_omer',
+#     #                                       registration_key='',
+#     #                                       registration_id=''))
+#     #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+#     #     auth.add_group('impersonate')
+#     #     auth.add_membership(user_id=1,
+#     #                              group_id=db(db.auth_user.username == 'bart'
+#     #                                               ).select(db.auth_user.id).first().id)
+#     #     auth.add_permission(group_id=db(db.auth_group.role == 'impersonate'
+#     #                                               ).select(db.auth_group.id).first().id,
+#     #                              name='impersonate',
+#     #                              table_name='auth_user',
+#     #                              record_id=0)
+#     #     # Bart login
+#     #     auth.login_bare(username='bart', password='bart_password')
+#     #     # Bart impersonate Omer
+#     #     omer_id = db(db.auth_user.username == 'omer').select(db.auth_user.id).first().id
+#     #     impersonate_form = auth.impersonate(user_id=omer_id)
+#     #     self.assertTrue(auth.is_impersonating())
+#     #     self.assertEqual(impersonate_form, 'test')
+class TestAuth(unittest.TestCase):
+    def setUp(self):
+        self.request = Request(env={})
+        self.request.application = 'a'
+        self.request.controller = 'c'
+        self.request.function = 'f'
+        self.request.folder = 'applications/admin'
+        self.response = Response()
+        self.session = Session()
+        T = translator('', 'en')
+        self.session.connect(self.request, self.response)
+        from gluon.globals import current
+        self.current = current
+        self.current.request = self.request
+        self.current.response = self.response
+        self.current.session = self.session
+        self.current.T = T
+        self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+        self.auth = Auth(self.db)
+        self.auth.define_tables(username=True, signature=False)
+        self.db.define_table('t0', Field('tt'), self.auth.signature)
+        self.auth.enable_record_versioning(self.db)
+        # Create a user
+        self.auth.get_or_create_user(dict(first_name='Bart',
+                                          last_name='Simpson',
+                                          username='bart',
+                                          email='bart@simpson.com',
+                                          password='bart_password',
+                                          registration_key='bart',
+                                          registration_id=''
+                                          ),
+                                     login=False)
+        self.db.commit()
+        self.assertFalse(self.auth.is_logged_in())
+        # self.auth.settings.registration_requires_verification = False
+        # self.auth.settings.registration_requires_approval = False
+
+    def test_assert_setup(self):
+        self.assertEqual(self.db(self.db.auth_user.username == 'bart').select().first()['username'], 'bart')
+        self.assertTrue('auth_user' in self.db)
+        self.assertTrue('auth_group' in self.db)
+        self.assertTrue('auth_membership' in self.db)
+        self.assertTrue('auth_permission' in self.db)
+        self.assertTrue('auth_event' in self.db)
+
+    # Just calling many form functions
+    def test_basic_blank_forms(self):
+        for f in ['login', 'retrieve_password', 'retrieve_username', 'register']:
+            html_form = getattr(self.auth, f)().xml()
+            self.assertTrue('name="_formkey"' in html_form)
+
+        for f in ['logout', 'verify_email', 'reset_password', 'change_password', 'profile', 'groups']:
+            self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
+
+        self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate)
+
+        try:
+            for t in ['t0_archive', 't0', 'auth_cas', 'auth_event',
+                      'auth_membership', 'auth_permission', 'auth_group',
+                      'auth_user']:
+                self.db[t].drop()
+        except SyntaxError as e:
+            # GAE doesn't support drop
+            pass
+        return
+
+    def test_get_vars_next(self):
+        self.current.request.vars._next = 'next_test'
+        self.assertEqual(self.auth.get_vars_next(), 'next_test')
+
+    # TODO: def test_navbar(self):
+    # TODO: def test___get_migrate(self):
+
+    def test_enable_record_versioning(self):
+        self.assertTrue('t0_archive' in self.db)
+
+    # TODO: def test_define_signature(self):
+    # TODO: def test_define_signature(self):
+    # TODO: def test_define_table(self):
+
+    def test_log_event(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        # user logged in
+        self.auth.log_event(description='some_log_event_description_%(var1)s',
+                            vars={"var1": "var1"},
+                            origin='log_event_test_1')
+        rtn = self.db(self.db.auth_event.origin == 'log_event_test_1'
+                      ).select(*[self.db.auth_event[f]
+                                 for f in self.db.auth_event.fields if f not in ('id', 'time_stamp')]).first().as_dict()
+        self.assertEqual(set(rtn.items()), set({'origin': 'log_event_test_1',
+                                                'client_ip': None,
+                                                'user_id': bart_id,
+                                                'description': 'some_log_event_description_var1'}.items()))
+        # user not logged
+        self.auth.logout_bare()
+        self.auth.log_event(description='some_log_event_description_%(var2)s',
+                            vars={"var2": "var2"},
+                            origin='log_event_test_2')
+        rtn = self.db(self.db.auth_event.origin == 'log_event_test_2'
+                      ).select(*[self.db.auth_event[f]
+                                 for f in self.db.auth_event.fields if f not in ('id', 'time_stamp')]).first().as_dict()
+        self.assertEqual(set(rtn.items()), set({'origin': 'log_event_test_2',
+                                                'client_ip': None,
+                                                'user_id': None,
+                                                'description': 'some_log_event_description_var2'}.items()))
+        # no logging tests
+        self.auth.settings.logging_enabled = False
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.auth.log_event(description='some_log_event_description_%(var3)s',
+                            vars={"var3": "var3"},
+                            origin='log_event_test_3')
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        self.auth.settings.logging_enabled = True
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.auth.log_event(description=None,
+                            vars={"var4": "var4"},
+                            origin='log_event_test_4')
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # TODO: Corner case translated description...
+
+    def test_get_or_create_user(self):
+        self.db.auth_user.insert(email='user1@test.com', username='user1', password='password_123')
+        self.db.commit()
+        # True case
+        self.assertEqual(self.auth.get_or_create_user({'email': 'user1@test.com',
+                                                       'username': 'user1',
+                                                       'password': 'password_123'
+                                                       })['username'], 'user1')
+        # user2 doesn't exist yet and get created
+        self.assertEqual(self.auth.get_or_create_user({'email': 'user2@test.com',
+                                                       'username': 'user2'})['username'], 'user2')
+        # user3 for corner case
+        self.assertEqual(self.auth.get_or_create_user({'first_name': 'Omer',
+                                                       'last_name': 'Simpson',
+                                                       'email': 'user3@test.com',
+                                                       'registration_id': 'user3',
+                                                       'username': 'user3'})['username'], 'user3')
+        # False case
+        self.assertEqual(self.auth.get_or_create_user({'email': ''}), None)
+        self.db.auth_user.truncate()
+        self.db.commit()
+
+    # TODO: def test_basic(self):
+    # TODO: def test_login_user(self):
+    # TODO: def test__get_login_settings(self):
+
+    # login_bare() seems broken see my post on web2py-developpers
+    # commented for now
+    # def test_login_bare(self):
+    #     # The following test case should succeed but failed as I never received the user record but False
+    #     self.auth.login_bare(username='bart', password='bart_password')
+    #     self.assertTrue(self.auth.is_logged_in())
+    #     # Failing login because bad_password
+    #     self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+    #     self.auth.logout_bare()
+    #     self.db.auth_user.truncate()
+
+    def test_register_bare(self):
+        # corner case empty register call register_bare without args
+        self.assertRaises(ValueError, self.auth.register_bare)
+        # failing register_bare user already exist
+        self.assertEqual(self.auth.register_bare(username='bart', password='wrong_password'), False)
+        # successful register_bare
+        self.assertEqual(self.auth.register_bare(username='user2',
+                                                 email='user2@test.com',
+                                                 password='password_123')['username'], 'user2')
+        # raise ValueError
+        self.assertRaises(ValueError, self.auth.register_bare,
+                          **dict(wrong_field_name='user3', password='password_123'))
+        # raise ValueError wrong email
+        self.assertRaises(ValueError, self.auth.register_bare,
+                          **dict(email='user4@', password='password_123'))
+        self.db.auth_user.truncate()
+        self.db.commit()
+
+    # TODO: def test_cas_login(self):
+    # TODO: def test_cas_validate(self):
+    # TODO: def test__reset_two_factor_auth(self):
+    # TODO: def test_when_is_logged_in_bypass_next_in_url(self):
+    # TODO: def test_login(self):
+    # TODO: def test_logout(self):
+
+    def test_logout_bare(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.logout_bare()
+        self.assertFalse(self.auth.is_logged_in())
+
+    # TODO: def test_register(self):
+
+    def test_is_logged_in(self):
+        self.auth.user = 'logged_in'
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.user = None
+        self.assertFalse(self.auth.is_logged_in())
+
+    # TODO: def test_verify_email(self):
+    # TODO: def test_retrieve_username(self):
+
+    def test_random_password(self):
+        # let just check that the function is callable
+        self.assertTrue(self.auth.random_password())
+
+    # TODO: def test_reset_password_deprecated(self):
+    # TODO: def test_confirm_registration(self):
+    # TODO: def test_email_registration(self):
+
+    def test_bulk_register(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.auth.settings.bulk_register_enabled = True
+        bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
+        self.assertTrue('name="_formkey"' in bulk_register_form)
+
+    # TODO: def test_manage_tokens(self):
+    # TODO: def test_reset_password(self):
+    # TODO: def test_request_reset_password(self):
+    # TODO: def test_email_reset_password(self):
+    # TODO: def test_retrieve_password(self):
+
+    def test_change_password(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        change_password_form = getattr(self.auth, 'change_password')().xml()
+        self.assertTrue('name="_formkey"' in change_password_form)
+
+    def test_profile(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        profile_form = getattr(self.auth, 'profile')().xml()
+        self.assertTrue('name="_formkey"' in profile_form)
+
+    # TODO: def test_run_login_onaccept(self):
+    # TODO: def test_jwt(self):
+    # TODO: def test_is_impersonating(self):
+
+    def test_impersonate(self):
+        # Create a user to be impersonated
+        self.auth.get_or_create_user(dict(first_name='Omer',
+                                          last_name='Simpson',
+                                          username='omer',
+                                          email='omer@test.com',
+                                          password='password_omer',
+                                          registration_key='',
+                                          registration_id=''),
+                                     login=False)
+        self.db.commit()
+        self.assertFalse(self.auth.is_logged_in())
+        # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+        group_id = self.auth.add_group('impersonate')
+        self.auth.add_membership(user_id=self.db(self.db.auth_user.username == 'bart'
+                                                 ).select(self.db.auth_user.id).first().id,
+                                 group_id=group_id)
+        self.auth.add_permission(group_id=group_id,
+                                 name='impersonate',
+                                 table_name='auth_user',
+                                 record_id=0)
+        # Bart login
+        # self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.is_logged_in())
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.assertEqual(self.auth.user_id, bart_id)
+        # self.session.auth = self.auth
+        # self.assertTrue(self.session.auth)
+
+        # basic impersonate() test that return a read form
+        self.assertEqual(self.auth.impersonate().xml(),
+                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="no_table_user_id__row"><td class="w2p_fl"><label class="" for="no_table_user_id" id="no_table_user_id__label">User Id: </label></td><td class="w2p_fw"><input class="integer" id="no_table_user_id" name="user_id" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
+        # bart impersonate itself
+        self.assertEqual(self.auth.impersonate(bart_id), None)
+        self.assertFalse(self.auth.is_impersonating())  # User shouldn't impersonate itself?
+        # Bart impersonate Omer
+        omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
+        impersonate_form = self.auth.impersonate(user_id=omer_id)
+        self.assertTrue(self.auth.is_impersonating())
+        self.assertEqual(self.auth.user_id, omer_id)  # we make it really sure
+        self.assertEqual(impersonate_form.xml(),
+                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="auth_user_id__row"><td class="w2p_fl"><label class="readonly" for="auth_user_id" id="auth_user_id__label">Id: </label></td><td class="w2p_fw"><span id="auth_user_id">2</span></td><td class="w2p_fc"></td></tr><tr id="auth_user_first_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_first_name" id="auth_user_first_name__label">First name: </label></td><td class="w2p_fw">Omer</td><td class="w2p_fc"></td></tr><tr id="auth_user_last_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_last_name" id="auth_user_last_name__label">Last name: </label></td><td class="w2p_fw">Simpson</td><td class="w2p_fc"></td></tr><tr id="auth_user_email__row"><td class="w2p_fl"><label class="readonly" for="auth_user_email" id="auth_user_email__label">E-mail: </label></td><td class="w2p_fw">omer@test.com</td><td class="w2p_fc"></td></tr><tr id="auth_user_username__row"><td class="w2p_fl"><label class="readonly" for="auth_user_username" id="auth_user_username__label">Username: </label></td><td class="w2p_fw">omer</td><td class="w2p_fc"></td></tr></table><div style="display:none;"><input name="id" type="hidden" value="2" /></div></form>')
+        self.auth.logout_bare()
+        # Failing impersonation
+        # User lacking impersonate membership
+        self.auth.login_user(self.db(self.db.auth_user.username == 'omer').select().first())  # bypass login_bare()
+        # self.assertTrue(self.auth.is_logged_in())  # For developing test
+        # self.assertFalse(self.auth.is_impersonating())  # For developing test
+        self.assertRaisesRegexp(HTTP, "403*", self.auth.impersonate, bart_id)
+        self.auth.logout_bare()
+        # Try impersonate a non existing user
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        # self.assertTrue(self.auth.is_logged_in())  # For developing test
+        # self.assertFalse(self.auth.is_impersonating())  # For developing test
+        self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate, 1000)  # user with id 1000 shouldn't exist
+        # Try impersonate user with id = 0 or '0' when bart impersonating omer
+        self.auth.impersonate(user_id=omer_id)
+        self.assertTrue(self.auth.is_impersonating())
+        self.assertEqual(self.auth.impersonate(user_id=0), None)
+
+    # TODO: def test_update_groups(self):
+
+    def test_groups(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertEqual(self.auth.groups().xml(),
+                         '<table><tr><td><h3>user_1(1)</h3></td></tr><tr><td><p></p></td></tr></table>')
+
+    def test_not_authorized(self):
+        self.current.request.ajax = 'facke_ajax_request'
+        self.assertRaisesRegexp(HTTP, "403*", self.auth.not_authorized)
+        self.current.request.ajax = None
+        self.assertEqual(self.auth.not_authorized(), self.auth.messages.access_denied)
+
+    def test_allows_jwt(self):
+        self.assertRaisesRegexp(HTTP, "400*", self.auth.allows_jwt)
+
+    # TODO: def test_requires(self):
+    # TODO: def test_requires_login(self):
+    # TODO: def test_requires_login_or_token(self):
+    # TODO: def test_requires_membership(self):
+    # TODO: def test_requires_permission(self):
+    # TODO: def test_requires_signature(self):
+
+    def test_add_group(self):
+        self.assertEqual(self.auth.add_group(role='a_group', description='a_group_role_description'),
+                         self.db(self.db.auth_group.role == 'a_group').select(self.db.auth_group.id).first().id)
+
+    def test_del_group(self):
+        bart_group_id = 1  # Should be group 1, 'user_1'
+        self.assertEqual(self.auth.del_group(group_id=bart_group_id), None)
+
+    def test_id_group(self):
+        self.assertEqual(self.auth.id_group(role='user_1'), 1)
+        # If role don't exist it return None
+        self.assertEqual(self.auth.id_group(role='non_existing_role_name'), None)
+
+    def test_user_group(self):
+        self.assertEqual(self.auth.user_group(user_id=1), 1)
+        # Bart should be user 1 and it unique group should be 1, 'user_1'
+
+    def test_user_group_role(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        user_group_role = 'user_%s' % self.db(self.db.auth_user.username == 'bart'
+                                              ).select(self.db.auth_user.id).first().id
+        self.assertEqual(self.auth.user_group_role(), user_group_role)
+        self.auth.logout_bare()
+        # with user_id args
+        self.assertEqual(self.auth.user_group_role(user_id=1), 'user_1')
+        # test None
+        self.auth.settings.create_user_groups = None
+        self.assertEqual(self.auth.user_group_role(user_id=1), None)
+
+    def test_has_membership(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.has_membership('user_1'))
+        self.assertFalse(self.auth.has_membership('user_555'))
+        self.assertTrue(self.auth.has_membership(group_id=1))
+        self.auth.logout_bare()
+        self.assertTrue(self.auth.has_membership(role='user_1', user_id=1))
+        self.assertTrue(self.auth.has_membership(group_id=1, user_id=1))
+        # check that event is logged
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.assertTrue(self.auth.has_membership(group_id=1, user_id=1))
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+
+    # Waiting guidance : https://github.com/web2py/web2py/issues/1300
+    # def test_add_membership(self):
+    #     self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+    #     # failing case
+    #     rtn = self.auth.add_membership('not_existing_role_name')
+    #     # self.assertEqual(rtn, 'test')
+    #     self.assertEqual(self.db(self.db.auth_group.role == 'not_existing_role_name').select().first(), 'test')
+
+    def test_del_membership(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        user_1_role_id = self.db(self.db.auth_membership.group_id == self.auth.id_group('user_1')
+                                 ).select(self.db.auth_membership.id).first().id
+        self.assertEqual(self.auth.del_membership('user_1'), user_1_role_id)
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # not logged in test case
+        group_id = self.auth.add_group('some_test_group')
+        membership_id = self.auth.add_membership('some_test_group')
+        self.assertEqual(self.auth.user_groups[group_id], 'some_test_group')
+        self.auth.logout_bare()
+        # not deleted
+        self.assertFalse(self.auth.del_membership('some_test_group'))
+        self.assertEqual(set(self.db.auth_membership(membership_id).as_dict().items()),
+                         set({'group_id': 2L, 'user_id': 1L, 'id': 2L}.items()))  # is not deleted
+        # deleted
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.assertTrue(self.auth.del_membership('some_test_group', user_id=bart_id))
+        self.assertEqual(self.db.auth_membership(membership_id), None)  # is really deleted
+
+    def test_has_permission(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                 name='some_permission',
+                                 table_name='auth_user',
+                                 record_id=0,
+                                 )
+        # True case
+        self.assertTrue(self.auth.has_permission(name='some_permission',
+                                                 table_name='auth_user',
+                                                 record_id=0,
+                                                 user_id=bart_id,
+                                                 group_id=self.auth.id_group('user_1')))
+        # False case
+        self.assertFalse(self.auth.has_permission(name='some_other_permission',
+                                                  table_name='auth_user',
+                                                  record_id=0,
+                                                  user_id=bart_id,
+                                                  group_id=self.auth.id_group('user_1')))
+
+    def test_add_permission(self):
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        permission_id = \
+            self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                     name='some_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # True case
+        permission_count = \
+            self.db(self.db.auth_permission.id == permission_id).count()
+        self.assertTrue(permission_count)
+        # False case
+        permission_count = \
+            self.db((self.db.auth_permission.group_id == self.auth.id_group('user_1')) &
+                    (self.db.auth_permission.name == 'no_permission') &
+                    (self.db.auth_permission.table_name == 'no_table') &
+                    (self.db.auth_permission.record_id == 0)).count()
+        self.assertFalse(permission_count)
+        # corner case
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        permission_id = \
+            self.auth.add_permission(group_id=0,
+                                     name='user_1_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        permission_name = \
+            self.db(self.db.auth_permission.id == permission_id).select(self.db.auth_permission.name).first().name
+        self.assertEqual(permission_name, 'user_1_permission')
+        # add an existing permission
+        permission_id =\
+            self.auth.add_permission(group_id=0,
+                                     name='user_1_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        self.assertTrue(permission_id)
+
+    def test_del_permission(self):
+        permission_id = \
+            self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                     name='del_permission_test',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.assertTrue(self.auth.del_permission(group_id=self.auth.id_group('user_1'),
+                                                 name='del_permission_test',
+                                                 table_name='auth_user',
+                                                 record_id=0,))
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # really deleted
+        permission_count = \
+            self.db(self.db.auth_permission.id == permission_id).count()
+        self.assertFalse(permission_count)
+
+    # TODO: def test_accessible_query(self):
+    # TODO: def test_archive(self):
+    # TODO: def test_wiki(self):
+    # TODO: def test_wikimenu(self):
+    # End Auth test
+
+
+# TODO: class TestCrud(unittest.TestCase):
+# It deprecated so far from a priority
+
+
+# TODO: class TestService(unittest.TestCase):
+
+
+# TODO: class TestPluginManager(unittest.TestCase):
+
+
+# TODO: class TestExpose(unittest.TestCase):
+
+
+# TODO: class TestWiki(unittest.TestCase):
+
+
+# TODO: class TestConfig(unittest.TestCase):
+
+
+class TestToolsFunctions(unittest.TestCase):
+    """
+    Test suite for all the tools.py functions
+    """
+    def test_prettydate(self):
+        # plain
+        now = datetime.datetime.now()
+        self.assertEqual(prettydate(d=now), 'now')
+        one_second = now - datetime.timedelta(seconds=1)
+        self.assertEqual(prettydate(d=one_second), '1 second ago')
+        more_than_one_second = now - datetime.timedelta(seconds=2)
+        self.assertEqual(prettydate(d=more_than_one_second), '2 seconds ago')
+        one_minute = now - datetime.timedelta(seconds=60)
+        self.assertEqual(prettydate(d=one_minute), '1 minute ago')
+        more_than_one_minute = now - datetime.timedelta(seconds=61)
+        self.assertEqual(prettydate(d=more_than_one_minute), '1 minute ago')
+        two_minutes = now - datetime.timedelta(seconds=120)
+        self.assertEqual(prettydate(d=two_minutes), '2 minutes ago')
+        more_than_two_minutes = now - datetime.timedelta(seconds=121)
+        self.assertEqual(prettydate(d=more_than_two_minutes), '2 minutes ago')
+        one_hour = now - datetime.timedelta(seconds=60 * 60)
+        self.assertEqual(prettydate(d=one_hour), '1 hour ago')
+        more_than_one_hour = now - datetime.timedelta(seconds=3601)
+        self.assertEqual(prettydate(d=more_than_one_hour), '1 hour ago')
+        two_hours = now - datetime.timedelta(seconds=2 * 60 * 60)
+        self.assertEqual(prettydate(d=two_hours), '2 hours ago')
+        more_than_two_hours = now - datetime.timedelta(seconds=2 * 60 * 60 + 1)
+        self.assertEqual(prettydate(d=more_than_two_hours), '2 hours ago')
+        one_day = now - datetime.timedelta(days=1)
+        self.assertEqual(prettydate(d=one_day), '1 day ago')
+        more_than_one_day = now - datetime.timedelta(days=2)
+        self.assertEqual(prettydate(d=more_than_one_day), '2 days ago')
+        one_week = now - datetime.timedelta(days=7)
+        self.assertEqual(prettydate(d=one_week), '1 week ago')
+        more_than_one_week = now - datetime.timedelta(days=8)
+        self.assertEqual(prettydate(d=more_than_one_week), '1 week ago')
+        two_weeks = now - datetime.timedelta(days=14)
+        self.assertEqual(prettydate(d=two_weeks), '2 weeks ago')
+        more_than_two_weeks = now - datetime.timedelta(days=15)
+        self.assertEqual(prettydate(d=more_than_two_weeks), '2 weeks ago')
+        three_weeks = now - datetime.timedelta(days=21)
+        self.assertEqual(prettydate(d=three_weeks), '3 weeks ago')
+        one_month = now - datetime.timedelta(days=27)
+        self.assertEqual(prettydate(d=one_month), '1 month ago')
+        more_than_one_month = now - datetime.timedelta(days=28)
+        self.assertEqual(prettydate(d=more_than_one_month), '1 month ago')
+        two_months = now - datetime.timedelta(days=60)
+        self.assertEqual(prettydate(d=two_months), '2 months ago')
+        three_months = now - datetime.timedelta(days=90)
+        self.assertEqual(prettydate(d=three_months), '3 months ago')
+        one_year = now - datetime.timedelta(days=365)
+        self.assertEqual(prettydate(d=one_year), '1 year ago')
+        more_than_one_year = now - datetime.timedelta(days=366)
+        self.assertEqual(prettydate(d=more_than_one_year), '1 year ago')
+        two_years = now - datetime.timedelta(days=2 * 365)
+        self.assertEqual(prettydate(d=two_years), '2 years ago')
+        more_than_two_years = now - datetime.timedelta(days=2 * 365 + 1)
+        self.assertEqual(prettydate(d=more_than_two_years), '2 years ago')
+        # date()
+        d = now.date()
+        self.assertEqual(prettydate(d=d), 'now')
+        one_day = now.date() - datetime.timedelta(days=1)
+        self.assertEqual(prettydate(d=one_day), '1 day ago')
+        tow_days = now.date() - datetime.timedelta(days=2)
+        self.assertEqual(prettydate(d=tow_days), '2 days ago')
+        # from now
+        # from now is picky depending of the execution time, so we can't use sharp value like 1 second or 1 day
+        in_one_minute = now - datetime.timedelta(seconds=-65)
+        self.assertEqual(prettydate(d=in_one_minute), '1 minute from now')
+        in_twenty_three_hours = now - datetime.timedelta(hours=-23.5)
+        self.assertEqual(prettydate(d=in_twenty_three_hours), '23 hours from now')
+        in_one_year = now - datetime.timedelta(days=-366)
+        self.assertEqual(prettydate(d=in_one_year), '1 year from now')
+        # utc=True
+        now = datetime.datetime.utcnow()
+        self.assertEqual(prettydate(d=now, utc=True), 'now')
+        one_second = now - datetime.timedelta(seconds=1)
+        self.assertEqual(prettydate(d=one_second, utc=True), '1 second ago')
+        # not d or invalid date
+        self.assertEqual(prettydate(d=None), '')
+        self.assertEqual(prettydate(d='invalid_date'), '[invalid date]')
 
 
 if __name__ == '__main__':

gluon/tests/test_validators.py

@@ -160,6 +160,14 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ({u'a': 100}, None))
         rtn = IS_JSON()('spam1234')
         self.assertEqual(rtn, ('spam1234', 'Invalid json'))
+        rtn = IS_JSON(native_json=True)('{"a": 100}')
+        self.assertEqual(rtn, ('{"a": 100}', None))
+        rtn = IS_JSON().formatter(None)
+        self.assertEqual(rtn, None)
+        rtn = IS_JSON().formatter({'a': 100})
+        self.assertEqual(rtn, '{"a": 100}')
+        rtn = IS_JSON(native_json=True).formatter({'a': 100})
+        self.assertEqual(rtn, {'a': 100})
 
     def test_IS_IN_SET(self):
         rtn = IS_IN_SET(['max', 'john'])('max')
@@ -174,6 +182,14 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('id1', None))
         rtn = IS_IN_SET({'id1': 'first label', 'id2': 'second label'})('id1')
         self.assertEqual(rtn, ('id1', None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)(None)
+        self.assertEqual(rtn, ([], None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)('')
+        self.assertEqual(rtn, ([], None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=True)('id1')
+        self.assertEqual(rtn, (['id1'], None))
+        rtn = IS_IN_SET(['id1', 'id2'], error_message='oops', multiple=(1, 2))(None)
+        self.assertEqual(rtn, ([], 'oops'))
         import itertools
         rtn = IS_IN_SET(itertools.chain(['1', '3', '5'], ['2', '4', '6']))('1')
         self.assertEqual(rtn, ('1', None))
@@ -181,6 +197,10 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('id1', None))
         rtn = IS_IN_SET([('id1', 'first label'), ('id2', 'second label')]).options(zero=False)
         self.assertEqual(rtn, [('id1', 'first label'), ('id2', 'second label')])
+        rtn = IS_IN_SET(['id1', 'id2']).options(zero=False)
+        self.assertEqual(rtn, [('id1', 'id1'), ('id2', 'id2')])
+        rtn = IS_IN_SET(['id2', 'id1'], sort=True).options(zero=False)
+        self.assertEqual(rtn, [('id1', 'id1'), ('id2', 'id2')])
 
     def test_IS_IN_DB(self):
         from gluon.dal import DAL, Field
@@ -190,6 +210,12 @@ class TestValidators(unittest.TestCase):
         costanza_id = db.person.insert(name='costanza')
         rtn = IS_IN_DB(db, 'person.id', '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
+        rtn = IS_IN_DB(db, 'person.name', '%(name)s')('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = IS_IN_DB(db, db.person, '%(name)s')(george_id)
+        self.assertEqual(rtn, (george_id, None))
+        rtn = IS_IN_DB(db(db.person.id > 0), db.person, '%(name)s')(george_id)
+        self.assertEqual(rtn, (george_id, None))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')(george_id + costanza_id)
         self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
         rtn = IS_IN_DB(db, db.person.id, '%(name)s')(george_id)
@@ -198,21 +224,86 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True)([george_id, costanza_id])
         self.assertEqual(rtn, ([george_id, costanza_id], None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, error_message='oops')("I'm not even an id")
+        self.assertEqual(rtn, (["I'm not even an id"], 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, delimiter=',')('%d,%d' % (george_id, costanza_id))
         self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 3), delimiter=',')('%d,%d' % (george_id, costanza_id))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 2), delimiter=',', error_message='oops')('%d,%d' % (george_id, costanza_id))
+        self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), 'oops'))
         rtn = IS_IN_DB(db, db.person.id, '%(name)s', error_message='oops').options(zero=False)
         self.assertEqual(sorted(rtn), [('%d' % george_id, 'george'), ('%d' % costanza_id, 'costanza')])
+        rtn = IS_IN_DB(db, db.person.id, db.person.name, error_message='oops', sort=True).options(zero=True)
+        self.assertEqual(rtn, [('', ''), ('%d' % costanza_id, 'costanza'), ('%d' % george_id, 'george')])
+        # Test using the set it made for options
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops')
+        vldtr.options()
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', multiple=True)
+        vldtr.options()
+        rtn = vldtr(['george', 'costanza'])
+        self.assertEqual(rtn, (['george', 'costanza'], None))
+        # Test it works with self reference
+        db.define_table('category',
+                        Field('parent_id', 'reference category', requires=IS_EMPTY_OR(IS_IN_DB(db, 'category.id', '%(name)s'))),
+                        Field('name')
+                        )
+        ret = db.category.validate_and_insert(name='seinfeld')
+        self.assertFalse(list(ret.errors))
+        ret = db.category.validate_and_insert(name='characters', parent_id=ret.id)
+        self.assertFalse(list(ret.errors))
+        rtn = IS_IN_DB(db, 'category.id', '%(name)s')(ret.id)
+        self.assertEqual(rtn, (ret.id, None))
+        # Test _and
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', _and=IS_LENGTH(maxsize=7, error_message='bad'))
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', 'bad'))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr.options() # test theset with _and
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        # Test auto_add
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', auto_add=True)('jerry')
+        self.assertEqual(rtn, (3, None))
         db.person.drop()
+        db.category.drop()
 
     def test_IS_NOT_IN_DB(self):
         from gluon.dal import DAL, Field
         db = DAL('sqlite:memory')
-        db.define_table('person', Field('name'))
+        db.define_table('person', Field('name'), Field('nickname'))
         db.person.insert(name='george')
+        db.person.insert(name='costanza', nickname='T Bone')
         rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops')('george')
         self.assertEqual(rtn, ('george', 'oops'))
+        rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops', allowed_override=['george'])('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops')('  ')
+        self.assertEqual(rtn, ('  ', 'oops'))
         rtn = IS_NOT_IN_DB(db, 'person.name')('jerry')
         self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db, 'person.name')(u'jerry')
+        self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db(db.person.id > 0), 'person.name')(u'jerry')
+        self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db, db.person, error_message='oops')(1)
+        self.assertEqual(rtn, ('1', 'oops'))
+        vldtr = IS_NOT_IN_DB(db, 'person.name', error_message='oops')
+        vldtr.set_self_id({'name': 'costanza', 'nickname': 'T Bone'})
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', 'oops'))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', None))
+
         db.person.drop()
 
     def test_IS_INT_IN_RANGE(self):
@@ -437,6 +528,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('localguy@localhost', 'Enter a valid email address'))
         rtn = IS_EMAIL(forced='^.*\.edu(|\..*)$')('localguy@example.edu')
         self.assertEqual(rtn, ('localguy@example.edu', None))
+        # test for not a string at all
+        rtn = IS_EMAIL(error_message='oops')(42)
+        self.assertEqual(rtn, (42, 'oops'))
 
     def test_IS_LIST_OF_EMAILS(self):
         emails = ['localguy@localhost', '_Yosemite.Sam@example.com']
@@ -512,16 +606,20 @@ class TestValidators(unittest.TestCase):
         # Test timezone is removed and value is properly converted
         #
         # https://github.com/web2py/web2py/issues/1094
+
         class DummyTimezone(datetime.tzinfo):
 
             ONE = datetime.timedelta(hours=1)
 
             def utcoffset(self, dt):
                 return DummyTimezone.ONE
+
             def tzname(self, dt):
                 return "UTC+1"
+
             def dst(self, dt):
                 return DummyTimezone.ONE
+
             def localize(self, dt, is_dst=False):
                 return dt.replace(tzinfo=self)
         v = IS_DATETIME(format="%Y-%m-%d %H:%M", error_message="oops", timezone=DummyTimezone())
@@ -668,6 +766,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('a@b.co', None))
         rtn = ANY_OF([IS_DATE(), IS_EMAIL()])('1982-12-14')
         self.assertEqual(rtn, (datetime.date(1982, 12, 14), None))
+        rtn = ANY_OF([IS_DATE(format='%m/%d/%Y'), IS_EMAIL()]).formatter(datetime.date(1834, 12, 14))
+        self.assertEqual(rtn, '12/14/1834')
 
     def test_IS_EMPTY_OR(self):
         rtn = IS_EMPTY_OR(IS_EMAIL())('abc@def.com')
@@ -764,6 +864,7 @@ class TestValidators(unittest.TestCase):
 
     def test_IS_IMAGE(self):
         class DummyImageFile(object):
+
             def __init__(self, filename, ext, width, height):
                 from StringIO import StringIO
                 import struct
@@ -823,10 +924,10 @@ class TestValidators(unittest.TestCase):
         rtn = IS_IMAGE(error_message='oops')(img)
         self.assertEqual(rtn, (img, 'oops'))
 
-
     def test_IS_UPLOAD_FILENAME(self):
         import cgi
         from StringIO import StringIO
+
         def gen_fake(filename):
             formdata_file_data = """
 ---123

gluon/tools.py

@@ -1217,7 +1217,7 @@ class AuthJWT(object):
         self.auth = auth
         self.algorithm = algorithm
         if self.algorithm not in ('HS256', 'HS384', 'HS512'):
-            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
+            raise NotImplementedError('Algorithm %s not allowed' % algorithm)
         self.verify_expiration = verify_expiration
         self.leeway = leeway
         self.expiration = expiration
@@ -1314,6 +1314,7 @@ class AuthJWT(object):
         We (mis)use the heavy default auth mechanism to avoid any further computation,
         while sticking to a somewhat-stable Auth API.
         """
+        # TODO: Check the following comment
         ## is the following safe or should we use
         ## calendar.timegm(datetime.datetime.utcnow().timetuple())
         ## result seem to be the same (seconds since epoch, in UTC)
@@ -1395,8 +1396,9 @@ class AuthJWT(object):
             self.alter_payload(payload)
             ret = {'token': self.generate_token(payload)}
         elif ret is None:
-            raise HTTP(
-                401, u'Not Authorized - need to be logged in, to pass a token for refresh or username and password for login',
+            raise HTTP(401,
+                       u'Not Authorized - need to be logged in, to pass a token '
+                       u'for refresh or username and password for login',
                        **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
         response.headers['Content-Type'] = 'application/json'
         return serializers.json(ret)
@@ -1735,6 +1737,7 @@ class Auth(object):
             host = host_names[0]
         else:
             host = 'localhost'
+        return host
 
     def __init__(self, environment=None, db=None, mailer=True,
                  hmac_key=None, controller='default', function='user',
@@ -1742,7 +1745,7 @@ class Auth(object):
                  csrf_prevention=True, propagate_extension=None,
                  url_index=None, jwt=None, host_names=None):
 
-        ## next two lines for backward compatibility
+        # next two lines for backward compatibility
         if not db and environment and isinstance(environment, DAL):
             db = environment
         self.db = db
@@ -1973,10 +1976,8 @@ class Auth(object):
         else:
             next = '?_next=' + urllib.quote(URL(args=request.args,
                                                 vars=request.get_vars))
-        href = lambda function: '%s/%s%s' % (action, function, next
-                                             if referrer_actions is DEFAULT
-                                             or function in referrer_actions
-                                             else '')
+        href = lambda function: \
+            '%s/%s%s' % (action, function, next if referrer_actions is DEFAULT or function in referrer_actions else '')
         if isinstance(prefix, str):
             prefix = T(prefix)
         if prefix:
@@ -1989,9 +1990,7 @@ class Auth(object):
         if self.user_id:  # User is logged in
             logout_next = self.settings.logout_next
             items.append({'name': T('Log Out'),
-                          'href': '%s/logout?_next=%s' % (action,
-                                                          urllib.quote(
-                                                          logout_next)),
+                          'href': '%s/logout?_next=%s' % (action, urllib.quote(logout_next)),
                           'icon': 'icon-off'})
             if 'profile' not in self.settings.actions_disabled:
                 items.append({'name': T('Profile'), 'href': href('profile'),
@@ -2206,8 +2205,7 @@ class Auth(object):
             if ('id' in fieldnames and
                 'modified_on' in fieldnames and
                 not current_record in fieldnames):
-                table._enable_record_versioning(
-                    archive_db=archive_db,
+                table._enable_record_versioning(archive_db=archive_db,
                                                 archive_name=archive_names,
                                                 current_record=current_record,
                                                 current_record_label=current_record_label)
@@ -2477,9 +2475,7 @@ class Auth(object):
                     Field('expires_on', 'datetime', default=datetime.datetime(2999, 12, 31)),
                     Field('token', writable=False, default=web2py_uuid, unique=True),
                     *extra_fields,
-                    **dict(
-                        migrate=self.__get_migrate(
-                            settings.table_token_name, migrate),
+                    **dict(migrate=self.__get_migrate(settings.table_token_name, migrate),
                            fake_migrate=fake_migrate))
         if not db._lazy_tables:
             settings.table_user = db[settings.table_user_name]
@@ -2531,9 +2527,7 @@ class Auth(object):
         # log messages should not be translated
         if type(description).__name__ == 'lazyT':
             description = description.m
-        self.table_event().insert(
-            description=str(description % vars),
-            origin=origin, user_id=user_id)
+        self.table_event().insert(description=str(description % vars), origin=origin, user_id=user_id)
 
     def get_or_create_user(self, keys, update_fields=['email'],
                            login=True, get=True):
@@ -2575,15 +2569,14 @@ class Auth(object):
                     update_keys[key] = keys[key]
             user.update_record(**update_keys)
         elif checks:
-            if not 'first_name' in keys and 'first_name' in table_user.fields:
+            if 'first_name' not in keys and 'first_name' in table_user.fields:
                 guess = keys.get('email', 'anonymous').split('@')[0]
                 keys['first_name'] = keys.get('username', guess)
             vars = table_user._filter_fields(keys)
             user_id = table_user.insert(**vars)
             user = table_user[user_id]
             if self.settings.create_user_groups:
-                group_id = self.add_group(
-                    self.settings.create_user_groups % user)
+                group_id = self.add_group(self.settings.create_user_groups % user)
                 self.add_membership(group_id, user_id)
             if self.settings.everybody_group_id:
                 self.add_membership(self.settings.everybody_group_id, user_id)
@@ -2698,8 +2691,7 @@ class Auth(object):
             fields[settings.passfield] = \
                 settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
         if not fields.get(settings.userfield):
-            raise ValueError('register_bare: ' +
-                             'userfield not provided or invalid')
+            raise ValueError('register_bare: userfield not provided or invalid')
         user = self.get_or_create_user(fields, login=False, get=False,
                                        update_fields=self.settings.update_fields)
         if not user:
@@ -2750,7 +2742,7 @@ class Auth(object):
             redirect(session._cas_service)
 
         def cas_onaccept(form, onaccept=onaccept):
-            if not onaccept is DEFAULT:
+            if onaccept is not DEFAULT:
                 onaccept(form)
             return allow_access(interactivelogin=True)
         return self.login(next, onvalidation, cas_onaccept, log)
@@ -2837,14 +2829,14 @@ class Auth(object):
         response = current.response
         session = current.session
 
-        ### use session for federated login
+        # use session for federated login
         snext = self.get_vars_next()
 
         if snext:
             session._auth_next = snext
         elif session._auth_next:
             snext = session._auth_next
-        ### pass
+        # pass
 
         if next is DEFAULT:
             # important for security
@@ -2950,7 +2942,7 @@ class Auth(object):
                                )
 
                 captcha = settings.login_captcha or \
-                    (settings.login_captcha != False and settings.captcha)
+                    (settings.login_captcha is not False and settings.captcha)
                 if captcha:
                     addrow(form, captcha.label, captcha, captcha.comment,
                            settings.formstyle, 'captcha__row')
@@ -2978,8 +2970,7 @@ class Auth(object):
                         elif temp_user.registration_key in ('disabled', 'blocked'):
                             response.flash = self.messages.login_disabled
                             return form
-                        elif (temp_user.registration_key is not None
-                              and temp_user.registration_key.strip()):
+                        elif (temp_user.registration_key is not None and temp_user.registration_key.strip()):
                             response.flash = \
                                 self.messages.registration_verifying
                             return form
@@ -3106,8 +3097,6 @@ class Auth(object):
                             hideerror=settings.hideerror):
                 accepted_form = True
 
-                accepted_form = True
-
                 '''
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
@@ -3251,12 +3240,16 @@ class Auth(object):
                 next = cas.logout_url(next)
 
         current.session.auth = None
+        self.user = None
         if self.settings.renew_session_onlogout:
             current.session.renew(clear_session=not self.settings.keep_session_onlogout)
         current.session.flash = self.messages.logged_out
         if next is not None:
             redirect(next)
 
+    def logout_bare(self):
+        self.logout(next=None, onlogout=None, log=None)
+
     def register(self,
                  next=DEFAULT,
                  onvalidation=DEFAULT,
@@ -3358,12 +3351,10 @@ class Auth(object):
                         hideerror=self.settings.hideerror):
             description = self.messages.group_description % form.vars
             if self.settings.create_user_groups:
-                group_id = self.add_group(
-                    self.settings.create_user_groups % form.vars, description)
+                group_id = self.add_group(self.settings.create_user_groups % form.vars, description)
                 self.add_membership(group_id, form.vars.id)
             if self.settings.everybody_group_id:
-                self.add_membership(
-                    self.settings.everybody_group_id, form.vars.id)
+                self.add_membership(self.settings.everybody_group_id, form.vars.id)
             if self.settings.registration_requires_verification:
                 link = self.url(
                     self.settings.function, args=('verify_email', key), scheme=True)
@@ -3381,8 +3372,7 @@ class Auth(object):
                not self.settings.registration_requires_verification:
                 table_user[form.vars.id] = dict(registration_key='pending')
                 session.flash = self.messages.registration_pending
-            elif (not self.settings.registration_requires_verification or
-                      self.settings.login_after_registration):
+            elif (not self.settings.registration_requires_verification or self.settings.login_after_registration):
                 if not self.settings.registration_requires_verification:
                     table_user[form.vars.id] = dict(registration_key='')
                 session.flash = self.messages.registration_successful
@@ -3460,7 +3450,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_username_captcha or \
-                  (self.settings.retrieve_username_captcha != False and self.settings.captcha)
+                  (self.settings.retrieve_username_captcha is not False and self.settings.captcha)
         if not self.settings.mailer:
             response.flash = self.messages.function_disabled
             return ''
@@ -3792,8 +3782,7 @@ class Auth(object):
                   requires=self.table_user()[passfield].requires),
             Field('new_password2', 'password',
                   label=self.messages.verify_password,
-                  requires=[IS_EXPR(
-                      'value==%s' % repr(request.vars.new_password),
+                  requires=[IS_EXPR('value==%s' % repr(request.vars.new_password),
                                     self.messages.mismatched_password)]),
             submit_button=self.messages.password_reset_button,
             hidden=dict(_next=next),
@@ -3827,7 +3816,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_password_captcha or \
-                (self.settings.retrieve_password_captcha != False and self.settings.captcha)
+                (self.settings.retrieve_password_captcha is not False and self.settings.captcha)
 
         if next is DEFAULT:
             next = self.get_vars_next() or self.settings.request_reset_password_next
@@ -4124,6 +4113,7 @@ class Auth(object):
             raise HTTP(401, "Not Authorized")
         current_id = auth.user.id
         requested_id = user_id
+        user = None
         if user_id is DEFAULT:
             user_id = current.request.post_vars.user_id
         if user_id and user_id != self.user.id and user_id != '0':
@@ -4152,6 +4142,9 @@ class Auth(object):
             return None
         if requested_id is DEFAULT and not request.post_vars:
             return SQLFORM.factory(Field('user_id', 'integer'))
+        elif not user:
+            return None
+        else:
             return SQLFORM(table_user, user.id, readonly=True)
 
     def update_groups(self):
@@ -4234,10 +4227,8 @@ class Auth(object):
                         else:
                             next = self.here()
                             current.session.flash = current.response.flash
-                            return call_or_redirect(
-                                self.settings.on_failed_authentication,
-                                self.settings.login_url +
-                                    '?_next=' + urllib.quote(next))
+                            return call_or_redirect(self.settings.on_failed_authentication,
+                                                    self.settings.login_url + '?_next=' + urllib.quote(next))
 
                 if callable(condition):
                     flag = condition()
@@ -4318,11 +4309,8 @@ class Auth(object):
         """
         Creates a group associated to a role
         """
-
-        group_id = self.table_group().insert(
-            role=role, description=description)
-        self.log_event(self.messages['add_group_log'],
-                       dict(group_id=group_id, role=role))
+        group_id = self.table_group().insert(role=role, description=description)
+        self.log_event(self.messages['add_group_log'], dict(group_id=group_id, role=role))
         return group_id
 
     def del_group(self, group_id):
@@ -4332,7 +4320,8 @@ class Auth(object):
         self.db(self.table_group().id == group_id).delete()
         self.db(self.table_membership().group_id == group_id).delete()
         self.db(self.table_permission().group_id == group_id).delete()
-        if group_id in self.user_groups: del self.user_groups[group_id]
+        if group_id in self.user_groups:
+            del self.user_groups[group_id]
         self.log_event(self.messages.del_group_log, dict(group_id=group_id))
 
     def id_group(self, role):
@@ -4364,7 +4353,6 @@ class Auth(object):
         """
         Checks if user is member of group_id or role
         """
-
         group_id = group_id or self.id_group(role)
         try:
             group_id = int(group_id)
@@ -4373,8 +4361,8 @@ class Auth(object):
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
-        if group_id and user_id and self.db((membership.user_id == user_id)
-                    & (membership.group_id == group_id)).select():
+        if group_id and user_id and self.db((membership.user_id == user_id) &
+                                            (membership.group_id == group_id)).select():
             r = True
         else:
             r = False
@@ -4421,15 +4409,18 @@ class Auth(object):
         """
 
         group_id = group_id or self.id_group(role)
+        try:
+            group_id = int(group_id)
+        except:
+            group_id = self.id_group(group_id)  # interpret group_id as a role
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
         self.log_event(self.messages['del_membership_log'],
                        dict(user_id=user_id, group_id=group_id))
-        ret = self.db(membership.user_id
-                      == user_id)(membership.group_id
-                                  == group_id).delete()
-        if group_id in self.user_groups: del self.user_groups[group_id]
+        ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
+        if group_id in self.user_groups:
+            del self.user_groups[group_id]
         return ret
 
     def has_permission(self,
@@ -4446,8 +4437,7 @@ class Auth(object):
         """
 
         if not group_id and self.settings.everybody_group_id and \
-                self.has_permission(
-            name, table_name, record_id, user_id=None,
+                self.has_permission(name, table_name, record_id, user_id=None,
                                     group_id=self.settings.everybody_group_id):
                 return True
 
@@ -4455,25 +4445,24 @@ class Auth(object):
             user_id = self.user.id
         if user_id:
             membership = self.table_membership()
-            rows = self.db(membership.user_id
-                           == user_id).select(membership.group_id)
+            rows = self.db(membership.user_id == user_id).select(membership.group_id)
             groups = set([row.group_id for row in rows])
             if group_id and group_id not in groups:
                 return False
         else:
             groups = set([group_id])
         permission = self.table_permission()
-        rows = self.db(permission.name == name)(permission.table_name
-                 == str(table_name))(permission.record_id
-                 == record_id).select(permission.group_id)
+        rows = self.db(permission.name ==
+                       name)(permission.table_name ==
+                             str(table_name))(permission.record_id ==
+                                              record_id).select(permission.group_id)
         groups_required = set([row.group_id for row in rows])
         if record_id:
-            rows = self.db(permission.name
-                            == name)(permission.table_name
-                     == str(table_name))(permission.record_id
-                     == 0).select(permission.group_id)
-            groups_required = groups_required.union(set([row.group_id
-                    for row in rows]))
+            rows = self.db(permission.name ==
+                           name)(permission.table_name ==
+                                 str(table_name))(permission.record_id ==
+                                                  0).select(permission.group_id)
+            groups_required = groups_required.union(set([row.group_id for row in rows]))
         if groups.intersection(groups_required):
             r = True
         else:
@@ -4501,10 +4490,10 @@ class Auth(object):
                          (permission.name == name) &
                          (permission.table_name == str(table_name)) &
                          (permission.record_id == long(record_id)),
-                          ignore_common_filters=True).select(
-            limitby=(0, 1), orderby_on_limitby=False).first()
+                         ignore_common_filters=True
+                         ).select(limitby=(0, 1), orderby_on_limitby=False).first()
         if record:
-            if hasattr(record, 'is_active') and not record.is_ctive:
+            if hasattr(record, 'is_active') and not record.is_active:
                 record.update_record(is_active=True)
             id = record.id
         else:
@@ -4531,10 +4520,11 @@ class Auth(object):
         self.log_event(self.messages['del_permission_log'],
                        dict(group_id=group_id, name=name,
                             table_name=table_name, record_id=record_id))
-        return self.db(permission.group_id == group_id)(permission.name
-                 == name)(permission.table_name
-                           == str(table_name))(permission.record_id
-                 == long(record_id)).delete()
+        return self.db(permission.group_id ==
+                       group_id)(permission.name ==
+                                 name)(permission.table_name ==
+                                       str(table_name))(permission.record_id ==
+                                                        long(record_id)).delete()
 
     def accessible_query(self, name, table, user_id=None):
         """
@@ -4561,8 +4551,7 @@ class Auth(object):
                 cquery = table
             tablenames = db._adapter.tables(cquery)
             for tablename in tablenames:
-                cquery &= self.accessible_query(name, tablename,
-                                                user_id=user_id)
+                cquery &= self.accessible_query(name, tablename, user_id=user_id)
             return cquery
         if not isinstance(table, str) and \
                 self.has_permission(name, table, 0, user_id):
@@ -4631,8 +4620,7 @@ class Auth(object):
         you can do, for example::
 
             db.define_table('myhistory',
-                Field('parent_record',db.mytable),
-                db.mytable)
+                Field('parent_record', db.mytable), db.mytable)
 
         and use it as::
 
@@ -4647,7 +4635,7 @@ class Auth(object):
         table = form.table
         if not archive_table:
             archive_table_name = '%s_archive' % table
-            if not archive_table_name in table._db:
+            if archive_table_name not in table._db:
                 table._db.define_table(
                     archive_table_name,
                     Field(current_record, table),
@@ -4902,18 +4890,16 @@ class Crud(object):
         self.deleted = False
         captcha = self.settings.update_captcha or self.settings.captcha
         if record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
-                         self.settings.formstyle, 'captcha__row')
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
         captcha = self.settings.create_captcha or self.settings.captcha
         if not record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
-                         self.settings.formstyle, 'captcha__row')
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
         if request.extension not in ('html', 'load'):
             (_session, _formname) = (None, None)
         else:
             (_session, _formname) = (
                 session, '%s/%s' % (table._tablename, form.record_id))
-        if not formname is DEFAULT:
+        if formname is not DEFAULT:
             _formname = formname
         keepvalues = self.settings.keepvalues
         if request.vars.delete_this_record:
@@ -5202,7 +5188,7 @@ class Crud(object):
                     elif validate:
                         value, error = field.validate(txtval)
                         if not error:
-                            ### TODO deal with 'starts with', 'ends with', 'contains' on GAE
+                            # TODO deal with 'starts with', 'ends with', 'contains' on GAE
                             query &= self.get_query(field, opval, value)
                         else:
                             row[3].append(DIV(error, _class='error'))
@@ -5215,7 +5201,7 @@ class Crud(object):
                 results = db(query).select(*selected, **attributes)
                 for r in refsearch:
                     results = results.find(r)
-            except:  # hmmm, we should do better here
+            except:  # TODO: hmmm, we should do better here
                 results = None
         return form, results
 
@@ -5227,7 +5213,7 @@ def fetch(url, data=None, headers=None,
           cookie=Cookie.SimpleCookie(),
           user_agent='Mozilla/5.0'):
     headers = headers or {}
-    if not data is None:
+    if data is not None:
         data = urllib.urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
@@ -5695,7 +5681,7 @@ class Service(object):
         id, method, params = data.get('id'), data.get('method'), data.get('params', [])
         if id is None:
             return return_error(0, 100, 'missing id')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, 100, 'method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -5719,8 +5705,7 @@ class Service(object):
         def return_response(id, result):
             if not must_respond:
                 return None
-            return serializers.json({'jsonrpc': '2.0',
-                'id': id, 'result': result})
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'result': result})
 
         def return_error(id, code, message=None, data=None):
             error = {'code': code}
@@ -5731,9 +5716,7 @@ class Service(object):
                 error['message'] = message
             if data is not None:
                 error['data'] = data
-            return serializers.json({'jsonrpc': '2.0',
-                                     'id': id,
-                                     'error': error})
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'error': error})
 
         def validate(data):
             """
@@ -5793,7 +5776,7 @@ class Service(object):
             return return_error(None, e.code, e.info)
 
         id, method, params = data.get('id'), data['method'], data.get('params', '')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, -32601, data='Method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -6033,7 +6016,7 @@ def prettydate(d, T=lambda x: x, utc=False):
         return T('1 year' + suffix)
     elif dt.days >= 60:
         return T('%d months' + suffix) % int(dt.days / 30)
-    elif dt.days > 21:
+    elif dt.days >= 27:  # 4 weeks ugly
         return T('1 month' + suffix)
     elif dt.days >= 14:
         return T('%d weeks' + suffix) % int(dt.days / 7)
@@ -6439,10 +6422,9 @@ class Wiki(object):
                 args += value['args']
                 db.define_table(key, *args, **value['vars'])
 
-        if self.settings.templates is None and not \
-           self.settings.manage_permissions:
-            self.settings.templates = db.wiki_page.tags.contains('template') & \
-                db.wiki_page.can_read.contains('everybody')
+        if self.settings.templates is None and not self.settings.manage_permissions:
+            self.settings.templates = \
+                db.wiki_page.tags.contains('template') & db.wiki_page.can_read.contains('everybody')
 
         def update_tags_insert(page, id, db=db):
             for tag in page.tags or []:
@@ -6465,8 +6447,10 @@ class Wiki(object):
             'wiki_editor' not in auth.user_groups.values() and
                 self.settings.groups == auth.user_groups.values()):
             group = db.auth_group(role='wiki_editor')
-            gid = group.id if group else db.auth_group.insert(
-                role='wiki_editor')
+            if group:
+                gid = group.id
+            else:
+                db.auth_group.insert(role='wiki_editor')
             auth.add_membership(gid)
 
         settings.lock_keys = True
@@ -6493,8 +6477,7 @@ class Wiki(object):
         groups = self.settings.groups
         return ('wiki_editor' in groups or
                 (page is None and 'wiki_author' in groups) or
-                not page is None and (
-                set(groups).intersection(set(page.can_edit)) or
+                page is not None and (set(groups).intersection(set(page.can_edit)) or
                                       page.created_by == self.auth.user.id))
 
     def can_manage(self):
@@ -6516,7 +6499,7 @@ class Wiki(object):
                     return True
         return False
 
-    ### END POLICY
+    # END POLICY
 
     def automenu(self):
         """adds the menu if not present"""
@@ -6734,19 +6717,14 @@ class Wiki(object):
         if self.settings.templates:
             fields.append(
                 Field("from_template", "reference wiki_page",
-                      requires=IS_EMPTY_OR(
-                                   IS_IN_DB(db(self.settings.templates),
-                                            db.wiki_page._id,
-                                            '%(slug)s')),
-                      comment=current.T(
-                        "Choose Template or empty for new Page")))
+                      requires=IS_EMPTY_OR(IS_IN_DB(db(self.settings.templates), db.wiki_page._id, '%(slug)s')),
+                      comment=current.T("Choose Template or empty for new Page")))
         form = SQLFORM.factory(*fields, **dict(_class="well"))
         form.element("[type=submit]").attributes["_value"] = \
             current.T("Create Page from Slug")
 
         if form.process().accepted:
-             form.vars.from_template = 0 if not form.vars.from_template \
-                 else form.vars.from_template
+            form.vars.from_template = 0 if not form.vars.from_template else form.vars.from_template
             redirect(URL(args=('_edit', form.vars.slug, form.vars.from_template or 0)))  # added param
         return dict(content=form)
 
@@ -6874,7 +6852,7 @@ class Wiki(object):
             if request.vars.q:
                 tags = [v.strip() for v in request.vars.q.split(',')]
                 tags = [v.lower() for v in tags if v]
-        if tags or not query is None:
+        if tags or query is not None:
             db = self.auth.db
             count = db.wiki_tag.wiki_page.count()
             fields = [db.wiki_page.id, db.wiki_page.slug,

gluon/utils.py

@@ -83,11 +83,9 @@ def compare(a, b):
     """ Compares two strings and not vulnerable to timing attacks """
     if HAVE_COMPARE_DIGEST:
         return hmac.compare_digest(a, b)
-    if len(a) != len(b):
-        return False
-    result = 0
-    for x, y in zip(a, b):
-        result |= ord(x) ^ ord(y)
+    result = len(a) ^ len(b)
+    for i in xrange(len(b)):
+		result |= ord(a[i%len(a)]) ^ ord(b[i])
     return result == 0
 
 

gluon/validators.py

@@ -459,7 +459,7 @@ class IS_IN_SET(Validator):
 
     def __call__(self, value):
         if self.multiple:
-            ### if below was values = re.compile("[\w\-:]+").findall(str(value))
+            # if below was values = re.compile("[\w\-:]+").findall(str(value))
             if not value:
                 values = []
             elif isinstance(value, (tuple, list)):
@@ -471,8 +471,6 @@ class IS_IN_SET(Validator):
         thestrset = [str(x) for x in self.theset]
         failures = [x for x in values if not str(x) in thestrset]
         if failures and self.theset:
-            if self.multiple and (value is None or value == ''):
-                return ([], None)
             return (value, translate(self.error_message))
         if self.multiple:
             if isinstance(self.multiple, (tuple, list)) and \
@@ -525,8 +523,8 @@ class IS_IN_DB(Validator):
             field = field._id
         elif isinstance(field, str):
             items = field.split('.')
-            if len(items)==1: items+=['id']
-            field = self.dbset.db[items[0]][items[1]]
+            if len(items) == 1:
+                field = items[0] + '.id'
 
         (ktable, kfield) = str(field).split('.')
         if not label:
@@ -544,7 +542,7 @@ class IS_IN_DB(Validator):
             fieldnames = '*'
         else:
             raise NotImplementedError
-        self.field = field # the lookup field
+
         self.fieldnames = fieldnames  # fields requires to build the formatting
         self.label = label
         self.ktable = ktable
@@ -629,7 +627,7 @@ class IS_IN_DB(Validator):
             else:
                 values = []
 
-            if self.field.type in ('id','integer'):
+            if field.type in ('id', 'integer'):
                 new_values = []
                 for value in values:
                     if not (isinstance(value, (int, long)) or value.isdigit()):
@@ -647,11 +645,10 @@ class IS_IN_DB(Validator):
                 if not [v for v in values if v not in self.theset]:
                     return (values, None)
             else:
-                from pydal.adapters import GoogleDatastoreAdapter
-
                 def count(values, s=self.dbset, f=field):
                     return s(f.belongs(map(int, values))).count()
-                if GoogleDatastoreAdapter is not None and isinstance(self.dbset.db._adapter, GoogleDatastoreAdapter):
+
+                if self.dbset.db._adapter.dbengine == "google:datastore":
                     range_ids = range(0, len(values), 30)
                     total = sum(count(values[i:i + 30]) for i in range_ids)
                     if total == len(values):
@@ -659,7 +656,7 @@ class IS_IN_DB(Validator):
                 elif count(values) == len(values):
                     return (values, None)
         else:
-            if self.field.type in ('id','integer'):
+            if field.type in ('id', 'integer'):
                 if isinstance(value, (int, long)) or value.isdigit():
                     value = int(value)
                 elif self.auto_add:
@@ -1200,7 +1197,12 @@ class IS_EMAIL(Validator):
         self.error_message = error_message
 
     def __call__(self, value):
+        try:
             match = self.regex.match(value)
+        except TypeError:
+            # Value may not be a string where we can look for matches.
+            # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
+            match = None
         if match:
             domain = value.split('@')[1]
             if (not self.banned or not self.banned.match(domain)) \
@@ -2344,6 +2346,7 @@ class IS_DATE_IN_RANGE(IS_DATE):
             (datetime.date(2010, 3, 3), 'oops')
 
     """
+
     def __init__(self,
                  minimum=None,
                  maximum=None,
@@ -2397,6 +2400,7 @@ class IS_DATETIME_IN_RANGE(IS_DATETIME):
             (datetime.datetime(2010, 3, 3, 0, 0), 'oops')
 
     """
+
     def __init__(self,
                  minimum=None,
                  maximum=None,
@@ -2608,7 +2612,7 @@ class ANY_OF(Validator):
         # Use the formatter of the first subvalidator
         # that validates the value and has a formatter
         for validator in self.subs:
-            if hasattr(validator, 'formatter') and validator(value)[1] != None:
+            if hasattr(validator, 'formatter') and validator(value)[1] is None:
                 return validator.formatter(value)
 
 
@@ -2702,6 +2706,7 @@ class LazyCrypt(object):
     """
     Stores a lazy password hash
     """
+
     def __init__(self, crypt, password):
         """
         crypt is an instance of the CRYPT validator,
@@ -3692,6 +3697,7 @@ class IS_IPADDRESS(Validator):
         >>> IS_IPADDRESS(subnets='invalidsubnet')('2001::8ffa:fe22:b3af')
         ('2001::8ffa:fe22:b3af', 'invalid subnet provided')
     """
+
     def __init__(
             self,
             minip='0.0.0.0',

scripts/lang_update_from_langfile.py

@@ -30,8 +30,15 @@ if __name__ == '__main__':
         dest="source",
         help="Specify language file (ro) where seek for translations"
     )
+    parser.add_argument(
+        '-f', '--force-update',
+        dest="force_update",
+        action="store_true",
+        default=False,
+        help="without it: add new + translate untranslated, if used: in addition update items if translation differs"
+    )
     args = parser.parse_args()
 
-    update_from_langfile(args.target, args.source)
+    update_from_langfile(args.target, args.source, force_update=args.force_update)
 
     print '%s was updated.' % args.target

scripts/setup-web2py-nginx-uwsgi-ubuntu.sh

@@ -212,21 +212,20 @@ if [ "$nopassword" -eq 0 ]
 then
    sudo -u www-data python -c "from gluon.main import save_password; save_password('$PW',443)"
 fi
-systemctl enable emperor.uwsgi.service
-systemctl start emperor.uwsgi.service
-/etc/init.d/nginx restart
+
+/etc/init.d/nginx start
+start uwsgi-emperor
 
 echo <<EOF
-you can reload uwsgi with
+you can stop uwsgi and nginx with
 
-  systemctl restart emperor.uwsgi
+  sudo /etc/init.d/nginx stop
+  sudo stop uwsgi-emperor
  
-and stop it with
+and start it with
 
-  systemctl stop emperor.uwsgi
+  sudo /etc/init.d/nginx start
+  sudo start uwsgi-emperor
 
-to reload web2py only (without restarting uwsgi)
-
-  touch /etc/uwsgi/web2py.ini
 EOF