R-2.14.6

simplified oneall dname handling

CHANGELOG

@@ -1,3 +1,11 @@
+## 2.14.6
+
+- Increased test coverage (thanks Richard)
+- Fixed some newly discovered security issues in admin:
+  CSRF vulnerability in admin that allows disabling apps
+  Brute force password attack vulnerability in admin
+  (thanks Narendra and Leonel)  
+
 ## 2.14.1-5
 
 - fixed two major security issues that caused the examples app to leak information

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.5-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

VERSION

@@ -1 +1 @@
-Version 2.14.5-stable+timestamp.2016.04.13.22.22.13
+Version 2.14.6-stable+timestamp.2016.05.09.19.18.48

applications/admin/controllers/default.py

@@ -74,8 +74,10 @@ def log_progress(app, mode='EDIT', filename=None, progress=0):
 def safe_open(a, b):
     if (DEMO_MODE or is_gae) and ('w' in b or 'a' in b):
         class tmp:
+
             def write(self, data):
                 pass
+
             def close(self):
                 pass
         return tmp()
@@ -119,6 +121,9 @@ def index():
         send = URL('site')
     if session.authorized:
         redirect(send)
+    elif failed_login_count() >= allowed_number_of_attempts:
+        time.sleep(2 ** allowed_number_of_attempts)
+        raise HTTP(403)
     elif request.vars.password:
         if verify_password(request.vars.password[:1024]):
             session.authorized = True
@@ -208,6 +213,7 @@ def site():
     file_or_appurl = 'file' in request.vars or 'appurl' in request.vars
 
     class IS_VALID_APPNAME(object):
+
         def __call__(self, value):
             if not re.compile('^\w+$').match(value):
                 return (value, T('Invalid application name'))
@@ -353,6 +359,7 @@ def pack():
         session.flash = T('internal error: %s', e)
         redirect(URL('site'))
 
+
 def pack_plugin():
     app = get_app()
     if len(request.args) == 2:
@@ -368,7 +375,6 @@ def pack_plugin():
         redirect(URL('plugin', args=request.args))
 
 
-
 def pack_exe(app, base, filenames=None):
     import urllib
     import zipfile
@@ -397,10 +403,20 @@ def pack_exe(app, base, filenames=None):
 def pack_custom():
     app = get_app()
     base = apath(app, r=request)
-    if request.post_vars.file:
 
+    def ignore(fs):
+        return [f for f in fs if not (
+                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
+    files = {}
+    for (r, d, f) in os.walk(base):
+        files[r] = {'folders': ignore(d), 'files': ignore(f)}
+
+    if request.post_vars.file:
+        valid_set = set(os.path.relpath(os.path.join(r, f), base) for r in files for f in files[r]['files'])
         files = request.post_vars.file
         files = [files] if not isinstance(files, list) else files
+        files = [file for file in files if file in valid_set]
+
         if request.post_vars.doexe is None:
             fname = 'web2py.app.%s.w2p' % app
             try:
@@ -417,12 +433,7 @@ def pack_custom():
                 redirect(URL(args=request.args))
         else:
             return pack_exe(app, base, files)
-    def ignore(fs):
+
-        return [f for f in fs if not (
-                f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
-    files = {}
-    for (r,d,f) in os.walk(base):
-        files[r] = {'folders':ignore(d),'files':ignore(f)}
     return locals()
 
 
@@ -533,8 +544,8 @@ def delete():
         redirect(URL(sender, anchor=request.vars.id2))
     return dict(dialog=dialog, filename=filename)
 
-
 def enable():
+    if not URL.verify(request, hmac_key=session.hmac_key): raise HTTP(401)
     app = get_app()
     filename = os.path.join(apath(app, r=request), 'DISABLED')
     if is_gae:
@@ -546,6 +557,7 @@ def enable():
         safe_open(filename, 'wb').write('disabled: True\ntime-disabled: %s' % request.now)
         return SPAN(T('Enable'), _style='color:red')
 
+
 def peek():
     """ Visualize object code """
     app = get_app(request.vars.app)
@@ -764,8 +776,8 @@ def edit():
             view = request.args[3].replace('.html', '')
             view_link = URL(request.args[0], request.args[2], view)
     elif filetype == 'python' and request.args[1] == 'controllers':
-        ## it's a controller file.
+        # it's a controller file.
-        ## Create links to all of the associated view files.
+        # Create links to all of the associated view files.
         app = get_app()
         viewname = os.path.splitext(request.args[2])[0]
         viewpath = os.path.join(app, 'views', viewname)
@@ -820,12 +832,14 @@ def edit():
         else:
             return response.json(file_details)
 
+
 def todolist():
     """ Returns all TODO of the requested app
     """
     app = request.vars.app or ''
     app_path = apath('%(app)s' % {'app': app}, r=request)
     dirs = ['models', 'controllers', 'modules', 'private']
+
     def listfiles(app, dir, regexp='.*\.py$'):
         files = sorted(listdir(apath('%(app)s/%(dir)s/' % {'app': app, 'dir': dir}, r=request), regexp))
         files = [x.replace(os.path.sep, '/') for x in files if not x.endswith('.bak')]
@@ -850,6 +864,7 @@ def todolist():
 
     return {'todo': output, 'app': app}
 
+
 def editor_sessions():
     config = Config(os.path.join(request.folder, 'settings.cfg'),
                     section='editor_sessions', default_values={})
@@ -866,6 +881,7 @@ def editor_sessions():
 
     return response.render('default/editor_sessions.html', {'editor_sessions': preferences})
 
+
 def resolve():
     """
     """
@@ -1298,6 +1314,7 @@ def plugin():
                 languages=languages,
                 crontab=crontab)
 
+
 def create_file():
     """ Create files handler """
     if request.vars and not request.vars.token == session.token:
@@ -1473,11 +1490,13 @@ def listfiles(app, dir, regexp='.*\.py$'):
     files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')]
     return files
 
+
 def editfile(path, file, vars={}, app=None):
     args = (path, file) if 'app' in vars else (app, path, file)
     url = URL('edit', args=args, vars=vars)
     return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;')
 
+
 def files_menu():
     app = request.vars.app or 'welcome'
     dirs = [{'name': 'models', 'reg': '.*\.py$'},
@@ -1495,6 +1514,7 @@ def files_menu():
                                        _id=dir['name'] + '_files', _style="display: none;")))
     return dict(result_files=result_files)
 
+
 def upload_file():
     """ File uploading handler """
     if request.vars and not request.vars.token == session.token:
@@ -1867,7 +1887,7 @@ def bulk_register():
         redirect(URL('site'))
     return locals()
 
-### Begin experimental stuff need fixes:
+# Begin experimental stuff need fixes:
 # 1) should run in its own process - cannot os.chdir
 # 2) should not prompt user at console
 # 3) should give option to force commit and not reuqire manual merge
@@ -1934,6 +1954,7 @@ def git_push():
             redirect(URL('site'))
     return dict(app=app, form=form)
 
+
 def plugins():
     app = request.args(0)
     from serializers import loads_json
@@ -1948,12 +1969,16 @@ def plugins():
             session.plugins = []
     return dict(plugins=session.plugins["results"], app=request.args(0))
 
+
 def install_plugin():
     app = request.args(0)
     source = request.vars.source
     plugin = request.vars.plugin
     if not (source and app):
         raise HTTP(500, T("Invalid request"))
+    # make sure no XSS attacks in source
+    if not source.lower().split('://')[0] in ('http','https'):
+        raise HTTP(500, T("Invalid request"))
     form = SQLFORM.factory()
     result = None
     if form.process().accepted:

applications/admin/models/access.py

@@ -4,6 +4,7 @@ import time
 from gluon import portalocker
 from gluon.admin import apath
 from gluon.fileutils import read_file
+from gluon.utils import web2py_uuid
 # ###########################################################
 # ## make sure administrator is on localhost or https
 # ###########################################################
@@ -49,15 +50,18 @@ except IOError:
 def verify_password(password):
     session.pam_user = None
     if DEMO_MODE:
-        return True
+        ret = True
     elif not _config.get('password'):
-        return False
+        ret - False
     elif _config['password'].startswith('pam_user:'):
         session.pam_user = _config['password'][9:].strip()
         import gluon.contrib.pam
-        return gluon.contrib.pam.authenticate(session.pam_user, password)
+        ret = gluon.contrib.pam.authenticate(session.pam_user, password)
     else:
-        return _config['password'] == CRYPT()(password)[0]
+        ret = _config['password'] == CRYPT()(password)[0]
+    if ret:
+        session.hmac_key = web2py_uuid()
+    return ret
 
 
 # ###########################################################
@@ -100,13 +104,12 @@ def write_hosts_deny(denied_hosts):
     portalocker.unlock(f)
     f.close()
 
-
 def login_record(success=True):
     denied_hosts = read_hosts_deny()
     val = (0, 0)
     if success and request.client in denied_hosts:
         del denied_hosts[request.client]
-    elif not success and not request.is_local:
+    elif not success:
         val = denied_hosts.get(request.client, (0, 0))
         if time.time() - val[1] < expiration_failed_logins \
             and val[0] >= allowed_number_of_attempts:
@@ -117,6 +120,11 @@ def login_record(success=True):
     write_hosts_deny(denied_hosts)
     return val[0]
 
+def failed_login_count():
+    denied_hosts = read_hosts_deny()
+    val = denied_hosts.get(request.client, (0, 0))
+    return val[0]
+
 
 # ###########################################################
 # ## session expiration

applications/admin/views/default/site.html

@@ -56,7 +56,7 @@
         {{pass}}
           </ul>
         </div>
-        {{=button_enable(URL('enable',args=a), a) if a!='admin' else ''}}
+        {{=button_enable(URL('enable',args=a, hmac_key=session.hmac_key), a) if a!='admin' else ''}}
       </td>
     </tr>
     {{pass}}

applications/examples/static/css/stupid.css

@@ -68,7 +68,7 @@ header, main, footer {display:block; with:100%} /* IE fix */
 input:not([type]), input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]), [type=file]:before {outline:none; padding:0.5em 1em; margin:0.5px; border-bottom:1px solid #ddd; width:100%}
 textarea {width:100%; border:1px solid #ddd; padding:4px 8px; outline:none; outline:none}
 select {-webkit-appearance:none; outline:none; padding:0.5em 1em; border-radius:0; margin:0.5px; border-bottom:1px solid #ddd; width:100%;background-color:transparent}
-input, textarea, select, button {font-size:12px}
+input, textarea, select, button, .btn {font-size:12px}
 input:not([type]):hover, input:not([type=checkbox]):not([type=radio]):not([type=button]):not([type=submit]):hover, select:hover, textarea:hover {background-color:#fbf6d9; transition:background-color 1s ease}
 input:invalid, input.error {background:#cc1f00;color:white}
 

applications/examples/views/default/who.html

@@ -100,6 +100,7 @@
 </li><li>Michael Willis (shell)
 </li><li>Michele Comitini (facebook)
 </li><li>Michael Toomim (scheduler)
+</li><li>Narendra Bhati (security)
 </li><li>Nathan Freeze (admin design, IS_STRONG, DAL features, <a href="http://web2pyslices.com">web2pyslices.com</a>)
 </li><li>Niall Sweeny (MSSQL support)
 </li><li>Niccolo Polo (epydoc)

fabfile.py

@@ -115,8 +115,9 @@ def deploy(appname=None, all=False):
     """fab -H username@host deploy:appname,all"""
     appname = appname or os.path.split(os.getcwd())[-1]
     appfolder = applications+'/'+appname
-    if os.path.exists('_update.zip'):
+    zipfile = os.path.join(appfolder, '_update.zip')
-        os.unlink('_update.zip')
+    if os.path.exists(zipfile):
+        os.unlink(zipfile)
 
     backup = mkdir_or_backup(appname)
             

gluon/compileapp.py

@@ -676,8 +676,8 @@ def run_view_in(environment):
     else:
         filename = pjoin(folder, 'views', view)
         if os.path.exists(path): # compiled views
-            x = view.replace('/', '.')
+            x = view.replace('/', '_')
-            files = ['views.%s.pyc' % x]
+            files = ['views_%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(path, files[0]))
             # Don't use a generic view if the non-compiled view exists.
             if is_compiled or (not is_compiled and not os.path.exists(filename)):

gluon/contrib/login_methods/oneall_account.py

@@ -51,7 +51,7 @@ class OneallAccount(object):
             reg_id=profile.get('identity_token','')
             username=profile.get('preferredUsername',email)
             first_name=name.get('givenName', dname.split(' ')[0])
-            last_name=profile.get('familyName', dname.split(' ')[1] if(len(dname.split(' ')) > 1) else None)
+            last_name=profile.get('familyName', dname.split(' ')[1] if(dname.count(' ') > 0) else None)
             return dict(registration_id=reg_id,username=username,email=email,
                         first_name=first_name,last_name=last_name)
         self.mappings.default = defaultmapping

gluon/contrib/markdown/markdown2.py

@@ -53,8 +53,9 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * header-ids: Adds "id" attributes to headers. The id value is a slug of
   the header text.
 * html-classes: Takes a dict mapping html tag names (lowercase) to a
-  string to use for a "class" tag attribute. Currently only supports
+  string to use for a "class" tag attribute. Currently only supports "img",
-  "pre" and "code" tags. Add an issue if you require this for other tags.
+  "table", "pre" and "code" tags. Add an issue if you require this for other
+  tags.
 * markdown-in-html: Allow the use of `markdown="1"` in a block HTML tag to
   have markdown processing be done on its contents. Similar to
   <http://michelf.com/projects/php-markdown/extra/#markdown-attr> but with
@@ -70,9 +71,14 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 * smarty-pants: Replaces ' and " with curly quotation marks or curly
   apostrophes.  Replaces --, ---, ..., and . . . with en dashes, em dashes,
   and ellipses.
+* spoiler: A special kind of blockquote commonly hidden behind a
+  click on SO. Syntax per <http://meta.stackexchange.com/a/72878>.
 * toc: The returned HTML string gets a new "toc_html" attribute which is
   a Table of Contents for the document. (experimental)
 * xml: Passes one-liner processing instructions and namespaced XML tags.
+* tables: Tables using the same format as GFM
+  <https://help.github.com/articles/github-flavored-markdown#tables> and
+  PHP-Markdown Extra <https://michelf.ca/projects/php-markdown/extra/#table>.
 * wiki-tables: Google Code Wiki-style tables. See
   <http://code.google.com/p/support/wiki/WikiSyntax#Tables>.
 """
@@ -82,13 +88,11 @@ see <https://github.com/trentm/python-markdown2/wiki/Extras> for details):
 #   not yet sure if there implications with this. Compare 'pydoc sre'
 #   and 'perldoc perlre'.
 
-__version_info__ = (2, 2, 4)
+__version_info__ = (2, 3, 1)
 __version__ = '.'.join(map(str, __version_info__))
 __author__ = "Trent Mick"
 
-import os
 import sys
-from pprint import pprint
 import re
 import logging
 try:
@@ -102,13 +106,7 @@ import codecs
 
 #---- Python version compat
 
-try:
-    from urllib.parse import quote # python3
-except ImportError:
-    from urllib import quote # python2
-
 if sys.version_info[:2] < (2,4):
-    from sets import Set as set
     def reversed(sequence):
         for i in sequence[::-1]:
             yield i
@@ -804,6 +802,8 @@ class Markdown(object):
             text = self._prepare_pyshell_blocks(text)
         if "wiki-tables" in self.extras:
             text = self._do_wiki_tables(text)
+        if "tables" in self.extras:
+            text = self._do_tables(text)
 
         text = self._do_code_blocks(text)
 
@@ -844,6 +844,79 @@ class Markdown(object):
 
         return _pyshell_block_re.sub(self._pyshell_block_sub, text)
 
+    def _table_sub(self, match):
+        trim_space_re = '^[ \t\n]+|[ \t\n]+$'
+        trim_bar_re = '^\||\|$'
+
+        head, underline, body = match.groups()
+
+        # Determine aligns for columns.
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", underline)).split('|')]
+        align_from_col_idx = {}
+        for col_idx, col in enumerate(cols):
+            if col[0] == ':' and col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="center"'
+            elif col[0] == ':':
+                align_from_col_idx[col_idx] = ' align="left"'
+            elif col[-1] == ':':
+                align_from_col_idx[col_idx] = ' align="right"'
+
+        # thead
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<thead>', '<tr>']
+        cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", head)).split('|')]
+        for col_idx, col in enumerate(cols):
+            hlines.append('  <th%s>%s</th>' % (
+                align_from_col_idx.get(col_idx, ''),
+                self._run_span_gamut(col)
+            ))
+        hlines.append('</tr>')
+        hlines.append('</thead>')
+
+        # tbody
+        hlines.append('<tbody>')
+        for line in body.strip('\n').split('\n'):
+            hlines.append('<tr>')
+            cols = [cell.strip() for cell in re.sub(trim_bar_re, "", re.sub(trim_space_re, "", line)).split('|')]
+            for col_idx, col in enumerate(cols):
+                hlines.append('  <td%s>%s</td>' % (
+                    align_from_col_idx.get(col_idx, ''),
+                    self._run_span_gamut(col)
+                ))
+            hlines.append('</tr>')
+        hlines.append('</tbody>')
+        hlines.append('</table>')
+
+        return '\n'.join(hlines) + '\n'
+
+    def _do_tables(self, text):
+        """Copying PHP-Markdown and GFM table syntax. Some regex borrowed from
+        https://github.com/michelf/php-markdown/blob/lib/Michelf/Markdown.php#L2538
+        """
+        less_than_tab = self.tab_width - 1
+        table_re = re.compile(r'''
+                (?:(?<=\n\n)|\A\n?)             # leading blank line
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (.*[|].*)  \n                   # $1: header row (at least one pipe)
+
+                ^[ ]{0,%d}                      # allowed whitespace
+                (                               # $2: underline row
+                    # underline row with leading bar
+                    (?:  \|\ *:?-+:?\ *  )+  \|?  \n
+                    |
+                    # or, underline row without leading bar
+                    (?:  \ *:?-+:?\ *\|  )+  (?:  \ *:?-+:?\ *  )?  \n
+                )
+
+                (                               # $3: data rows
+                    (?:
+                        ^[ ]{0,%d}(?!\ )         # ensure line begins with 0 to less_than_tab spaces
+                        .*\|.*  \n
+                    )+
+                )
+            ''' % (less_than_tab, less_than_tab, less_than_tab), re.M | re.X)
+        return table_re.sub(self._table_sub, text)
+
     def _wiki_table_sub(self, match):
         ttext = match.group(0).strip()
         #print 'wiki table: %r' % match.group(0)
@@ -853,7 +926,7 @@ class Markdown(object):
             row = [c.strip() for c in re.split(r'(?<!\\)\|\|', line)]
             rows.append(row)
         #pprint(rows)
-        hlines = ['<table>', '<tbody>']
+        hlines = ['<table%s>' % self._html_class_str_from_tag('table'), '<tbody>']
         for row in rows:
             hrow = ['<tr>']
             for cell in row:
@@ -899,6 +972,9 @@ class Markdown(object):
 
         text = self._encode_amps_and_angles(text)
 
+        if "strike" in self.extras:
+            text = self._do_strike(text)
+
         text = self._do_italics_and_bold(text)
 
         if "smarty-pants" in self.extras:
@@ -1206,7 +1282,6 @@ class Markdown(object):
                                  .replace('_', self._escape_table['_'])
                         title = self.titles.get(link_id)
                         if title:
-                            before = title
                             title = _xml_escape_attr(title) \
                                 .replace('*', self._escape_table['*']) \
                                 .replace('_', self._escape_table['_'])
@@ -1418,7 +1493,6 @@ class Markdown(object):
     def _list_item_sub(self, match):
         item = match.group(4)
         leading_line = match.group(1)
-        leading_space = match.group(2)
         if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
             item = self._run_block_gamut(self._outdent(item))
         else:
@@ -1654,6 +1728,11 @@ class Markdown(object):
         self._escape_table[text] = hashed
         return hashed
 
+    _strike_re = re.compile(r"~~(?=\S)(.+?)(?<=\S)~~", re.S)
+    def _do_strike(self, text):
+        text = self._strike_re.sub(r"<strike>\1</strike>", text)
+        return text
+
     _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
     _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
     _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
@@ -1714,37 +1793,52 @@ class Markdown(object):
         text = text.replace(". . .", "&#8230;")
         return text
 
-    _block_quote_re = re.compile(r'''
+    _block_quote_base = r'''
         (                           # Wrap whole match in \1
           (
-            ^[ \t]*>[ \t]?          # '>' at the start of a line
+            ^[ \t]*>%s[ \t]?        # '>' at the start of a line
               .+\n                  # rest of the first line
             (.+\n)*                 # subsequent consecutive lines
             \n*                     # blanks
           )+
         )
-        ''', re.M | re.X)
+    '''
+    _block_quote_re = re.compile(_block_quote_base % '', re.M | re.X)
+    _block_quote_re_spoiler = re.compile(_block_quote_base % '[ \t]*?!?', re.M | re.X)
     _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M);
-
+    _bq_one_level_re_spoiler = re.compile('^[ \t]*>[ \t]*?![ \t]?', re.M);
+    _bq_all_lines_spoilers = re.compile(r'\A(?:^[ \t]*>[ \t]*?!.*[\n\r]*)+\Z', re.M)
     _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
     def _dedent_two_spaces_sub(self, match):
         return re.sub(r'(?m)^  ', '', match.group(1))
 
     def _block_quote_sub(self, match):
         bq = match.group(1)
-        bq = self._bq_one_level_re.sub('', bq)  # trim one level of quoting
+        is_spoiler = 'spoiler' in self.extras and self._bq_all_lines_spoilers.match(bq)
-        bq = self._ws_only_line_re.sub('', bq)  # trim whitespace-only lines
+        # trim one level of quoting
+        if is_spoiler:
+            bq = self._bq_one_level_re_spoiler.sub('', bq)
+        else:
+            bq = self._bq_one_level_re.sub('', bq)
+        # trim whitespace-only lines
+        bq = self._ws_only_line_re.sub('', bq)
         bq = self._run_block_gamut(bq)          # recurse
 
         bq = re.sub('(?m)^', '  ', bq)
         # These leading spaces screw with <pre> content, so we need to fix that:
         bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
 
-        return "<blockquote>\n%s\n</blockquote>\n\n" % bq
+        if is_spoiler:
+            return '<blockquote class="spoiler">\n%s\n</blockquote>\n\n' % bq
+        else:
+            return '<blockquote>\n%s\n</blockquote>\n\n' % bq
 
     def _do_block_quotes(self, text):
         if '>' not in text:
             return text
+        if 'spoiler' in self.extras:
+            return self._block_quote_re_spoiler.sub(self._block_quote_sub, text)
+        else:
             return self._block_quote_re.sub(self._block_quote_sub, text)
 
     def _form_paragraphs(self, text):
@@ -2053,7 +2147,6 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False):
     if DEBUG:
         print("dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
               % (tabsize, skip_first_line))
-    indents = []
     margin = None
     for i, line in enumerate(lines):
         if i == 0 and skip_first_line: continue

gluon/contrib/redis_scheduler.py

@@ -9,6 +9,17 @@ Scheduler with redis backend
 ---------------------------------
 """
 
+import os
+import time
+import socket
+import datetime
+import logging
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+from gluon.scheduler import *
+from gluon.scheduler import _decode_dict
+from gluon.contrib.redis_utils import RWatchError
+
 USAGE = """
 ## Example
 
@@ -35,11 +46,6 @@ mysched = RScheduler(db, dict(demo1=demo1,demo2=demo2), ...., redis_conn=rconn)
 
 """
 
-import os
-import time
-import socket
-import datetime
-import logging
 
 path = os.getcwd()
 
@@ -47,20 +53,19 @@ if 'WEB2PY_PATH' not in os.environ:
     os.environ['WEB2PY_PATH'] = path
 
 try:
-    from gluon.contrib.simplejson import loads, dumps
+    # try external module
-except:
     from simplejson import loads, dumps
+except ImportError:
+    try:
+        # try stdlib (Python >= 2.6)
+        from json import loads, dumps
+    except:
+        # fallback to pure-Python module
+        from gluon.contrib.simplejson import loads, dumps
 
 IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
-logger = logging.getLogger('web2py.rscheduler.%s' % IDENTIFIER)
+logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
-
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-from gluon.scheduler import *
-from gluon.scheduler import _decode_dict
-from gluon.contrib.redis_utils import RWatchError
-
 
 POLLING = 'POLLING'
 
@@ -111,8 +116,7 @@ class RScheduler(Scheduler):
         self._application = current.request.application or 'appname'
 
     def _nkey(self, key):
-        """Helper to restrict all keys to a namespace
+        """Helper to restrict all keys to a namespace and track them."""
-        and track them"""
         prefix = 'w2p:rsched:%s' % self._application
         allkeys = '%s:allkeys' % prefix
         newkey = "%s:%s" % (prefix, key)
@@ -120,10 +124,7 @@ class RScheduler(Scheduler):
         return newkey
 
     def prune_all(self):
-        """
+        """Global housekeeping."""
-        Just to be fair and implement a method
-        that does housekeeping
-        """
         all_keys = self._nkey('allkeys')
         with self.r_server.pipeline() as pipe:
             while True:
@@ -148,8 +149,9 @@ class RScheduler(Scheduler):
 
     def send_heartbeat(self, counter):
         """
-        workers coordination has evolved into something is not that
+        Workers coordination in redis.
-        easy. Here we try to do what we need in a single transaction,
+        It has evolved into something is not that easy.
+        Here we try to do what we need in a single transaction,
         and retry that transaction if something goes wrong
         """
         with self.r_server.pipeline() as pipe:
@@ -167,7 +169,9 @@ class RScheduler(Scheduler):
 
     def inner_send_heartbeat(self, counter, pipe):
         """
-        Does a few things:
+        Do a few things in the "maintenance" thread.
+
+        Specifically:
         - registers the workers
         - accepts commands sent to workers (KILL, TERMINATE, PICK, DISABLED, etc)
         - adjusts sleep
@@ -269,6 +273,8 @@ class RScheduler(Scheduler):
 
     def being_a_ticker(self, pipe):
         """
+        Elects a ticker.
+
         This is slightly more convoluted than the original
         but if far more efficient
         """
@@ -311,7 +317,9 @@ class RScheduler(Scheduler):
 
     def assign_tasks(self, db):
         """
-        The real beauty. We don't need to ASSIGN tasks, we just put
+        The real beauty.
+
+        We don't need to ASSIGN tasks, we just put
         them into the relevant queue
         """
         st, sd = db.scheduler_task, db.scheduler_task_deps
@@ -375,9 +383,6 @@ class RScheduler(Scheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -437,6 +442,7 @@ class RScheduler(Scheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def pop_task(self, db):
+        """Lift a task off a queue."""
         r_server = self.r_server
         st = self.db.scheduler_task
         task = None
@@ -533,7 +539,9 @@ class RScheduler(Scheduler):
 
     def report_task(self, task, task_report):
         """
-        Needs overwriting only because we need to pop from the
+        Override.
+
+        Needs it only because we need to pop from the
         running tasks
         """
         r_server = self.r_server
@@ -558,12 +566,12 @@ class RScheduler(Scheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
+        is_expired = (task.stop_time and
-                      and task.next_run_time > task.stop_time
+                      task.next_run_time > task.stop_time and
-                      and True or False)
+                      True or False)
-        status = (task.run_again and is_expired and EXPIRED
+        status = (task.run_again and is_expired and EXPIRED or
-                  or task.run_again and not is_expired
+                  task.run_again and not is_expired and
-                  and QUEUED or COMPLETED)
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             # assigned calculations
             d = dict(status=status,
@@ -579,10 +587,10 @@ class RScheduler(Scheduler):
             st_mapping = {'FAILED': 'FAILED',
                           'TIMEOUT': 'TIMEOUT',
                           'STOPPED': 'FAILED'}[task_report.status]
-            status = (task.retry_failed
+            status = (task.retry_failed and
-                      and task.times_failed < task.retry_failed
+                      task.times_failed < task.retry_failed and
-                      and QUEUED or task.retry_failed == -1
+                      QUEUED or task.retry_failed == -1 and
-                      and QUEUED or st_mapping)
+                      QUEUED or st_mapping)
             db(st.id == task.task_id).update(
                 times_failed=st.times_failed + 1,
                 next_run_time=task.next_run_time,
@@ -596,7 +604,7 @@ class RScheduler(Scheduler):
         r_server.hdel(running_dict, task.task_id)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -620,8 +628,8 @@ class RScheduler(Scheduler):
                 time.sleep(0.5)
 
     def get_workers(self, only_ticker=False):
-        """ Returns a dict holding worker_name : {**columns}
+        """Return a dict holding worker_name : {**columns}
-        representing all "registered" workers
+        representing all "registered" workers.
         only_ticker returns only the worker running as a TICKER,
         if there is any
         """

gluon/html.py

@@ -330,11 +330,9 @@ def URL(a=None,
     if args:
         if url_encode:
             if encode_embedded_slash:
-                other = '/' + '/'.join([urllib.quote(str(
+                other = '/' + '/'.join([urllib.quote(str(x), '') for x in args])
-                    x), '') for x in args])
             else:
-                other = args and urllib.quote(
+                other = args and urllib.quote('/' + '/'.join([str(x) for x in args]))
-                    '/' + '/'.join([str(x) for x in args]))
         else:
             other = args and ('/' + '/'.join([str(x) for x in args]))
     else:

gluon/languages.py

@@ -1002,17 +1002,19 @@ def update_all_languages(application_path):
             findT(application_path, language[:-3])
 
 
-def update_from_langfile(target, source):
+def update_from_langfile(target, source, force_update=False):
     """this will update untranslated messages in target from source (where both are language files)
     this can be used as first step when creating language file for new but very similar language
         or if you want update your app from welcome app of newer web2py version
         or in non-standard scenarios when you work on target and from any reason you have partial translation in source
+    Args:
+        force_update: if False existing translations remain unchanged, if True existing translations will update from source
     """
     src = read_dict(source)
     sentences = read_dict(target)
     for key in sentences:
         val = sentences[key]
-        if not val or val == key:
+        if not val or val == key or force_update:
             new_val = src.get(key)
             if new_val and new_val != val:
                 sentences[key] = new_val

gluon/scheduler.py

@@ -9,6 +9,26 @@ Background processes made simple
 ---------------------------------
 """
 
+
+import os
+import time
+import multiprocessing
+import sys
+import threading
+import traceback
+import signal
+import socket
+import datetime
+import logging
+import optparse
+import tempfile
+import types
+import Queue
+from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
+from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
+from gluon.utils import web2py_uuid
+from gluon.storage import Storage
+
 USAGE = """
 ## Example
 
@@ -67,20 +87,6 @@ sudo restart web2py-scheduler
 sudo status web2py-scheduler
 """
 
-import os
-import time
-import multiprocessing
-import sys
-import threading
-import traceback
-import signal
-import socket
-import datetime
-import logging
-import optparse
-import types
-import Queue
-
 path = os.getcwd()
 
 if 'WEB2PY_PATH' not in os.environ:
@@ -101,12 +107,6 @@ IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
 logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
-from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
-from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
-from gluon.utils import web2py_uuid
-from gluon.storage import Storage
-
-
 QUEUED = 'QUEUED'
 ASSIGNED = 'ASSIGNED'
 RUNNING = 'RUNNING'
@@ -168,24 +168,25 @@ class TaskReport(object):
 
 
 class JobGraph(object):
-    """Experimental: with JobGraph you can specify
+    """Experimental: dependencies amongs tasks"""
-    dependencies amongs tasks"""
 
     def __init__(self, db, job_name):
         self.job_name = job_name or 'job_0'
         self.db = db
 
     def add_deps(self, task_parent, task_child):
-        """Creates a dependency between task_parent and task_child"""
+        """Create a dependency between task_parent and task_child."""
         self.db.scheduler_task_deps.insert(task_parent=task_parent,
                                            task_child=task_child,
                                            job_name=self.job_name)
 
     def validate(self, job_name=None):
-        """Validates if all tasks job_name can be completed, i.e. there
+        """Validate if all tasks job_name can be completed.
-        are no mutual dependencies among tasks.
+
+        Checks if there are no mutual dependencies among tasks.
         Commits at the end if successfull, or it rollbacks the entire
-        transaction. Handle with care!"""
+        transaction. Handle with care!
+        """
         db = self.db
         sd = db.scheduler_task_deps
         if job_name:
@@ -223,14 +224,6 @@ class JobGraph(object):
             db.rollback()
             return None
 
-
-def demo_function(*argv, **kwargs):
-    """ test function """
-    for i in range(argv[0]):
-        print 'click', i
-        time.sleep(1)
-    return 'done'
-
 # the two functions below deal with simplejson decoding as unicode, esp for the dict decode
 # and subsequent usage as function Keyword arguments unicode variable names won't work!
 # borrowed from http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-unicode-ones-from-json-in-python
@@ -261,11 +254,12 @@ def _decode_dict(dct):
 
 
 def executor(queue, task, out):
-    """The function used to execute tasks in the background process"""
+    """The function used to execute tasks in the background process."""
     logger.debug('    task started')
 
     class LogOutput(object):
-        """Facility to log output at intervals"""
+        """Facility to log output at intervals."""
+
         def __init__(self, out_queue):
             self.out_queue = out_queue
             self.stdout = sys.stdout
@@ -280,7 +274,11 @@ def executor(queue, task, out):
         def write(self, data):
             self.out_queue.put(data)
 
-    W2P_TASK = Storage({'id': task.task_id, 'uuid': task.uuid})
+    W2P_TASK = Storage({
+                       'id': task.task_id,
+                       'uuid': task.uuid,
+                       'run_id': task.run_id
+                       })
     stdout = LogOutput(out)
     try:
         if task.app:
@@ -318,6 +316,11 @@ def executor(queue, task, out):
             result = eval(task.function)(
                 *loads(task.args, object_hook=_decode_dict),
                 **loads(task.vars, object_hook=_decode_dict))
+        if len(result) >= 1024:
+            fd, temp_path = tempfile.mkstemp(suffix='.w2p_sched')
+            with os.fdopen(fd, 'w') as f:
+                f.write(result)
+            result = 'w2p_special:%s' % temp_path
         queue.put(TaskReport('COMPLETED', result=result))
     except BaseException, e:
         tb = traceback.format_exc()
@@ -335,7 +338,7 @@ class MetaScheduler(threading.Thread):
         self.empty_runs = 0
 
     def async(self, task):
-        """Starts the background process
+        """Start the background process.
 
         Args:
             task : a `Task` object
@@ -410,6 +413,12 @@ class MetaScheduler(threading.Thread):
             else:
                 logger.debug('  task completed or failed')
                 tr = queue.get()
+        result = tr.result
+        if result and result.startswith('w2p_special'):
+            temp_path = result.replace('w2p_special:', '', 1)
+            with open(temp_path) as f:
+                tr.result = f.read()
+            os.unlink(temp_path)
         tr.output = task_output
         return tr
 
@@ -444,50 +453,23 @@ class MetaScheduler(threading.Thread):
         self.start()
 
     def send_heartbeat(self, counter):
-        print 'thum'
+        raise NotImplementedError
-        time.sleep(1)
 
     def pop_task(self):
         """Fetches a task ready to be executed"""
-        return Task(
+        raise NotImplementedError
-            app=None,
-            function='demo_function',
-            timeout=7,
-            args='[2]',
-            vars='{}')
 
     def report_task(self, task, task_report):
         """Creates a task report"""
-        print 'reporting task'
+        raise NotImplementedError
-        pass
 
     def sleep(self):
-        pass
+        raise NotImplementedError
 
     def loop(self):
         """Main loop, fetching tasks and starting executor's background
         processes"""
-        try:
+        raise NotImplementedError
-            self.start_heartbeats()
-            while True and self.have_heartbeat:
-                logger.debug('looping...')
-                task = self.pop_task()
-                if task:
-                    self.empty_runs = 0
-                    self.report_task(task, self.async(task))
-                else:
-                    self.empty_runs += 1
-                    logger.debug('sleeping...')
-                    if self.max_empty_runs != 0:
-                        logger.debug('empty runs %s/%s',
-                                     self.empty_runs, self.max_empty_runs)
-                        if self.empty_runs >= self.max_empty_runs:
-                            logger.info(
-                                'empty runs limit reached, killing myself')
-                            self.die()
-                    self.sleep()
-        except KeyboardInterrupt:
-            self.die()
 
 
 TASK_STATUS = (QUEUED, RUNNING, COMPLETED, FAILED, TIMEOUT, STOPPED, EXPIRED)
@@ -594,11 +576,11 @@ class Scheduler(MetaScheduler):
         return True
 
     def now(self):
-        """Shortcut that fetches current time based on UTC preferences"""
+        """Shortcut that fetches current time based on UTC preferences."""
         return self.utc_time and datetime.datetime.utcnow() or datetime.datetime.now()
 
     def set_requirements(self, scheduler_task):
-        """Called to set defaults for lazy_tables connections"""
+        """Called to set defaults for lazy_tables connections."""
         from gluon import current
         if hasattr(current, 'request'):
             scheduler_task.application_name.default = '%s/%s' % (
@@ -606,7 +588,7 @@ class Scheduler(MetaScheduler):
             )
 
     def define_tables(self, db, migrate):
-        """Defines Scheduler tables structure"""
+        """Define Scheduler tables structure."""
         from pydal.base import DEFAULT
         logger.debug('defining tables (migrate=%s)', migrate)
         now = self.now
@@ -693,14 +675,14 @@ class Scheduler(MetaScheduler):
 
     @staticmethod
     def total_seconds(td):
-        # backport for py2.6
+        """Backport for py2.6."""
         if hasattr(td, 'total_seconds'):
             return td.total_seconds()
         else:
             return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6
 
     def loop(self, worker_name=None):
-        """Main loop
+        """Main loop.
 
         This works basically as a neverending loop that:
 
@@ -752,7 +734,8 @@ class Scheduler(MetaScheduler):
             self.die()
 
     def wrapped_assign_tasks(self, db):
-        """Commodity function to call `assign_tasks` and trap exceptions
+        """Commodity function to call `assign_tasks` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `assign_task` after 0.5 seconds
         """
@@ -773,7 +756,8 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def wrapped_pop_task(self):
-        """Commodity function to call `pop_task` and trap exceptions
+        """Commodity function to call `pop_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -793,7 +777,7 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def pop_task(self, db):
-        """Grabs a task ready to be executed from the queue"""
+        """Grab a task ready to be executed from the queue."""
         now = self.now()
         st = self.db.scheduler_task
         if self.is_a_ticker and self.do_assign_tasks:
@@ -874,7 +858,8 @@ class Scheduler(MetaScheduler):
             uuid=task.uuid)
 
     def wrapped_report_task(self, task, task_report):
-        """Commodity function to call `report_task` and trap exceptions
+        """Commodity function to call `report_task` and trap exceptions.
+
         If an exception is raised, assume it happened because of database
         contention and retries `pop_task` after 0.5 seconds
         """
@@ -891,8 +876,10 @@ class Scheduler(MetaScheduler):
                 time.sleep(0.5)
 
     def report_task(self, task, task_report):
-        """Takes care of storing the result according to preferences
+        """Take care of storing the result according to preferences.
-        and deals with logic for repeating tasks"""
+
+        Deals with logic for repeating tasks.
+        """
         db = self.db
         now = self.now()
         st = db.scheduler_task
@@ -914,12 +901,12 @@ class Scheduler(MetaScheduler):
                 logger.debug(' deleting task report in db because of no result')
                 db(sr.id == task.run_id).delete()
         # if there is a stop_time and the following run would exceed it
-        is_expired = (task.stop_time
+        is_expired = (task.stop_time and
-                      and task.next_run_time > task.stop_time
+                      task.next_run_time > task.stop_time and
-                      and True or False)
+                      True or False)
-        status = (task.run_again and is_expired and EXPIRED
+        status = (task.run_again and is_expired and EXPIRED or
-                  or task.run_again and not is_expired
+                  task.run_again and not is_expired and
-                  and QUEUED or COMPLETED)
+                  QUEUED or COMPLETED)
         if task_report.status == COMPLETED:
             d = dict(status=status,
                      next_run_time=task.next_run_time,
@@ -945,27 +932,26 @@ class Scheduler(MetaScheduler):
         logger.info('task completed (%s)', task_report.status)
 
     def update_dependencies(self, db, task_id):
+        """Unblock execution paths for Jobs."""
         db(db.scheduler_task_deps.task_child == task_id).update(can_visit=True)
 
     def adj_hibernation(self):
-        """Used to increase the "sleep" interval for DISABLED workers"""
+        """Used to increase the "sleep" interval for DISABLED workers."""
         if self.w_stats.status == DISABLED:
             wk_st = self.w_stats.sleep
             hibernation = wk_st + HEARTBEAT if wk_st < MAXHIBERNATION else MAXHIBERNATION
             self.w_stats.sleep = hibernation
 
     def send_heartbeat(self, counter):
-        """This function is vital for proper coordination among available
+        """Coordination among available workers.
-        workers.
-        It:
 
+        It:
         - sends the heartbeat
         - elects a ticker among available workers (the only process that
             effectively dispatch tasks to workers)
         - deals with worker's statuses
         - does "housecleaning" for dead workers
         - triggers tasks assignment to workers
-
         """
         if not self.db_thread:
             logger.debug('thread building own DAL object')
@@ -1053,7 +1039,8 @@ class Scheduler(MetaScheduler):
         self.sleep()
 
     def being_a_ticker(self):
-        """Elects a TICKER process that assigns tasks to available workers.
+        """Elect a TICKER process that assigns tasks to available workers.
+
         Does its best to elect a worker that is not busy processing other tasks
         to allow a proper distribution of tasks among all active workers ASAP
         """
@@ -1087,7 +1074,7 @@ class Scheduler(MetaScheduler):
             return False
 
     def assign_tasks(self, db):
-        """Assigns task to workers, that can then pop them from the queue
+        """Assign task to workers, that can then pop them from the queue.
 
         Deals with group_name(s) logic, in order to assign linearly tasks
         to available workers for those groups
@@ -1137,9 +1124,6 @@ class Scheduler(MetaScheduler):
 
         all_available = db(
             (st.status.belongs((QUEUED, ASSIGNED))) &
-            ((st.times_run < st.repeats) | (st.repeats == 0)) &
-            (st.start_time <= now) &
-            ((st.stop_time == None) | (st.stop_time > now)) &
             (st.next_run_time <= now) &
             (st.enabled == True) &
             (st.id.belongs(no_deps))
@@ -1151,8 +1135,8 @@ class Scheduler(MetaScheduler):
         # intelligence (like esteeming how many tasks will a worker complete
         # before the ticker reassign them around, but the gain is quite small
         # 50 is a sweet spot also for fast tasks, with sane heartbeat values
-        # NB: ticker reassign tasks every 5 cycles, so if a worker completes its
+        # NB: ticker reassign tasks every 5 cycles, so if a worker completes
-        # 50 tasks in less than heartbeat*5 seconds,
+        # its 50 tasks in less than heartbeat*5 seconds,
         # it won't pick new tasks until heartbeat*5 seconds pass.
 
         # If a worker is currently elaborating a long task, its tasks needs to
@@ -1183,8 +1167,6 @@ class Scheduler(MetaScheduler):
                         status=ASSIGNED,
                         assigned_worker_name=assigned_wn
                     )
-                    if not task.task_name:
-                        d['task_name'] = task.function_name
                     db(
                         (st.id == task.id) &
                         (st.status.belongs((QUEUED, ASSIGNED)))
@@ -1204,14 +1186,13 @@ class Scheduler(MetaScheduler):
         logger.info('TICKER: tasks are %s', x)
 
     def sleep(self):
-        """Calculates the number of seconds to sleep according to worker's
+        """Calculate the number of seconds to sleep."""
-        status and `heartbeat` parameter"""
         time.sleep(self.w_stats.sleep)
         # should only sleep until next available task
 
     def set_worker_status(self, group_names=None, action=ACTIVE,
                           exclude=None, limit=None, worker_name=None):
-        """Internal function to set worker's status"""
+        """Internal function to set worker's status."""
         ws = self.db.scheduler_worker
         if not group_names:
             group_names = self.group_names
@@ -1235,10 +1216,12 @@ class Scheduler(MetaScheduler):
                 self.db(ws.id.belongs(workers)).update(status=action)
 
     def disable(self, group_names=None, limit=None, worker_name=None):
-        """Sets DISABLED on the workers processing `group_names` tasks.
+        """Set DISABLED on the workers processing `group_names` tasks.
+
         A DISABLED worker will be kept alive but it won't be able to process
         any waiting tasks, essentially putting it to sleep.
-        By default, all group_names of Scheduler's instantation are selected"""
+        By default, all group_names of Scheduler's instantation are selected
+        """
         self.set_worker_status(
             group_names=group_names,
             action=DISABLED,
@@ -1283,8 +1266,9 @@ class Scheduler(MetaScheduler):
             pvars: "raw" kwargs to be passed to the function. Automatically
                 jsonified
             kwargs: all the parameters available (basically, every
-                `scheduler_task` column). If args and vars are here, they should
+                `scheduler_task` column). If args and vars are here, they
-                be jsonified already, and they will override pargs and pvars
+                should be jsonified already, and they will override pargs
+                and pvars
 
         Returns:
             a dict just as a normal validate_and_insert(), plus a uuid key

gluon/tests/__init__.py

@@ -21,6 +21,7 @@ from test_contribs import *
 from test_web import *
 from test_dal import *
 from test_tools import *
+from test_appadmin import *
 from test_scheduler import *
 
 if sys.version[:3] == '2.7':

gluon/tests/test_appadmin.py

@@ -0,0 +1,175 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+    Unit tests for gluon.sqlhtml
+"""
+import os
+import sys
+if sys.version < "2.7":
+    import unittest2 as unittest
+else:
+    import unittest
+
+from fix_path import fix_sys_path
+
+fix_sys_path(__file__)
+
+
+from compileapp import run_controller_in, run_view_in
+from languages import translator
+from gluon.storage import Storage, List
+import gluon.fileutils
+from gluon.dal import DAL, Field, Table
+from gluon.http import HTTP
+
+DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
+
+try:
+    import json
+except ImportError:
+    from gluon.contrib import simplejson as json
+
+
+def fake_check_credentials(foo):
+    return True
+
+
+class TestAppAdmin(unittest.TestCase):
+
+    def setUp(self):
+        from gluon.globals import Request, Response, Session, current
+        from gluon.html import A, DIV, FORM, MENU, TABLE, TR, INPUT, URL, XML
+        from gluon.validators import IS_NOT_EMPTY
+        from compileapp import LOAD
+        from gluon.http import HTTP, redirect
+        from gluon.tools import Auth
+        from gluon.sql import SQLDB
+        from gluon.sqlhtml import SQLTABLE, SQLFORM
+        self.original_check_credentials = gluon.fileutils.check_credentials
+        gluon.fileutils.check_credentials = fake_check_credentials
+        request = Request(env={})
+        request.application = 'welcome'
+        request.controller = 'appadmin'
+        request.function = self._testMethodName.split('_')[1]
+        request.folder = 'applications/welcome'
+        request.env.http_host = '127.0.0.1:8000'
+        request.env.remote_addr = '127.0.0.1'
+        response = Response()
+        session = Session()
+        T = translator('', 'en')
+        session.connect(request, response)
+        current.request = request
+        current.response = response
+        current.session = session
+        current.T = T
+        db = DAL(DEFAULT_URI, check_reserved=['all'])
+        auth = Auth(db)
+        auth.define_tables(username=True, signature=False)
+        db.define_table('t0', Field('tt'), auth.signature)
+        # Create a user
+        db.auth_user.insert(first_name='Bart',
+                            last_name='Simpson',
+                            username='user1',
+                            email='user1@test.com',
+                            password='password_123',
+                            registration_key=None,
+                            registration_id=None)
+        self.env = locals()
+
+    def tearDown(self):
+        gluon.fileutils.check_credentials = self.original_check_credentials
+
+    def run_function(self):
+        return run_controller_in(self.env['request'].controller, self.env['request'].function, self.env)
+
+    def run_view(self):
+        return run_view_in(self.env)
+
+    def test_index(self):
+        result = self.run_function()
+        self.assertTrue('db' in result['databases'])
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_select(self):
+        request = self.env['request']
+        request.args = List(['db'])
+        request.env.query_string = 'query=db.auth_user.id>0'
+        result = self.run_function()
+        self.assertTrue('table' in result and 'query' in result)
+        self.assertTrue(result['table'] == 'auth_user')
+        self.assertTrue(result['query'] == 'db.auth_user.id>0')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        result = self.run_function()
+        self.assertTrue('table' in result)
+        self.assertTrue('form' in result)
+        self.assertTrue(str(result['table']) is 'auth_user')
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+
+    def test_insert_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        data['first_name'] = 'Lisa'
+        data['last_name'] = 'Simpson'
+        data['username'] = 'lisasimpson'
+        data['password'] = 'password_123'
+        data['email'] = 'lisa@example.com'
+        request._vars = data
+        result = self.run_function()
+        self.env.update(result)
+        try:
+            self.run_view()
+        except Exception as e:
+            print e.message
+            self.fail('Could not make the view')
+        db = self.env['db']
+        lisa_record = db(db.auth_user.username == 'lisasimpson').select().first()
+        self.assertIsNotNone(lisa_record)
+        del data['_formkey']
+        del data['_formname']
+        del data['password']
+        for key in data:
+            self.assertEqual(data[key], lisa_record[key])
+
+    def test_update_submit(self):
+        request = self.env['request']
+        request.args = List(['db', 'auth_user', '1'])
+        form = self.run_function()['form']
+        hidden_fields = form.hidden_fields()
+        data = {}
+        data['_formkey'] = hidden_fields.element('input', _name='_formkey')['_value']
+        data['_formname'] = hidden_fields.element('input', _name='_formname')['_value']
+        for element in form.elements('input'):
+            data[element['_name']] = element['_value']
+        data['email'] = 'user1@example.com'
+        data['id'] = '1'
+        request._vars = data
+        self.assertRaises(HTTP, self.run_function)
+
+
+if __name__ == '__main__':
+    unittest.main()

gluon/tests/test_html.py

@@ -98,6 +98,11 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
         rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'p': (1, 3), 'q': 2}, hmac_key='key', hash_vars='p')
         self.assertEqual(rtn, '/a/c/f/x/y/z?p=1&p=3&q=2&_signature=5d01b982fd72b39674b012e0288071034e156d7a')
+        # test url_encode
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=False)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li\xc3\xa9=2&ma\xc3\xaf=1&ma\xc3\xaf=3')
+        rtn = URL('a', 'c', 'f', args=['x', 'y', 'z'], vars={'maï': (1, 3), 'lié': 2}, url_encode=True)
+        self.assertEqual(rtn, '/a/c/f/x/y/z?li%C3%A9=2&ma%C3%AF=1&ma%C3%AF=3')
         # test CRLF detection
         self.assertRaises(SyntaxError, URL, *['a\n', 'c', 'f'])
         self.assertRaises(SyntaxError, URL, *['a\r', 'c', 'f'])
@@ -394,18 +399,12 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(HR(_a='1', _b='2').xml(), '<hr a="1" b="2" />')
 
     def test_A(self):
-        self.assertEqual(
+        self.assertEqual(A('<>', _a='1', _b='2').xml(),
-            A('<>', _a='1', _b='2').xml(),
+                         '<a a="1" b="2">&lt;&gt;</a>')
-            '<a a="1" b="2">&lt;&gt;</a>'
+        self.assertEqual(A('a', cid='b').xml(),
-            )
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', callback='b', _id='c').xml(),
-            A('a', cid='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="c">a</a>'
-            )
         # Callback with no id trigger web2py_uuid() call
         from html import web2pyHTMLParser
         a = A('a', callback='b').xml()
@@ -413,38 +412,22 @@ class TestBareHelpers(unittest.TestCase):
             uuid_generated = tag.attributes['_id']
         self.assertEqual(a,
                          '<a data-w2p_disable_with="default" data-w2p_method="POST" href="b" id="{id}">a</a>'.format(id=uuid_generated))
-        self.assertEqual(
+        self.assertEqual(A('a', delete='tr').xml(),
-            A('a', delete='tr').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_remove="tr">a</a>'
+        self.assertEqual(A('a', _id='b', target='<self>').xml(),
-            )
+                         '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', component='b').xml(),
-            A('a', _id='b', target='<self>').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_target="b" id="b">a</a>'
+        self.assertEqual(A('a', _id='b', callback='c', noconfirm=True).xml(),
-            )
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', cid='b').xml(),
-            A('a', component='b').xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" href="b">a</a>'
+        self.assertEqual(A('a', cid='b', _disable_with='processing...').xml(),
-            )
+                         '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>')
-        self.assertEqual(
+        self.assertEqual(A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
-            A('a', _id='b', callback='c', noconfirm=True).xml(),
+                         '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" href="c" id="b">a</a>'
+        self.assertEqual(A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
-            )
+                         '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>')
-        self.assertEqual(
-            A('a', cid='b').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', cid='b', _disable_with='processing...').xml(),
-            '<a data-w2p_disable_with="processing..." data-w2p_method="GET" data-w2p_target="b">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', noconfirm=True, _id='c').xml(),
-            '<a data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
-        self.assertEqual(
-            A('a', callback='b', delete='tr', confirm='Are you sure?', _id='c').xml(),
-            '<a data-w2p_confirm="Are you sure?" data-w2p_disable_with="default" data-w2p_method="POST" data-w2p_remove="tr" href="b" id="c">a</a>'
-            )
 
     def test_BUTTON(self):
         self.assertEqual(BUTTON('test', _type='button').xml(),

gluon/tests/test_recfile.py

@@ -23,8 +23,8 @@ class TestRecfile(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree('tests')
 
-    def testgeneration(self):
+    def test_generation(self):
-        for k in range(20):
+        for k in range(10):
             teststring = 'test%s' % k
             filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
             with recfile.open(filename, "w") as g:
@@ -35,6 +35,40 @@ class TestRecfile(unittest.TestCase):
             recfile.remove(filename)
             is_there = recfile.exists(filename)
             self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = str(uuid.uuid4()) + '.test'
+            with recfile.open(filename, "w", path='tests') as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r", path='tests').read(), teststring)
+            is_there = recfile.exists(filename, path='tests')
+            self.assertTrue(is_there)
+            recfile.remove(filename, path='tests')
+            is_there = recfile.exists(filename, path='tests')
+            self.assertFalse(is_there)
+        for k in range(10):
+            teststring = 'test%s' % k
+            filename = os.path.join('tests', str(uuid.uuid4()), str(uuid.uuid4()) + '.test')
+            with recfile.open(filename, "w") as g:
+                g.write(teststring)
+            self.assertEqual(recfile.open(filename, "r").read(), teststring)
+            is_there = recfile.exists(filename)
+            self.assertTrue(is_there)
+            recfile.remove(filename)
+            is_there = recfile.exists(filename)
+            self.assertFalse(is_there)
+
+    def test_existing(self):
+        filename = os.path.join('tests', str(uuid.uuid4()) + '.test')
+        with open(filename, 'w') as g:
+            g.write('this file exists')
+        self.assertTrue(recfile.exists(filename))
+        self.assertTrue(hasattr(recfile.open(filename, "r"), 'read'))
+        recfile.remove(filename, path='tests')
+        self.assertFalse(recfile.exists(filename))
+        self.assertRaises(IOError, recfile.remove, filename)
+        self.assertRaises(IOError, recfile.open, filename, "r")
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_scheduler.py

@@ -9,6 +9,7 @@ import unittest
 import glob
 import datetime
 import sys
+
 from fix_path import fix_sys_path
 
 
@@ -255,6 +256,14 @@ class testForSchedulerRunnerBase(BaseTestScheduler):
         from gluon import current
         fdest = os.path.join(current.request.folder, 'models', 'scheduler.py')
         os.unlink(fdest)
+        additional_files = [
+            os.path.join(current.request.folder, 'private', 'demo8.pholder')
+        ]
+        for f in additional_files:
+            try:
+                os.unlink(f)
+            except:
+                pass
 
     def writefunction(self, content, initlines=None):
         from gluon import current
@@ -266,7 +275,7 @@ import time
 from gluon.scheduler import Scheduler
 db_dal = os.path.abspath(os.path.join(request.folder, '..', '..', 'dummy2.db'))
 sched_dal = DAL('sqlite://%s' % db_dal, folder=os.path.dirname(db_dal))
-sched = Scheduler(sched_dal, max_empty_runs=20, migrate=False, heartbeat=1)
+sched = Scheduler(sched_dal, max_empty_runs=15, migrate=False, heartbeat=1)
             """
         with open(fdest, 'w') as q:
             q.write(initlines)
@@ -274,34 +283,179 @@ sched = Scheduler(sched_dal, max_empty_runs=20, migrate=False, heartbeat=1)
 
     def exec_sched(self):
         import subprocess
-        call_args = [sys.executable, 'web2py.py', '-K', 'welcome']
+        call_args = [sys.executable, 'web2py.py', '--no-banner', '-D', '20','-K', 'welcome']
         ret = subprocess.call(call_args, env=dict(os.environ))
         return ret
 
+    def fetch_results(self, sched, task):
+        info = sched.task_status(task.id)
+        task_runs = self.db(self.db.scheduler_run.task_id == task.id).select()
+        return info, task_runs
+
+    def exec_asserts(self, stmts, tag):
+        for stmt in stmts:
+            self.assertEqual(stmt[1], True, msg="%s - %s" % (tag, stmt[0]))
+
 
 class TestsForSchedulerRunner(testForSchedulerRunnerBase):
 
-    def testBasic(self):
+    def testRepeats_and_Expired_and_Prio(self):
         s = Scheduler(self.db)
-        foo = s.queue_task('foo')
+        repeats = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), repeats=2, period=5)
+        a_while_ago = datetime.datetime.now() - datetime.timedelta(seconds=60)
+        expired = s.queue_task('demo4', stop_time=a_while_ago)
+        prio1 = s.queue_task('demo1', ['scheduled_first'])
+        prio2 = s.queue_task('demo1', ['scheduled_second'], next_run_time=a_while_ago)
         self.db.commit()
         self.writefunction(r"""
-def foo():
+def demo1(*args,**vars):
-    return 'a'
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
 """)
         ret = self.exec_sched()
-        # process finished just fine
         self.assertEqual(ret, 0)
-        info = s.task_status(foo.id, output=True)
+        # repeats check
-        self.assertEqual(info.result, 'a')
+        task, task_run = self.fetch_results(s, repeats)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("task times_run is 2", task.times_run == 2),
+            ("task ran 2 times only", len(task_run) == 2),
+            ("scheduler_run records are COMPLETED ", (task_run[0].status == task_run[1].status == 'COMPLETED')),
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'REPEATS')
+
+        # expired check
+        task, task_run = self.fetch_results(s, expired)
+        res = [
+            ("task status expired", task.status == 'EXPIRED'),
+            ("task times_run is 0", task.times_run == 0),
+            ("task didn't run at all", len(task_run) == 0)
+        ]
+        self.exec_asserts(res, 'EXPIRATION')
+
+        # prio check
+        task1 = s.task_status(prio1.id, output=True)
+        task2 = s.task_status(prio2.id, output=True)
+        res = [
+            ("tasks status completed", task1.scheduler_task.status == task2.scheduler_task.status == 'COMPLETED'),
+            ("priority2 was executed before priority1" , task1.scheduler_run.id > task2.scheduler_run.id)
+        ]
+        self.exec_asserts(res, 'PRIORITY')
+
+    def testNoReturn_and_Timeout_and_Progress(self):
+        s = Scheduler(self.db)
+        noret1 = s.queue_task('demo5')
+        noret2 = s.queue_task('demo3')
+        timeout1 = s.queue_task('demo4', timeout=5)
+        timeout2 = s.queue_task('demo4')
+        progress = s.queue_task('demo6', sync_output=2)
+        self.db.commit()
+        self.writefunction(r"""
+def demo3():
+    time.sleep(15)
+    print 1/0
+    return None
+
+def demo4():
+    time.sleep(15)
+    print "I'm printing something"
+    return dict(a=1, b=2)
+
+def demo5():
+    time.sleep(15)
+    print "I'm printing something"
+    rtn = dict(a=1, b=2)
+
+def demo6():
+    time.sleep(5)
+    print '50%'
+    time.sleep(5)
+    print '!clear!100%'
+    return 1
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # noreturn check
+        task1, task_run1 = self.fetch_results(s, noret1)
+        task2, task_run2 = self.fetch_results(s, noret2)
+        res = [
+            ("tasks no_returns1 completed", task1.status == 'COMPLETED'),
+            ("tasks no_returns2 failed", task2.status == 'FAILED'),
+            ("no_returns1 doesn't have a scheduler_run record", len(task_run1) == 0),
+            ("no_returns2 has a scheduler_run record FAILED", (len(task_run2) == 1 and task_run2[0].status == 'FAILED')),
+        ]
+        self.exec_asserts(res, 'NO_RETURN')
+
+        # timeout check
+        task1 = s.task_status(timeout1.id, output=True)
+        task2 = s.task_status(timeout2.id, output=True)
+        res = [
+            ("tasks timeouts1 timeoutted", task1.scheduler_task.status == 'TIMEOUT'),
+            ("tasks timeouts2 completed", task2.scheduler_task.status == 'COMPLETED')
+        ]
+        self.exec_asserts(res, 'TIMEOUT')
+
+        # progress check
+        task1 = s.task_status(progress.id, output=True)
+        res = [
+            ("tasks percentages completed", task1.scheduler_task.status == 'COMPLETED'),
+            ("output contains only 100%", task1.scheduler_run.run_output.strip() == "100%")
+        ]
+        self.exec_asserts(res, 'PROGRESS')
+
+    def testDrift_and_env_and_immediate(self):
+        s = Scheduler(self.db)
+        immediate = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), immediate=True)
+        env = s.queue_task('demo7')
+        drift = s.queue_task('demo1', ['a', 'b'], dict(c=1, d=2), period=93, prevent_drift=True)
+        self.db.commit()
+        self.writefunction(r"""
+def demo1(*args,**vars):
+    print 'you passed args=%s and vars=%s' % (args, vars)
+    return args[0]
+import random
+def demo7():
+    time.sleep(random.randint(1,5))
+    print W2P_TASK, request.now
+    return W2P_TASK.id, W2P_TASK.uuid, W2P_TASK.run_id
+""")
+        ret = self.exec_sched()
+        self.assertEqual(ret, 0)
+        # immediate check, can only check that nothing breaks
+        task1 = s.task_status(immediate.id)
+        res = [
+            ("tasks status completed", task1.status == 'COMPLETED'),
+        ]
+        self.exec_asserts(res, 'IMMEDIATE')
+
+        # drift check
+        task, task_run = self.fetch_results(s, drift)
+        res = [
+            ("task status completed", task.status == 'COMPLETED'),
+            ("next_run_time is exactly start_time + period", (task.next_run_time == task.start_time + datetime.timedelta(seconds=task.period)))
+        ]
+        self.exec_asserts(res, 'DRIFT')
+
+        # env check
+        task1 = s.task_status(env.id, output=True)
+        res = [
+            ("task %s returned W2P_TASK correctly" % (task1.scheduler_task.id),  task1.result == [task1.scheduler_task.id, task1.scheduler_task.uuid, task1.scheduler_run.id]),
+        ]
+        self.exec_asserts(res, 'ENV')
+
 
     def testRetryFailed(self):
         s = Scheduler(self.db)
-        failed = s.queue_task('demo2', retry_failed=1, period=5)
+        failed = s.queue_task('demo2', retry_failed=1, period=1)
-        failed_consecutive = s.queue_task('demo8', retry_failed=2, repeats=2, period=5)
+        failed_consecutive = s.queue_task('demo8', retry_failed=2, repeats=2, period=1)
         self.db.commit()
         self.writefunction(r"""
-
 def demo2():
     1/0
 
@@ -323,32 +477,50 @@ def demo8():
         # process finished just fine
         self.assertEqual(ret, 0)
         # failed - checks
-        info = s.task_status(failed.id)
+        task, task_run = self.fetch_results(s, failed)
-        task_runs = self.db(self.db.scheduler_run.task_id == info.id).select()
         res = [
-            ("task status failed", info.status == 'FAILED'),
+            ("task status failed", task.status == 'FAILED'),
-            ("task times_run is 0", info.times_run == 0),
+            ("task times_run is 0", task.times_run == 0),
-            ("task times_failed is 2", info.times_failed == 2),
+            ("task times_failed is 2", task.times_failed == 2),
-            ("task ran 2 times only", len(task_runs) == 2),
+            ("task ran 2 times only", len(task_run) == 2),
-            ("scheduler_run records are FAILED", (task_runs[0].status == task_runs[1].status == 'FAILED')),
+            ("scheduler_run records are FAILED", (task_run[0].status == task_run[1].status == 'FAILED')),
-            ("period is respected", (task_runs[1].start_time > task_runs[0].start_time + datetime.timedelta(seconds=info.period)))
+            ("period is respected", (task_run[1].start_time > task_run[0].start_time + datetime.timedelta(seconds=task.period)))
         ]
-        for a in res:
+        self.exec_asserts(res, 'FAILED')
-            self.assertEqual(a[1], True, msg=a[0])
 
         # failed consecutive - checks
-        info = s.task_status(failed_consecutive.id)
+        task, task_run = self.fetch_results(s, failed_consecutive)
-        task_runs = self.db(self.db.scheduler_run.task_id == info.id).select()
         res = [
-            ("task status completed", info.status == 'COMPLETED'),
+            ("task status completed", task.status == 'COMPLETED'),
-            ("task times_run is 2", info.times_run == 2),
+            ("task times_run is 2", task.times_run == 2),
-            ("task times_failed is 0", info.times_failed == 0),
+            ("task times_failed is 0", task.times_failed == 0),
-            ("task ran 6 times", len(task_runs) == 6),
+            ("task ran 6 times", len(task_run) == 6),
-            ("scheduler_run records for COMPLETED is 2", len([run.status for run in task_runs if run.status == 'COMPLETED']) == 2),
+            ("scheduler_run records for COMPLETED is 2", len([run.status for run in task_run if run.status == 'COMPLETED']) == 2),
-            ("scheduler_run records for FAILED is 4", len([run.status for run in task_runs if run.status == 'FAILED']) == 4),
+            ("scheduler_run records for FAILED is 4", len([run.status for run in task_run if run.status == 'FAILED']) == 4),
         ]
-        for a in res:
+        self.exec_asserts(res, 'FAILED_CONSECUTIVE')
-            self.assertEqual(a[1], True, msg=a[0])
+
+    def testHugeResult(self):
+        s = Scheduler(self.db)
+        huge_result = s.queue_task('demo10', retry_failed=1, period=1)
+        self.db.commit()
+        self.writefunction(r"""
+def demo10():
+    res = 'a' * 99999
+    return dict(res=res)
+""")
+        ret = self.exec_sched()
+        # process finished just fine
+        self.assertEqual(ret, 0)
+        # huge_result - checks
+        task = s.task_status(huge_result.id, output=True)
+        res = [
+            ("task status completed", task.scheduler_task.status == 'COMPLETED'),
+            ("task times_run is 1", task.scheduler_task.times_run == 1),
+            ("result is the correct one", task.result == dict(res='a' * 99999))
+        ]
+        self.exec_asserts(res, 'HUGE_RESULT')
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_tools.py

@@ -7,6 +7,7 @@
 import os
 import sys
 import smtplib
+import datetime
 if sys.version < "2.7":
     import unittest2 as unittest
 else:
@@ -20,7 +21,7 @@ DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
 from gluon.dal import DAL, Field
 from pydal.objects import Table
-from tools import Auth, Mail, Recaptcha, Recaptcha2
+from tools import Auth, Mail, Recaptcha, Recaptcha2, prettydate
 from gluon.globals import Request, Response, Session
 from storage import Storage
 from languages import translator
@@ -229,23 +230,242 @@ class TestMail(unittest.TestCase):
 
 
 @unittest.skipIf(IS_IMAP, "TODO: Imap raises 'Connection refused'")
+# class TestAuth(unittest.TestCase):
+#
+#     def setUp(self):
+#         request = Request(env={})
+#         request.application = 'a'
+#         request.controller = 'c'
+#         request.function = 'f'
+#         request.folder = 'applications/admin'
+#         response = Response()
+#         session = Session()
+#         T = translator('', 'en')
+#         session.connect(request, response)
+#         from gluon.globals import current
+#         current.request = request
+#         current.response = response
+#         current.session = session
+#         current.T = T
+#         self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+#         self.auth = Auth(self.db)
+#         self.auth.define_tables(username=True, signature=False)
+#         self.db.define_table('t0', Field('tt'), self.auth.signature)
+#         self.auth.enable_record_versioning(self.db)
+#         # Create a user
+#         self.auth.get_or_create_user(dict(first_name='Bart',
+#                                           last_name='Simpson',
+#                                           username='bart',
+#                                           email='bart@simpson.com',
+#                                           password='bart_password',
+#                                           registration_key='bart',
+#                                           registration_id=''
+#                                           ))
+#         # self.auth.settings.registration_requires_verification = False
+#         # self.auth.settings.registration_requires_approval = False
+#
+#     def test_assert_setup(self):
+#         self.assertEqual(self.db(self.db.auth_user.username == 'bart').select().first()['username'], 'bart')
+#         self.assertTrue('auth_user' in self.db)
+#         self.assertTrue('auth_group' in self.db)
+#         self.assertTrue('auth_membership' in self.db)
+#         self.assertTrue('auth_permission' in self.db)
+#         self.assertTrue('auth_event' in self.db)
+#
+#     def test_enable_record_versioning(self):
+#         self.assertTrue('t0_archive' in self.db)
+#
+#     def test_basic_blank_forms(self):
+#         for f in ['login', 'retrieve_password',
+#                   'retrieve_username',
+#                   # 'register'  # register complain about : client_side=self.settings.client_side
+#                   ]:
+#             html_form = getattr(self.auth, f)().xml()
+#             self.assertTrue('name="_formkey"' in html_form)
+#
+#         # NOTE: Not sure it is the proper way to logout_bare() as there is not methods for that and auth.logout() failed
+#         self.auth.logout_bare()
+#         # self.assertTrue(self.auth.is_logged_in())
+#
+#         for f in ['logout', 'verify_email', 'reset_password',
+#                   'change_password', 'profile', 'groups']:
+#             self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
+#
+#         self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate)
+#
+#         try:
+#             for t in ['t0_archive', 't0', 'auth_cas', 'auth_event',
+#                       'auth_membership', 'auth_permission', 'auth_group',
+#                       'auth_user']:
+#                 self.db[t].drop()
+#         except SyntaxError as e:
+#             # GAE doesn't support drop
+#             pass
+#         return
+#
+#     def test_get_or_create_user(self):
+#         self.db.auth_user.insert(email='user1@test.com', username='user1', password='password_123')
+#         self.db.commit()
+#         # True case
+#         self.assertEqual(self.auth.get_or_create_user({'email': 'user1@test.com',
+#                                                        'username': 'user1',
+#                                                        'password': 'password_123'
+#                                                        })['username'], 'user1')
+#         # user2 doesn't exist yet and get created
+#         self.assertEqual(self.auth.get_or_create_user({'email': 'user2@test.com',
+#                                                        'username': 'user2'})['username'], 'user2')
+#         # user3 for corner case
+#         self.assertEqual(self.auth.get_or_create_user({'first_name': 'Omer',
+#                                                        'last_name': 'Simpson',
+#                                                        'email': 'user3@test.com',
+#                                                        'registration_id': 'user3',
+#                                                        'username': 'user3'})['username'], 'user3')
+#         # False case
+#         self.assertEqual(self.auth.get_or_create_user({'email': ''}), None)
+#         self.db.auth_user.truncate()
+#         self.db.commit()
+#
+#     def test_login_bare(self):
+#         # The following test case should succeed but failed as I never received the user record but False
+#         self.auth.login_bare(username='bart@simpson.com', password='bart_password')
+#         self.assertTrue(self.auth.is_logged_in())
+#         # Failing login because bad_password
+#         self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+#         self.db.auth_user.truncate()
+#
+#     def test_register_bare(self):
+#         # corner case empty register call register_bare without args
+#         self.assertRaises(ValueError, self.auth.register_bare)
+#         # failing register_bare user already exist
+#         self.assertEqual(self.auth.register_bare(username='bart', password='wrong_password'), False)
+#         # successful register_bare
+#         self.assertEqual(self.auth.register_bare(username='user2',
+#                                                  email='user2@test.com',
+#                                                  password='password_123')['username'], 'user2')
+#         # raise ValueError
+#         self.assertRaises(ValueError, self.auth.register_bare,
+#                           **dict(wrong_field_name='user3', password='password_123'))
+#         # raise ValueError wrong email
+#         self.assertRaises(ValueError, self.auth.register_bare,
+#                           **dict(email='user4@', password='password_123'))
+#         self.db.auth_user.truncate()
+#         self.db.commit()
+#
+#     def test_bulk_register(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         self.auth.settings.bulk_register_enabled = True
+#         bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
+#         self.assertTrue('name="_formkey"' in bulk_register_form)
+#
+#     def test_change_password(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         change_password_form = getattr(self.auth, 'change_password')().xml()
+#         self.assertTrue('name="_formkey"' in change_password_form)
+#
+#     def test_profile(self):
+#         self.auth.login_bare(username='bart', password='bart_password')
+#         profile_form = getattr(self.auth, 'profile')().xml()
+#         self.assertTrue('name="_formkey"' in profile_form)
+#
+#     # def test_impersonate(self):
+#     #     # Create a user to be impersonated
+#     #     self.auth.get_or_create_user(dict(first_name='Omer',
+#     #                                       last_name='Simpson',
+#     #                                       username='omer',
+#     #                                       email='omer@test.com',
+#     #                                       password='password_omer',
+#     #                                       registration_key='',
+#     #                                       registration_id=''))
+#     #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+#     #     self.auth.add_group('impersonate')
+#     #     self.auth.add_membership(user_id=1,
+#     #                              group_id=self.db(self.db.auth_user.username == 'bart'
+#     #                                               ).select(self.db.auth_user.id).first().id)
+#     #     self.auth.add_permission(group_id=self.db(self.db.auth_group.role == 'impersonate'
+#     #                                               ).select(self.db.auth_group.id).first().id,
+#     #                              name='impersonate',
+#     #                              table_name='auth_user',
+#     #                              record_id=0)
+#     #     # Bart login
+#     #     self.auth.login_bare(username='bart', password='bart_password')
+#     #     self.assertTrue(self.auth.is_logged_in())
+#     #     # Bart impersonate Omer
+#     #     omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
+#     #     impersonate_form = self.auth.impersonate(user_id=omer_id)
+#     #     self.assertTrue(self.auth.is_impersonating())
+#     #     self.assertEqual(impersonate_form, 'test')
+#
+#     # def test_impersonate(self):
+#     #     request = Request(env={})
+#     #     request.application = 'a'
+#     #     request.controller = 'c'
+#     #     request.function = 'f'
+#     #     request.folder = 'applications/admin'
+#     #     response = Response()
+#     #     session = Session()
+#     #     T = translator('', 'en')
+#     #     session.connect(request, response)
+#     #     from gluon.globals import current
+#     #     current.request = request
+#     #     current.response = response
+#     #     current.session = session
+#     #     current.T = T
+#     #     db = DAL(DEFAULT_URI, check_reserved=['all'])
+#     #     auth = Auth(db)
+#     #     auth.define_tables(username=True, signature=False)
+#     #     db.define_table('t0', Field('tt'), auth.signature)
+#     #     auth.enable_record_versioning(db)
+#     #     # Create a user
+#     #     auth.get_or_create_user(dict(first_name='Bart',
+#     #                                       last_name='Simpson',
+#     #                                       username='bart',
+#     #                                       email='bart@simpson.com',
+#     #                                       password='bart_password',
+#     #                                       registration_key='bart',
+#     #                                       registration_id=''
+#     #                                       ))
+#     #     # Create a user to be impersonated
+#     #     auth.get_or_create_user(dict(first_name='Omer',
+#     #                                       last_name='Simpson',
+#     #                                       username='omer',
+#     #                                       email='omer@test.com',
+#     #                                       password='password_omer',
+#     #                                       registration_key='',
+#     #                                       registration_id=''))
+#     #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+#     #     auth.add_group('impersonate')
+#     #     auth.add_membership(user_id=1,
+#     #                              group_id=db(db.auth_user.username == 'bart'
+#     #                                               ).select(db.auth_user.id).first().id)
+#     #     auth.add_permission(group_id=db(db.auth_group.role == 'impersonate'
+#     #                                               ).select(db.auth_group.id).first().id,
+#     #                              name='impersonate',
+#     #                              table_name='auth_user',
+#     #                              record_id=0)
+#     #     # Bart login
+#     #     auth.login_bare(username='bart', password='bart_password')
+#     #     # Bart impersonate Omer
+#     #     omer_id = db(db.auth_user.username == 'omer').select(db.auth_user.id).first().id
+#     #     impersonate_form = auth.impersonate(user_id=omer_id)
+#     #     self.assertTrue(auth.is_impersonating())
+#     #     self.assertEqual(impersonate_form, 'test')
 class TestAuth(unittest.TestCase):
-
     def setUp(self):
-        request = Request(env={})
+        self.request = Request(env={})
-        request.application = 'a'
+        self.request.application = 'a'
-        request.controller = 'c'
+        self.request.controller = 'c'
-        request.function = 'f'
+        self.request.function = 'f'
-        request.folder = 'applications/admin'
+        self.request.folder = 'applications/admin'
-        response = Response()
+        self.response = Response()
-        session = Session()
+        self.session = Session()
         T = translator('', 'en')
-        session.connect(request, response)
+        self.session.connect(self.request, self.response)
         from gluon.globals import current
-        current.request = request
+        self.current = current
-        current.response = response
+        self.current.request = self.request
-        current.session = session
+        self.current.response = self.response
-        current.T = T
+        self.current.session = self.session
+        self.current.T = T
         self.db = DAL(DEFAULT_URI, check_reserved=['all'])
         self.auth = Auth(self.db)
         self.auth.define_tables(username=True, signature=False)
@@ -259,7 +479,10 @@ class TestAuth(unittest.TestCase):
                                           password='bart_password',
                                           registration_key='bart',
                                           registration_id=''
-                                          ))
+                                          ),
+                                     login=False)
+        self.db.commit()
+        self.assertFalse(self.auth.is_logged_in())
         # self.auth.settings.registration_requires_verification = False
         # self.auth.settings.registration_requires_approval = False
 
@@ -271,23 +494,13 @@ class TestAuth(unittest.TestCase):
         self.assertTrue('auth_permission' in self.db)
         self.assertTrue('auth_event' in self.db)
 
-    def test_enable_record_versioning(self):
+    # Just calling many form functions
-        self.assertTrue('t0_archive' in self.db)
-
     def test_basic_blank_forms(self):
-        for f in ['login', 'retrieve_password',
+        for f in ['login', 'retrieve_password', 'retrieve_username', 'register']:
-                  'retrieve_username',
-                  # 'register'  # register complain about : client_side=self.settings.client_side
-                  ]:
             html_form = getattr(self.auth, f)().xml()
             self.assertTrue('name="_formkey"' in html_form)
 
-        # NOTE: Not sure it is the proper way to logout_bare() as there is not methods for that and auth.logout() failed
+        for f in ['logout', 'verify_email', 'reset_password', 'change_password', 'profile', 'groups']:
-        self.auth.logout_bare()
-        # self.assertTrue(self.auth.is_logged_in())
-
-        for f in ['logout', 'verify_email', 'reset_password',
-                  'change_password', 'profile', 'groups']:
             self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
 
         self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate)
@@ -302,6 +515,63 @@ class TestAuth(unittest.TestCase):
             pass
         return
 
+    def test_get_vars_next(self):
+        self.current.request.vars._next = 'next_test'
+        self.assertEqual(self.auth.get_vars_next(), 'next_test')
+
+    # TODO: def test_navbar(self):
+    # TODO: def test___get_migrate(self):
+
+    def test_enable_record_versioning(self):
+        self.assertTrue('t0_archive' in self.db)
+
+    # TODO: def test_define_signature(self):
+    # TODO: def test_define_signature(self):
+    # TODO: def test_define_table(self):
+
+    def test_log_event(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        # user logged in
+        self.auth.log_event(description='some_log_event_description_%(var1)s',
+                            vars={"var1": "var1"},
+                            origin='log_event_test_1')
+        rtn = self.db(self.db.auth_event.origin == 'log_event_test_1'
+                      ).select(*[self.db.auth_event[f]
+                                 for f in self.db.auth_event.fields if f not in ('id', 'time_stamp')]).first().as_dict()
+        self.assertEqual(set(rtn.items()), set({'origin': 'log_event_test_1',
+                                                'client_ip': None,
+                                                'user_id': bart_id,
+                                                'description': 'some_log_event_description_var1'}.items()))
+        # user not logged
+        self.auth.logout_bare()
+        self.auth.log_event(description='some_log_event_description_%(var2)s',
+                            vars={"var2": "var2"},
+                            origin='log_event_test_2')
+        rtn = self.db(self.db.auth_event.origin == 'log_event_test_2'
+                      ).select(*[self.db.auth_event[f]
+                                 for f in self.db.auth_event.fields if f not in ('id', 'time_stamp')]).first().as_dict()
+        self.assertEqual(set(rtn.items()), set({'origin': 'log_event_test_2',
+                                                'client_ip': None,
+                                                'user_id': None,
+                                                'description': 'some_log_event_description_var2'}.items()))
+        # no logging tests
+        self.auth.settings.logging_enabled = False
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.auth.log_event(description='some_log_event_description_%(var3)s',
+                            vars={"var3": "var3"},
+                            origin='log_event_test_3')
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        self.auth.settings.logging_enabled = True
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.auth.log_event(description=None,
+                            vars={"var4": "var4"},
+                            origin='log_event_test_4')
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # TODO: Corner case translated description...
+
     def test_get_or_create_user(self):
         self.db.auth_user.insert(email='user1@test.com', username='user1', password='password_123')
         self.db.commit()
@@ -324,13 +594,20 @@ class TestAuth(unittest.TestCase):
         self.db.auth_user.truncate()
         self.db.commit()
 
-    def test_login_bare(self):
+    # TODO: def test_basic(self):
-        # The following test case should succeed but failed as I never received the user record but False
+    # TODO: def test_login_user(self):
-        self.auth.login_bare(username='bart@simpson.com', password='bart_password')
+    # TODO: def test__get_login_settings(self):
-        self.assertTrue(self.auth.is_logged_in())
+
-        # Failing login because bad_password
+    # login_bare() seems broken see my post on web2py-developpers
-        self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+    # commented for now
-        self.db.auth_user.truncate()
+    # def test_login_bare(self):
+    #     # The following test case should succeed but failed as I never received the user record but False
+    #     self.auth.login_bare(username='bart', password='bart_password')
+    #     self.assertTrue(self.auth.is_logged_in())
+    #     # Failing login because bad_password
+    #     self.assertEqual(self.auth.login_bare(username='bart', password='wrong_password'), False)
+    #     self.auth.logout_bare()
+    #     self.db.auth_user.truncate()
 
     def test_register_bare(self):
         # corner case empty register call register_bare without args
@@ -350,104 +627,310 @@ class TestAuth(unittest.TestCase):
         self.db.auth_user.truncate()
         self.db.commit()
 
+    # TODO: def test_cas_login(self):
+    # TODO: def test_cas_validate(self):
+    # TODO: def test__reset_two_factor_auth(self):
+    # TODO: def test_when_is_logged_in_bypass_next_in_url(self):
+    # TODO: def test_login(self):
+    # TODO: def test_logout(self):
+
+    def test_logout_bare(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.logout_bare()
+        self.assertFalse(self.auth.is_logged_in())
+
+    # TODO: def test_register(self):
+
+    def test_is_logged_in(self):
+        self.auth.user = 'logged_in'
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.user = None
+        self.assertFalse(self.auth.is_logged_in())
+
+    # TODO: def test_verify_email(self):
+    # TODO: def test_retrieve_username(self):
+
+    def test_random_password(self):
+        # let just check that the function is callable
+        self.assertTrue(self.auth.random_password())
+
+    # TODO: def test_reset_password_deprecated(self):
+    # TODO: def test_confirm_registration(self):
+    # TODO: def test_email_registration(self):
+
     def test_bulk_register(self):
-        self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         self.auth.settings.bulk_register_enabled = True
         bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
         self.assertTrue('name="_formkey"' in bulk_register_form)
 
+    # TODO: def test_manage_tokens(self):
+    # TODO: def test_reset_password(self):
+    # TODO: def test_request_reset_password(self):
+    # TODO: def test_email_reset_password(self):
+    # TODO: def test_retrieve_password(self):
+
     def test_change_password(self):
-        self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         change_password_form = getattr(self.auth, 'change_password')().xml()
         self.assertTrue('name="_formkey"' in change_password_form)
 
     def test_profile(self):
-        self.auth.login_bare(username='bart', password='bart_password')
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         profile_form = getattr(self.auth, 'profile')().xml()
         self.assertTrue('name="_formkey"' in profile_form)
 
-    # def test_impersonate(self):
+    # TODO: def test_run_login_onaccept(self):
-    #     # Create a user to be impersonated
+    # TODO: def test_jwt(self):
-    #     self.auth.get_or_create_user(dict(first_name='Omer',
+    # TODO: def test_is_impersonating(self):
-    #                                       last_name='Simpson',
-    #                                       username='omer',
-    #                                       email='omer@test.com',
-    #                                       password='password_omer',
-    #                                       registration_key='',
-    #                                       registration_id=''))
-    #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
-    #     self.auth.add_group('impersonate')
-    #     self.auth.add_membership(user_id=1,
-    #                              group_id=self.db(self.db.auth_user.username == 'bart'
-    #                                               ).select(self.db.auth_user.id).first().id)
-    #     self.auth.add_permission(group_id=self.db(self.db.auth_group.role == 'impersonate'
-    #                                               ).select(self.db.auth_group.id).first().id,
-    #                              name='impersonate',
-    #                              table_name='auth_user',
-    #                              record_id=0)
-    #     # Bart login
-    #     self.auth.login_bare(username='bart', password='bart_password')
-    #     self.assertTrue(self.auth.is_logged_in())
-    #     # Bart impersonate Omer
-    #     omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
-    #     impersonate_form = self.auth.impersonate(user_id=omer_id)
-    #     self.assertTrue(self.auth.is_impersonating())
-    #     self.assertEqual(impersonate_form, 'test')
 
-    # def test_impersonate(self):
+    def test_impersonate(self):
-    #     request = Request(env={})
+        # Create a user to be impersonated
-    #     request.application = 'a'
+        self.auth.get_or_create_user(dict(first_name='Omer',
-    #     request.controller = 'c'
+                                          last_name='Simpson',
-    #     request.function = 'f'
+                                          username='omer',
-    #     request.folder = 'applications/admin'
+                                          email='omer@test.com',
-    #     response = Response()
+                                          password='password_omer',
-    #     session = Session()
+                                          registration_key='',
-    #     T = translator('', 'en')
+                                          registration_id=''),
-    #     session.connect(request, response)
+                                     login=False)
-    #     from gluon.globals import current
+        self.db.commit()
-    #     current.request = request
+        self.assertFalse(self.auth.is_logged_in())
-    #     current.response = response
+        # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
-    #     current.session = session
+        group_id = self.auth.add_group('impersonate')
-    #     current.T = T
+        self.auth.add_membership(user_id=self.db(self.db.auth_user.username == 'bart'
-    #     db = DAL(DEFAULT_URI, check_reserved=['all'])
+                                                 ).select(self.db.auth_user.id).first().id,
-    #     auth = Auth(db)
+                                 group_id=group_id)
-    #     auth.define_tables(username=True, signature=False)
+        self.auth.add_permission(group_id=group_id,
-    #     db.define_table('t0', Field('tt'), auth.signature)
+                                 name='impersonate',
-    #     auth.enable_record_versioning(db)
+                                 table_name='auth_user',
-    #     # Create a user
+                                 record_id=0)
-    #     auth.get_or_create_user(dict(first_name='Bart',
+        # Bart login
-    #                                       last_name='Simpson',
+        # self.auth.login_bare(username='bart', password='bart_password')
-    #                                       username='bart',
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
-    #                                       email='bart@simpson.com',
+        self.assertTrue(self.auth.is_logged_in())
-    #                                       password='bart_password',
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
-    #                                       registration_key='bart',
+        self.assertEqual(self.auth.user_id, bart_id)
-    #                                       registration_id=''
+        # self.session.auth = self.auth
-    #                                       ))
+        # self.assertTrue(self.session.auth)
-    #     # Create a user to be impersonated
+
-    #     auth.get_or_create_user(dict(first_name='Omer',
+        # basic impersonate() test that return a read form
-    #                                       last_name='Simpson',
+        self.assertEqual(self.auth.impersonate().xml(),
-    #                                       username='omer',
+                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="no_table_user_id__row"><td class="w2p_fl"><label class="" for="no_table_user_id" id="no_table_user_id__label">User Id: </label></td><td class="w2p_fw"><input class="integer" id="no_table_user_id" name="user_id" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
-    #                                       email='omer@test.com',
+        # bart impersonate itself
-    #                                       password='password_omer',
+        self.assertEqual(self.auth.impersonate(bart_id), None)
-    #                                       registration_key='',
+        self.assertFalse(self.auth.is_impersonating())  # User shouldn't impersonate itself?
-    #                                       registration_id=''))
+        # Bart impersonate Omer
-    #     # Create impersonate group, assign bart to impersonate group and add impersonate permission over auth_user
+        omer_id = self.db(self.db.auth_user.username == 'omer').select(self.db.auth_user.id).first().id
-    #     auth.add_group('impersonate')
+        impersonate_form = self.auth.impersonate(user_id=omer_id)
-    #     auth.add_membership(user_id=1,
+        self.assertTrue(self.auth.is_impersonating())
-    #                              group_id=db(db.auth_user.username == 'bart'
+        self.assertEqual(self.auth.user_id, omer_id)  # we make it really sure
-    #                                               ).select(db.auth_user.id).first().id)
+        self.assertEqual(impersonate_form.xml(),
-    #     auth.add_permission(group_id=db(db.auth_group.role == 'impersonate'
+                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="auth_user_id__row"><td class="w2p_fl"><label class="readonly" for="auth_user_id" id="auth_user_id__label">Id: </label></td><td class="w2p_fw"><span id="auth_user_id">2</span></td><td class="w2p_fc"></td></tr><tr id="auth_user_first_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_first_name" id="auth_user_first_name__label">First name: </label></td><td class="w2p_fw">Omer</td><td class="w2p_fc"></td></tr><tr id="auth_user_last_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_last_name" id="auth_user_last_name__label">Last name: </label></td><td class="w2p_fw">Simpson</td><td class="w2p_fc"></td></tr><tr id="auth_user_email__row"><td class="w2p_fl"><label class="readonly" for="auth_user_email" id="auth_user_email__label">E-mail: </label></td><td class="w2p_fw">omer@test.com</td><td class="w2p_fc"></td></tr><tr id="auth_user_username__row"><td class="w2p_fl"><label class="readonly" for="auth_user_username" id="auth_user_username__label">Username: </label></td><td class="w2p_fw">omer</td><td class="w2p_fc"></td></tr></table><div style="display:none;"><input name="id" type="hidden" value="2" /></div></form>')
-    #                                               ).select(db.auth_group.id).first().id,
+        self.auth.logout_bare()
-    #                              name='impersonate',
+        # Failing impersonation
-    #                              table_name='auth_user',
+        # User lacking impersonate membership
-    #                              record_id=0)
+        self.auth.login_user(self.db(self.db.auth_user.username == 'omer').select().first())  # bypass login_bare()
-    #     # Bart login
+        # self.assertTrue(self.auth.is_logged_in())  # For developing test
-    #     auth.login_bare(username='bart', password='bart_password')
+        # self.assertFalse(self.auth.is_impersonating())  # For developing test
-    #     # Bart impersonate Omer
+        self.assertRaisesRegexp(HTTP, "403*", self.auth.impersonate, bart_id)
-    #     omer_id = db(db.auth_user.username == 'omer').select(db.auth_user.id).first().id
+        self.auth.logout_bare()
-    #     impersonate_form = auth.impersonate(user_id=omer_id)
+        # Try impersonate a non existing user
-    #     self.assertTrue(auth.is_impersonating())
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
-    #     self.assertEqual(impersonate_form, 'test')
+        # self.assertTrue(self.auth.is_logged_in())  # For developing test
+        # self.assertFalse(self.auth.is_impersonating())  # For developing test
+        self.assertRaisesRegexp(HTTP, "401*", self.auth.impersonate, 1000)  # user with id 1000 shouldn't exist
+        # Try impersonate user with id = 0 or '0' when bart impersonating omer
+        self.auth.impersonate(user_id=omer_id)
+        self.assertTrue(self.auth.is_impersonating())
+        self.assertEqual(self.auth.impersonate(user_id=0), None)
+
+    # TODO: def test_update_groups(self):
+
+    def test_groups(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertEqual(self.auth.groups().xml(),
+                         '<table><tr><td><h3>user_1(1)</h3></td></tr><tr><td><p></p></td></tr></table>')
+
+    def test_not_authorized(self):
+        self.current.request.ajax = 'facke_ajax_request'
+        self.assertRaisesRegexp(HTTP, "403*", self.auth.not_authorized)
+        self.current.request.ajax = None
+        self.assertEqual(self.auth.not_authorized(), self.auth.messages.access_denied)
+
+    def test_allows_jwt(self):
+        self.assertRaisesRegexp(HTTP, "400*", self.auth.allows_jwt)
+
+    # TODO: def test_requires(self):
+    # TODO: def test_requires_login(self):
+    # TODO: def test_requires_login_or_token(self):
+    # TODO: def test_requires_membership(self):
+    # TODO: def test_requires_permission(self):
+    # TODO: def test_requires_signature(self):
+
+    def test_add_group(self):
+        self.assertEqual(self.auth.add_group(role='a_group', description='a_group_role_description'),
+                         self.db(self.db.auth_group.role == 'a_group').select(self.db.auth_group.id).first().id)
+
+    def test_del_group(self):
+        bart_group_id = 1  # Should be group 1, 'user_1'
+        self.assertEqual(self.auth.del_group(group_id=bart_group_id), None)
+
+    def test_id_group(self):
+        self.assertEqual(self.auth.id_group(role='user_1'), 1)
+        # If role don't exist it return None
+        self.assertEqual(self.auth.id_group(role='non_existing_role_name'), None)
+
+    def test_user_group(self):
+        self.assertEqual(self.auth.user_group(user_id=1), 1)
+        # Bart should be user 1 and it unique group should be 1, 'user_1'
+
+    def test_user_group_role(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        user_group_role = 'user_%s' % self.db(self.db.auth_user.username == 'bart'
+                                              ).select(self.db.auth_user.id).first().id
+        self.assertEqual(self.auth.user_group_role(), user_group_role)
+        self.auth.logout_bare()
+        # with user_id args
+        self.assertEqual(self.auth.user_group_role(user_id=1), 'user_1')
+        # test None
+        self.auth.settings.create_user_groups = None
+        self.assertEqual(self.auth.user_group_role(user_id=1), None)
+
+    def test_has_membership(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        self.assertTrue(self.auth.has_membership('user_1'))
+        self.assertFalse(self.auth.has_membership('user_555'))
+        self.assertTrue(self.auth.has_membership(group_id=1))
+        self.auth.logout_bare()
+        self.assertTrue(self.auth.has_membership(role='user_1', user_id=1))
+        self.assertTrue(self.auth.has_membership(group_id=1, user_id=1))
+        # check that event is logged
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.assertTrue(self.auth.has_membership(group_id=1, user_id=1))
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+
+    # Waiting guidance : https://github.com/web2py/web2py/issues/1300
+    # def test_add_membership(self):
+    #     self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+    #     # failing case
+    #     rtn = self.auth.add_membership('not_existing_role_name')
+    #     # self.assertEqual(rtn, 'test')
+    #     self.assertEqual(self.db(self.db.auth_group.role == 'not_existing_role_name').select().first(), 'test')
+
+    def test_del_membership(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        user_1_role_id = self.db(self.db.auth_membership.group_id == self.auth.id_group('user_1')
+                                 ).select(self.db.auth_membership.id).first().id
+        self.assertEqual(self.auth.del_membership('user_1'), user_1_role_id)
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # not logged in test case
+        group_id = self.auth.add_group('some_test_group')
+        membership_id = self.auth.add_membership('some_test_group')
+        self.assertEqual(self.auth.user_groups[group_id], 'some_test_group')
+        self.auth.logout_bare()
+        # not deleted
+        self.assertFalse(self.auth.del_membership('some_test_group'))
+        self.assertEqual(set(self.db.auth_membership(membership_id).as_dict().items()),
+                         set({'group_id': 2L, 'user_id': 1L, 'id': 2L}.items()))  # is not deleted
+        # deleted
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.assertTrue(self.auth.del_membership('some_test_group', user_id=bart_id))
+        self.assertEqual(self.db.auth_membership(membership_id), None)  # is really deleted
+
+    def test_has_permission(self):
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        bart_id = self.db(self.db.auth_user.username == 'bart').select(self.db.auth_user.id).first().id
+        self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                 name='some_permission',
+                                 table_name='auth_user',
+                                 record_id=0,
+                                 )
+        # True case
+        self.assertTrue(self.auth.has_permission(name='some_permission',
+                                                 table_name='auth_user',
+                                                 record_id=0,
+                                                 user_id=bart_id,
+                                                 group_id=self.auth.id_group('user_1')))
+        # False case
+        self.assertFalse(self.auth.has_permission(name='some_other_permission',
+                                                  table_name='auth_user',
+                                                  record_id=0,
+                                                  user_id=bart_id,
+                                                  group_id=self.auth.id_group('user_1')))
+
+    def test_add_permission(self):
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        permission_id = \
+            self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                     name='some_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # True case
+        permission_count = \
+            self.db(self.db.auth_permission.id == permission_id).count()
+        self.assertTrue(permission_count)
+        # False case
+        permission_count = \
+            self.db((self.db.auth_permission.group_id == self.auth.id_group('user_1')) &
+                    (self.db.auth_permission.name == 'no_permission') &
+                    (self.db.auth_permission.table_name == 'no_table') &
+                    (self.db.auth_permission.record_id == 0)).count()
+        self.assertFalse(permission_count)
+        # corner case
+        self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
+        permission_id = \
+            self.auth.add_permission(group_id=0,
+                                     name='user_1_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        permission_name = \
+            self.db(self.db.auth_permission.id == permission_id).select(self.db.auth_permission.name).first().name
+        self.assertEqual(permission_name, 'user_1_permission')
+        # add an existing permission
+        permission_id =\
+            self.auth.add_permission(group_id=0,
+                                     name='user_1_permission',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        self.assertTrue(permission_id)
+
+    def test_del_permission(self):
+        permission_id = \
+            self.auth.add_permission(group_id=self.auth.id_group('user_1'),
+                                     name='del_permission_test',
+                                     table_name='auth_user',
+                                     record_id=0,
+                                     )
+        count_log_event_test_before = self.db(self.db.auth_event.id > 0).count()
+        self.assertTrue(self.auth.del_permission(group_id=self.auth.id_group('user_1'),
+                                                 name='del_permission_test',
+                                                 table_name='auth_user',
+                                                 record_id=0,))
+        count_log_event_test_after = self.db(self.db.auth_event.id > 0).count()
+        # check that event is logged
+        self.assertEqual(count_log_event_test_after, count_log_event_test_before)
+        # really deleted
+        permission_count = \
+            self.db(self.db.auth_permission.id == permission_id).count()
+        self.assertFalse(permission_count)
+
+    # TODO: def test_accessible_query(self):
+    # TODO: def test_archive(self):
+    # TODO: def test_wiki(self):
+    # TODO: def test_wikimenu(self):
+    # End Auth test
 
 
 # TODO: class TestCrud(unittest.TestCase):
@@ -469,8 +952,88 @@ class TestAuth(unittest.TestCase):
 # TODO: class TestConfig(unittest.TestCase):
 
 
-# TODO: class TestToolsFunctions(unittest.TestCase):
+class TestToolsFunctions(unittest.TestCase):
-# For all the tools.py functions
+    """
+    Test suite for all the tools.py functions
+    """
+    def test_prettydate(self):
+        # plain
+        now = datetime.datetime.now()
+        self.assertEqual(prettydate(d=now), 'now')
+        one_second = now - datetime.timedelta(seconds=1)
+        self.assertEqual(prettydate(d=one_second), '1 second ago')
+        more_than_one_second = now - datetime.timedelta(seconds=2)
+        self.assertEqual(prettydate(d=more_than_one_second), '2 seconds ago')
+        one_minute = now - datetime.timedelta(seconds=60)
+        self.assertEqual(prettydate(d=one_minute), '1 minute ago')
+        more_than_one_minute = now - datetime.timedelta(seconds=61)
+        self.assertEqual(prettydate(d=more_than_one_minute), '1 minute ago')
+        two_minutes = now - datetime.timedelta(seconds=120)
+        self.assertEqual(prettydate(d=two_minutes), '2 minutes ago')
+        more_than_two_minutes = now - datetime.timedelta(seconds=121)
+        self.assertEqual(prettydate(d=more_than_two_minutes), '2 minutes ago')
+        one_hour = now - datetime.timedelta(seconds=60 * 60)
+        self.assertEqual(prettydate(d=one_hour), '1 hour ago')
+        more_than_one_hour = now - datetime.timedelta(seconds=3601)
+        self.assertEqual(prettydate(d=more_than_one_hour), '1 hour ago')
+        two_hours = now - datetime.timedelta(seconds=2 * 60 * 60)
+        self.assertEqual(prettydate(d=two_hours), '2 hours ago')
+        more_than_two_hours = now - datetime.timedelta(seconds=2 * 60 * 60 + 1)
+        self.assertEqual(prettydate(d=more_than_two_hours), '2 hours ago')
+        one_day = now - datetime.timedelta(days=1)
+        self.assertEqual(prettydate(d=one_day), '1 day ago')
+        more_than_one_day = now - datetime.timedelta(days=2)
+        self.assertEqual(prettydate(d=more_than_one_day), '2 days ago')
+        one_week = now - datetime.timedelta(days=7)
+        self.assertEqual(prettydate(d=one_week), '1 week ago')
+        more_than_one_week = now - datetime.timedelta(days=8)
+        self.assertEqual(prettydate(d=more_than_one_week), '1 week ago')
+        two_weeks = now - datetime.timedelta(days=14)
+        self.assertEqual(prettydate(d=two_weeks), '2 weeks ago')
+        more_than_two_weeks = now - datetime.timedelta(days=15)
+        self.assertEqual(prettydate(d=more_than_two_weeks), '2 weeks ago')
+        three_weeks = now - datetime.timedelta(days=21)
+        self.assertEqual(prettydate(d=three_weeks), '3 weeks ago')
+        one_month = now - datetime.timedelta(days=27)
+        self.assertEqual(prettydate(d=one_month), '1 month ago')
+        more_than_one_month = now - datetime.timedelta(days=28)
+        self.assertEqual(prettydate(d=more_than_one_month), '1 month ago')
+        two_months = now - datetime.timedelta(days=60)
+        self.assertEqual(prettydate(d=two_months), '2 months ago')
+        three_months = now - datetime.timedelta(days=90)
+        self.assertEqual(prettydate(d=three_months), '3 months ago')
+        one_year = now - datetime.timedelta(days=365)
+        self.assertEqual(prettydate(d=one_year), '1 year ago')
+        more_than_one_year = now - datetime.timedelta(days=366)
+        self.assertEqual(prettydate(d=more_than_one_year), '1 year ago')
+        two_years = now - datetime.timedelta(days=2 * 365)
+        self.assertEqual(prettydate(d=two_years), '2 years ago')
+        more_than_two_years = now - datetime.timedelta(days=2 * 365 + 1)
+        self.assertEqual(prettydate(d=more_than_two_years), '2 years ago')
+        # date()
+        d = now.date()
+        self.assertEqual(prettydate(d=d), 'now')
+        one_day = now.date() - datetime.timedelta(days=1)
+        self.assertEqual(prettydate(d=one_day), '1 day ago')
+        tow_days = now.date() - datetime.timedelta(days=2)
+        self.assertEqual(prettydate(d=tow_days), '2 days ago')
+        # from now
+        # from now is picky depending of the execution time, so we can't use sharp value like 1 second or 1 day
+        in_one_minute = now - datetime.timedelta(seconds=-65)
+        self.assertEqual(prettydate(d=in_one_minute), '1 minute from now')
+        in_twenty_three_hours = now - datetime.timedelta(hours=-23.5)
+        self.assertEqual(prettydate(d=in_twenty_three_hours), '23 hours from now')
+        in_one_year = now - datetime.timedelta(days=-366)
+        self.assertEqual(prettydate(d=in_one_year), '1 year from now')
+        # utc=True
+        now = datetime.datetime.utcnow()
+        self.assertEqual(prettydate(d=now, utc=True), 'now')
+        one_second = now - datetime.timedelta(seconds=1)
+        self.assertEqual(prettydate(d=one_second, utc=True), '1 second ago')
+        # not d or invalid date
+        self.assertEqual(prettydate(d=None), '')
+        self.assertEqual(prettydate(d='invalid_date'), '[invalid date]')
+
 
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_validators.py

@@ -210,6 +210,8 @@ class TestValidators(unittest.TestCase):
         costanza_id = db.person.insert(name='costanza')
         rtn = IS_IN_DB(db, 'person.id', '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
+        rtn = IS_IN_DB(db, 'person.name', '%(name)s')('george')
+        self.assertEqual(rtn, ('george', None))
         rtn = IS_IN_DB(db, db.person, '%(name)s')(george_id)
         self.assertEqual(rtn, (george_id, None))
         rtn = IS_IN_DB(db(db.person.id > 0), db.person, '%(name)s')(george_id)
@@ -222,6 +224,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (george_id + costanza_id, 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True)([george_id, costanza_id])
         self.assertEqual(rtn, ([george_id, costanza_id], None))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, error_message='oops')("I'm not even an id")
+        self.assertEqual(rtn, (["I'm not even an id"], 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=True, delimiter=',')('%d,%d' % (george_id, costanza_id))
         self.assertEqual(rtn, (('%d,%d' % (george_id, costanza_id)).split(','), None))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', multiple=(1, 3), delimiter=',')('%d,%d' % (george_id, costanza_id))
@@ -232,6 +236,17 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(sorted(rtn), [('%d' % george_id, 'george'), ('%d' % costanza_id, 'costanza')])
         rtn = IS_IN_DB(db, db.person.id, db.person.name, error_message='oops', sort=True).options(zero=True)
         self.assertEqual(rtn, [('', ''), ('%d' % costanza_id, 'costanza'), ('%d' % george_id, 'george')])
+        # Test using the set it made for options
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops')
+        vldtr.options()
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', multiple=True)
+        vldtr.options()
+        rtn = vldtr(['george', 'costanza'])
+        self.assertEqual(rtn, (['george', 'costanza'], None))
         # Test it works with self reference
         db.define_table('category',
                         Field('parent_id', 'reference category', requires=IS_EMPTY_OR(IS_IN_DB(db, 'category.id', '%(name)s'))),
@@ -243,14 +258,31 @@ class TestValidators(unittest.TestCase):
         self.assertFalse(list(ret.errors))
         rtn = IS_IN_DB(db, 'category.id', '%(name)s')(ret.id)
         self.assertEqual(rtn, (ret.id, None))
+        # Test _and
+        vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops', _and=IS_LENGTH(maxsize=7, error_message='bad'))
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', 'bad'))
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        vldtr.options() # test theset with _and
+        rtn = vldtr('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        # Test auto_add
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')('jerry')
+        self.assertEqual(rtn, ('jerry', 'oops'))
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', auto_add=True)('jerry')
+        self.assertEqual(rtn, (3, None))
         db.person.drop()
         db.category.drop()
 
     def test_IS_NOT_IN_DB(self):
         from gluon.dal import DAL, Field
         db = DAL('sqlite:memory')
-        db.define_table('person', Field('name'))
+        db.define_table('person', Field('name'), Field('nickname'))
         db.person.insert(name='george')
+        db.person.insert(name='costanza', nickname='T Bone')
         rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops')('george')
         self.assertEqual(rtn, ('george', 'oops'))
         rtn = IS_NOT_IN_DB(db, 'person.name', error_message='oops', allowed_override=['george'])('george')
@@ -261,6 +293,17 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('jerry', None))
         rtn = IS_NOT_IN_DB(db, 'person.name')(u'jerry')
         self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db(db.person.id > 0), 'person.name')(u'jerry')
+        self.assertEqual(rtn, ('jerry', None))
+        rtn = IS_NOT_IN_DB(db, db.person, error_message='oops')(1)
+        self.assertEqual(rtn, ('1', 'oops'))
+        vldtr = IS_NOT_IN_DB(db, 'person.name', error_message='oops')
+        vldtr.set_self_id({'name': 'costanza', 'nickname': 'T Bone'})
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', 'oops'))
+        rtn = vldtr('costanza')
+        self.assertEqual(rtn, ('costanza', None))
+
         db.person.drop()
 
     def test_IS_INT_IN_RANGE(self):
@@ -485,6 +528,9 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('localguy@localhost', 'Enter a valid email address'))
         rtn = IS_EMAIL(forced='^.*\.edu(|\..*)$')('localguy@example.edu')
         self.assertEqual(rtn, ('localguy@example.edu', None))
+        # test for not a string at all
+        rtn = IS_EMAIL(error_message='oops')(42)
+        self.assertEqual(rtn, (42, 'oops'))
 
     def test_IS_LIST_OF_EMAILS(self):
         emails = ['localguy@localhost', '_Yosemite.Sam@example.com']
@@ -560,16 +606,20 @@ class TestValidators(unittest.TestCase):
         # Test timezone is removed and value is properly converted
         #
         # https://github.com/web2py/web2py/issues/1094
+
         class DummyTimezone(datetime.tzinfo):
 
             ONE = datetime.timedelta(hours=1)
 
             def utcoffset(self, dt):
                 return DummyTimezone.ONE
+
             def tzname(self, dt):
                 return "UTC+1"
+
             def dst(self, dt):
                 return DummyTimezone.ONE
+
             def localize(self, dt, is_dst=False):
                 return dt.replace(tzinfo=self)
         v = IS_DATETIME(format="%Y-%m-%d %H:%M", error_message="oops", timezone=DummyTimezone())
@@ -814,6 +864,7 @@ class TestValidators(unittest.TestCase):
 
     def test_IS_IMAGE(self):
         class DummyImageFile(object):
+
             def __init__(self, filename, ext, width, height):
                 from StringIO import StringIO
                 import struct
@@ -873,10 +924,10 @@ class TestValidators(unittest.TestCase):
         rtn = IS_IMAGE(error_message='oops')(img)
         self.assertEqual(rtn, (img, 'oops'))
 
-
     def test_IS_UPLOAD_FILENAME(self):
         import cgi
         from StringIO import StringIO
+
         def gen_fake(filename):
             formdata_file_data = """
 ---123

gluon/tools.py

@@ -1217,7 +1217,7 @@ class AuthJWT(object):
         self.auth = auth
         self.algorithm = algorithm
         if self.algorithm not in ('HS256', 'HS384', 'HS512'):
-            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
+            raise NotImplementedError('Algorithm %s not allowed' % algorithm)
         self.verify_expiration = verify_expiration
         self.leeway = leeway
         self.expiration = expiration
@@ -1314,6 +1314,7 @@ class AuthJWT(object):
         We (mis)use the heavy default auth mechanism to avoid any further computation,
         while sticking to a somewhat-stable Auth API.
         """
+        # TODO: Check the following comment
         ## is the following safe or should we use
         ## calendar.timegm(datetime.datetime.utcnow().timetuple())
         ## result seem to be the same (seconds since epoch, in UTC)
@@ -1395,8 +1396,9 @@ class AuthJWT(object):
             self.alter_payload(payload)
             ret = {'token': self.generate_token(payload)}
         elif ret is None:
-            raise HTTP(
+            raise HTTP(401,
-                401, u'Not Authorized - need to be logged in, to pass a token for refresh or username and password for login',
+                       u'Not Authorized - need to be logged in, to pass a token '
+                       u'for refresh or username and password for login',
                        **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
         response.headers['Content-Type'] = 'application/json'
         return serializers.json(ret)
@@ -1743,7 +1745,7 @@ class Auth(object):
                  csrf_prevention=True, propagate_extension=None,
                  url_index=None, jwt=None, host_names=None):
 
-        ## next two lines for backward compatibility
+        # next two lines for backward compatibility
         if not db and environment and isinstance(environment, DAL):
             db = environment
         self.db = db
@@ -1974,10 +1976,8 @@ class Auth(object):
         else:
             next = '?_next=' + urllib.quote(URL(args=request.args,
                                                 vars=request.get_vars))
-        href = lambda function: '%s/%s%s' % (action, function, next
+        href = lambda function: \
-                                             if referrer_actions is DEFAULT
+            '%s/%s%s' % (action, function, next if referrer_actions is DEFAULT or function in referrer_actions else '')
-                                             or function in referrer_actions
-                                             else '')
         if isinstance(prefix, str):
             prefix = T(prefix)
         if prefix:
@@ -1990,9 +1990,7 @@ class Auth(object):
         if self.user_id:  # User is logged in
             logout_next = self.settings.logout_next
             items.append({'name': T('Log Out'),
-                          'href': '%s/logout?_next=%s' % (action,
+                          'href': '%s/logout?_next=%s' % (action, urllib.quote(logout_next)),
-                                                          urllib.quote(
-                                                          logout_next)),
                           'icon': 'icon-off'})
             if 'profile' not in self.settings.actions_disabled:
                 items.append({'name': T('Profile'), 'href': href('profile'),
@@ -2207,8 +2205,7 @@ class Auth(object):
             if ('id' in fieldnames and
                 'modified_on' in fieldnames and
                 not current_record in fieldnames):
-                table._enable_record_versioning(
+                table._enable_record_versioning(archive_db=archive_db,
-                    archive_db=archive_db,
                                                 archive_name=archive_names,
                                                 current_record=current_record,
                                                 current_record_label=current_record_label)
@@ -2478,9 +2475,7 @@ class Auth(object):
                     Field('expires_on', 'datetime', default=datetime.datetime(2999, 12, 31)),
                     Field('token', writable=False, default=web2py_uuid, unique=True),
                     *extra_fields,
-                    **dict(
+                    **dict(migrate=self.__get_migrate(settings.table_token_name, migrate),
-                        migrate=self.__get_migrate(
-                            settings.table_token_name, migrate),
                            fake_migrate=fake_migrate))
         if not db._lazy_tables:
             settings.table_user = db[settings.table_user_name]
@@ -2532,9 +2527,7 @@ class Auth(object):
         # log messages should not be translated
         if type(description).__name__ == 'lazyT':
             description = description.m
-        self.table_event().insert(
+        self.table_event().insert(description=str(description % vars), origin=origin, user_id=user_id)
-            description=str(description % vars),
-            origin=origin, user_id=user_id)
 
     def get_or_create_user(self, keys, update_fields=['email'],
                            login=True, get=True):
@@ -2583,8 +2576,7 @@ class Auth(object):
             user_id = table_user.insert(**vars)
             user = table_user[user_id]
             if self.settings.create_user_groups:
-                group_id = self.add_group(
+                group_id = self.add_group(self.settings.create_user_groups % user)
-                    self.settings.create_user_groups % user)
                 self.add_membership(group_id, user_id)
             if self.settings.everybody_group_id:
                 self.add_membership(self.settings.everybody_group_id, user_id)
@@ -2750,7 +2742,7 @@ class Auth(object):
             redirect(session._cas_service)
 
         def cas_onaccept(form, onaccept=onaccept):
-            if not onaccept is DEFAULT:
+            if onaccept is not DEFAULT:
                 onaccept(form)
             return allow_access(interactivelogin=True)
         return self.login(next, onvalidation, cas_onaccept, log)
@@ -2837,14 +2829,14 @@ class Auth(object):
         response = current.response
         session = current.session
 
-        ### use session for federated login
+        # use session for federated login
         snext = self.get_vars_next()
 
         if snext:
             session._auth_next = snext
         elif session._auth_next:
             snext = session._auth_next
-        ### pass
+        # pass
 
         if next is DEFAULT:
             # important for security
@@ -2950,7 +2942,7 @@ class Auth(object):
                                )
 
                 captcha = settings.login_captcha or \
-                    (settings.login_captcha != False and settings.captcha)
+                    (settings.login_captcha is not False and settings.captcha)
                 if captcha:
                     addrow(form, captcha.label, captcha, captcha.comment,
                            settings.formstyle, 'captcha__row')
@@ -2978,8 +2970,7 @@ class Auth(object):
                         elif temp_user.registration_key in ('disabled', 'blocked'):
                             response.flash = self.messages.login_disabled
                             return form
-                        elif (temp_user.registration_key is not None
+                        elif (temp_user.registration_key is not None and temp_user.registration_key.strip()):
-                              and temp_user.registration_key.strip()):
                             response.flash = \
                                 self.messages.registration_verifying
                             return form
@@ -3106,8 +3097,6 @@ class Auth(object):
                             hideerror=settings.hideerror):
                 accepted_form = True
 
-                accepted_form = True
-
                 '''
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
@@ -3362,12 +3351,10 @@ class Auth(object):
                         hideerror=self.settings.hideerror):
             description = self.messages.group_description % form.vars
             if self.settings.create_user_groups:
-                group_id = self.add_group(
+                group_id = self.add_group(self.settings.create_user_groups % form.vars, description)
-                    self.settings.create_user_groups % form.vars, description)
                 self.add_membership(group_id, form.vars.id)
             if self.settings.everybody_group_id:
-                self.add_membership(
+                self.add_membership(self.settings.everybody_group_id, form.vars.id)
-                    self.settings.everybody_group_id, form.vars.id)
             if self.settings.registration_requires_verification:
                 link = self.url(
                     self.settings.function, args=('verify_email', key), scheme=True)
@@ -3385,8 +3372,7 @@ class Auth(object):
                not self.settings.registration_requires_verification:
                 table_user[form.vars.id] = dict(registration_key='pending')
                 session.flash = self.messages.registration_pending
-            elif (not self.settings.registration_requires_verification or
+            elif (not self.settings.registration_requires_verification or self.settings.login_after_registration):
-                      self.settings.login_after_registration):
                 if not self.settings.registration_requires_verification:
                     table_user[form.vars.id] = dict(registration_key='')
                 session.flash = self.messages.registration_successful
@@ -3464,7 +3450,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_username_captcha or \
-                  (self.settings.retrieve_username_captcha != False and self.settings.captcha)
+                  (self.settings.retrieve_username_captcha is not False and self.settings.captcha)
         if not self.settings.mailer:
             response.flash = self.messages.function_disabled
             return ''
@@ -3796,8 +3782,7 @@ class Auth(object):
                   requires=self.table_user()[passfield].requires),
             Field('new_password2', 'password',
                   label=self.messages.verify_password,
-                  requires=[IS_EXPR(
+                  requires=[IS_EXPR('value==%s' % repr(request.vars.new_password),
-                      'value==%s' % repr(request.vars.new_password),
                                     self.messages.mismatched_password)]),
             submit_button=self.messages.password_reset_button,
             hidden=dict(_next=next),
@@ -3831,7 +3816,7 @@ class Auth(object):
         response = current.response
         session = current.session
         captcha = self.settings.retrieve_password_captcha or \
-                (self.settings.retrieve_password_captcha != False and self.settings.captcha)
+                (self.settings.retrieve_password_captcha is not False and self.settings.captcha)
 
         if next is DEFAULT:
             next = self.get_vars_next() or self.settings.request_reset_password_next
@@ -4242,10 +4227,8 @@ class Auth(object):
                         else:
                             next = self.here()
                             current.session.flash = current.response.flash
-                            return call_or_redirect(
+                            return call_or_redirect(self.settings.on_failed_authentication,
-                                self.settings.on_failed_authentication,
+                                                    self.settings.login_url + '?_next=' + urllib.quote(next))
-                                self.settings.login_url +
-                                    '?_next=' + urllib.quote(next))
 
                 if callable(condition):
                     flag = condition()
@@ -4326,11 +4309,8 @@ class Auth(object):
         """
         Creates a group associated to a role
         """
-
+        group_id = self.table_group().insert(role=role, description=description)
-        group_id = self.table_group().insert(
+        self.log_event(self.messages['add_group_log'], dict(group_id=group_id, role=role))
-            role=role, description=description)
-        self.log_event(self.messages['add_group_log'],
-                       dict(group_id=group_id, role=role))
         return group_id
 
     def del_group(self, group_id):
@@ -4340,7 +4320,8 @@ class Auth(object):
         self.db(self.table_group().id == group_id).delete()
         self.db(self.table_membership().group_id == group_id).delete()
         self.db(self.table_permission().group_id == group_id).delete()
-        if group_id in self.user_groups: del self.user_groups[group_id]
+        if group_id in self.user_groups:
+            del self.user_groups[group_id]
         self.log_event(self.messages.del_group_log, dict(group_id=group_id))
 
     def id_group(self, role):
@@ -4372,7 +4353,6 @@ class Auth(object):
         """
         Checks if user is member of group_id or role
         """
-
         group_id = group_id or self.id_group(role)
         try:
             group_id = int(group_id)
@@ -4381,8 +4361,8 @@ class Auth(object):
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
-        if group_id and user_id and self.db((membership.user_id == user_id)
+        if group_id and user_id and self.db((membership.user_id == user_id) &
-                    & (membership.group_id == group_id)).select():
+                                            (membership.group_id == group_id)).select():
             r = True
         else:
             r = False
@@ -4429,15 +4409,18 @@ class Auth(object):
         """
 
         group_id = group_id or self.id_group(role)
+        try:
+            group_id = int(group_id)
+        except:
+            group_id = self.id_group(group_id)  # interpret group_id as a role
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
         self.log_event(self.messages['del_membership_log'],
                        dict(user_id=user_id, group_id=group_id))
-        ret = self.db(membership.user_id
+        ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
-                      == user_id)(membership.group_id
+        if group_id in self.user_groups:
-                                  == group_id).delete()
+            del self.user_groups[group_id]
-        if group_id in self.user_groups: del self.user_groups[group_id]
         return ret
 
     def has_permission(self,
@@ -4454,8 +4437,7 @@ class Auth(object):
         """
 
         if not group_id and self.settings.everybody_group_id and \
-                self.has_permission(
+                self.has_permission(name, table_name, record_id, user_id=None,
-            name, table_name, record_id, user_id=None,
                                     group_id=self.settings.everybody_group_id):
                 return True
 
@@ -4463,25 +4445,24 @@ class Auth(object):
             user_id = self.user.id
         if user_id:
             membership = self.table_membership()
-            rows = self.db(membership.user_id
+            rows = self.db(membership.user_id == user_id).select(membership.group_id)
-                           == user_id).select(membership.group_id)
             groups = set([row.group_id for row in rows])
             if group_id and group_id not in groups:
                 return False
         else:
             groups = set([group_id])
         permission = self.table_permission()
-        rows = self.db(permission.name == name)(permission.table_name
+        rows = self.db(permission.name ==
-                 == str(table_name))(permission.record_id
+                       name)(permission.table_name ==
-                 == record_id).select(permission.group_id)
+                             str(table_name))(permission.record_id ==
+                                              record_id).select(permission.group_id)
         groups_required = set([row.group_id for row in rows])
         if record_id:
-            rows = self.db(permission.name
+            rows = self.db(permission.name ==
-                            == name)(permission.table_name
+                           name)(permission.table_name ==
-                     == str(table_name))(permission.record_id
+                                 str(table_name))(permission.record_id ==
-                     == 0).select(permission.group_id)
+                                                  0).select(permission.group_id)
-            groups_required = groups_required.union(set([row.group_id
+            groups_required = groups_required.union(set([row.group_id for row in rows]))
-                    for row in rows]))
         if groups.intersection(groups_required):
             r = True
         else:
@@ -4509,8 +4490,8 @@ class Auth(object):
                          (permission.name == name) &
                          (permission.table_name == str(table_name)) &
                          (permission.record_id == long(record_id)),
-                          ignore_common_filters=True).select(
+                         ignore_common_filters=True
-            limitby=(0, 1), orderby_on_limitby=False).first()
+                         ).select(limitby=(0, 1), orderby_on_limitby=False).first()
         if record:
             if hasattr(record, 'is_active') and not record.is_active:
                 record.update_record(is_active=True)
@@ -4539,10 +4520,11 @@ class Auth(object):
         self.log_event(self.messages['del_permission_log'],
                        dict(group_id=group_id, name=name,
                             table_name=table_name, record_id=record_id))
-        return self.db(permission.group_id == group_id)(permission.name
+        return self.db(permission.group_id ==
-                 == name)(permission.table_name
+                       group_id)(permission.name ==
-                           == str(table_name))(permission.record_id
+                                 name)(permission.table_name ==
-                 == long(record_id)).delete()
+                                       str(table_name))(permission.record_id ==
+                                                        long(record_id)).delete()
 
     def accessible_query(self, name, table, user_id=None):
         """
@@ -4569,8 +4551,7 @@ class Auth(object):
                 cquery = table
             tablenames = db._adapter.tables(cquery)
             for tablename in tablenames:
-                cquery &= self.accessible_query(name, tablename,
+                cquery &= self.accessible_query(name, tablename, user_id=user_id)
-                                                user_id=user_id)
             return cquery
         if not isinstance(table, str) and \
                 self.has_permission(name, table, 0, user_id):
@@ -4639,8 +4620,7 @@ class Auth(object):
         you can do, for example::
 
             db.define_table('myhistory',
-                Field('parent_record',db.mytable),
+                Field('parent_record', db.mytable), db.mytable)
-                db.mytable)
 
         and use it as::
 
@@ -4655,7 +4635,7 @@ class Auth(object):
         table = form.table
         if not archive_table:
             archive_table_name = '%s_archive' % table
-            if not archive_table_name in table._db:
+            if archive_table_name not in table._db:
                 table._db.define_table(
                     archive_table_name,
                     Field(current_record, table),
@@ -4910,18 +4890,16 @@ class Crud(object):
         self.deleted = False
         captcha = self.settings.update_captcha or self.settings.captcha
         if record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
-                         self.settings.formstyle, 'captcha__row')
         captcha = self.settings.create_captcha or self.settings.captcha
         if not record and captcha:
-            addrow(form, captcha.label, captcha, captcha.comment,
+            addrow(form, captcha.label, captcha, captcha.comment, self.settings.formstyle, 'captcha__row')
-                         self.settings.formstyle, 'captcha__row')
         if request.extension not in ('html', 'load'):
             (_session, _formname) = (None, None)
         else:
             (_session, _formname) = (
                 session, '%s/%s' % (table._tablename, form.record_id))
-        if not formname is DEFAULT:
+        if formname is not DEFAULT:
             _formname = formname
         keepvalues = self.settings.keepvalues
         if request.vars.delete_this_record:
@@ -5210,7 +5188,7 @@ class Crud(object):
                     elif validate:
                         value, error = field.validate(txtval)
                         if not error:
-                            ### TODO deal with 'starts with', 'ends with', 'contains' on GAE
+                            # TODO deal with 'starts with', 'ends with', 'contains' on GAE
                             query &= self.get_query(field, opval, value)
                         else:
                             row[3].append(DIV(error, _class='error'))
@@ -5223,7 +5201,7 @@ class Crud(object):
                 results = db(query).select(*selected, **attributes)
                 for r in refsearch:
                     results = results.find(r)
-            except:  # hmmm, we should do better here
+            except:  # TODO: hmmm, we should do better here
                 results = None
         return form, results
 
@@ -5235,7 +5213,7 @@ def fetch(url, data=None, headers=None,
           cookie=Cookie.SimpleCookie(),
           user_agent='Mozilla/5.0'):
     headers = headers or {}
-    if not data is None:
+    if data is not None:
         data = urllib.urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
@@ -5703,7 +5681,7 @@ class Service(object):
         id, method, params = data.get('id'), data.get('method'), data.get('params', [])
         if id is None:
             return return_error(0, 100, 'missing id')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, 100, 'method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -5727,8 +5705,7 @@ class Service(object):
         def return_response(id, result):
             if not must_respond:
                 return None
-            return serializers.json({'jsonrpc': '2.0',
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'result': result})
-                'id': id, 'result': result})
 
         def return_error(id, code, message=None, data=None):
             error = {'code': code}
@@ -5739,9 +5716,7 @@ class Service(object):
                 error['message'] = message
             if data is not None:
                 error['data'] = data
-            return serializers.json({'jsonrpc': '2.0',
+            return serializers.json({'jsonrpc': '2.0', 'id': id, 'error': error})
-                                     'id': id,
-                                     'error': error})
 
         def validate(data):
             """
@@ -5801,7 +5776,7 @@ class Service(object):
             return return_error(None, e.code, e.info)
 
         id, method, params = data.get('id'), data['method'], data.get('params', '')
-        if not method in methods:
+        if method not in methods:
             return return_error(id, -32601, data='Method "%s" does not exist' % method)
         try:
             if isinstance(params, dict):
@@ -6041,7 +6016,7 @@ def prettydate(d, T=lambda x: x, utc=False):
         return T('1 year' + suffix)
     elif dt.days >= 60:
         return T('%d months' + suffix) % int(dt.days / 30)
-    elif dt.days > 21:
+    elif dt.days >= 27:  # 4 weeks ugly
         return T('1 month' + suffix)
     elif dt.days >= 14:
         return T('%d weeks' + suffix) % int(dt.days / 7)
@@ -6447,10 +6422,9 @@ class Wiki(object):
                 args += value['args']
                 db.define_table(key, *args, **value['vars'])
 
-        if self.settings.templates is None and not \
+        if self.settings.templates is None and not self.settings.manage_permissions:
-           self.settings.manage_permissions:
+            self.settings.templates = \
-            self.settings.templates = db.wiki_page.tags.contains('template') & \
+                db.wiki_page.tags.contains('template') & db.wiki_page.can_read.contains('everybody')
-                db.wiki_page.can_read.contains('everybody')
 
         def update_tags_insert(page, id, db=db):
             for tag in page.tags or []:
@@ -6473,8 +6447,10 @@ class Wiki(object):
             'wiki_editor' not in auth.user_groups.values() and
                 self.settings.groups == auth.user_groups.values()):
             group = db.auth_group(role='wiki_editor')
-            gid = group.id if group else db.auth_group.insert(
+            if group:
-                role='wiki_editor')
+                gid = group.id
+            else:
+                db.auth_group.insert(role='wiki_editor')
             auth.add_membership(gid)
 
         settings.lock_keys = True
@@ -6501,8 +6477,7 @@ class Wiki(object):
         groups = self.settings.groups
         return ('wiki_editor' in groups or
                 (page is None and 'wiki_author' in groups) or
-                not page is None and (
+                page is not None and (set(groups).intersection(set(page.can_edit)) or
-                set(groups).intersection(set(page.can_edit)) or
                                       page.created_by == self.auth.user.id))
 
     def can_manage(self):
@@ -6524,7 +6499,7 @@ class Wiki(object):
                     return True
         return False
 
-    ### END POLICY
+    # END POLICY
 
     def automenu(self):
         """adds the menu if not present"""
@@ -6742,19 +6717,14 @@ class Wiki(object):
         if self.settings.templates:
             fields.append(
                 Field("from_template", "reference wiki_page",
-                      requires=IS_EMPTY_OR(
+                      requires=IS_EMPTY_OR(IS_IN_DB(db(self.settings.templates), db.wiki_page._id, '%(slug)s')),
-                                   IS_IN_DB(db(self.settings.templates),
+                      comment=current.T("Choose Template or empty for new Page")))
-                                            db.wiki_page._id,
-                                            '%(slug)s')),
-                      comment=current.T(
-                        "Choose Template or empty for new Page")))
         form = SQLFORM.factory(*fields, **dict(_class="well"))
         form.element("[type=submit]").attributes["_value"] = \
             current.T("Create Page from Slug")
 
         if form.process().accepted:
-             form.vars.from_template = 0 if not form.vars.from_template \
+            form.vars.from_template = 0 if not form.vars.from_template else form.vars.from_template
-                 else form.vars.from_template
             redirect(URL(args=('_edit', form.vars.slug, form.vars.from_template or 0)))  # added param
         return dict(content=form)
 
@@ -6882,7 +6852,7 @@ class Wiki(object):
             if request.vars.q:
                 tags = [v.strip() for v in request.vars.q.split(',')]
                 tags = [v.lower() for v in tags if v]
-        if tags or not query is None:
+        if tags or query is not None:
             db = self.auth.db
             count = db.wiki_tag.wiki_page.count()
             fields = [db.wiki_page.id, db.wiki_page.slug,

gluon/utils.py

@@ -83,11 +83,9 @@ def compare(a, b):
     """ Compares two strings and not vulnerable to timing attacks """
     if HAVE_COMPARE_DIGEST:
         return hmac.compare_digest(a, b)
-    if len(a) != len(b):
+    result = len(a) ^ len(b)
-        return False
+    for i in xrange(len(b)):
-    result = 0
+		result |= ord(a[i%len(a)]) ^ ord(b[i])
-    for x, y in zip(a, b):
-        result |= ord(x) ^ ord(y)
     return result == 0
 
 

gluon/validators.py

@@ -645,11 +645,10 @@ class IS_IN_DB(Validator):
                 if not [v for v in values if v not in self.theset]:
                     return (values, None)
             else:
-                from pydal.adapters import GoogleDatastoreAdapter
-
                 def count(values, s=self.dbset, f=field):
                     return s(f.belongs(map(int, values))).count()
-                if GoogleDatastoreAdapter is not None and isinstance(self.dbset.db._adapter, GoogleDatastoreAdapter):
+
+                if self.dbset.db._adapter.dbengine == "google:datastore":
                     range_ids = range(0, len(values), 30)
                     total = sum(count(values[i:i + 30]) for i in range_ids)
                     if total == len(values):

scripts/lang_update_from_langfile.py

@@ -30,8 +30,15 @@ if __name__ == '__main__':
         dest="source",
         help="Specify language file (ro) where seek for translations"
     )
+    parser.add_argument(
+        '-f', '--force-update',
+        dest="force_update",
+        action="store_true",
+        default=False,
+        help="without it: add new + translate untranslated, if used: in addition update items if translation differs"
+    )
     args = parser.parse_args()
 
-    update_from_langfile(args.target, args.source)
+    update_from_langfile(args.target, args.source, force_update=args.force_update)
 
     print '%s was updated.' % args.target