R-2.12.3

Revert "added default HttpOnly cookies"

.gitignore

@@ -58,3 +58,4 @@ HOWTO-web2py-devel
 *.sublime-project
 *.sublime-workspace
 .idea/*
+site-packages/

.travis.yml

@@ -17,14 +17,14 @@ install:
 before_script:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install --download-cache $HOME/.pip-cache unittest2; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache coverage; fi;
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache python-coveralls; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache codecov; fi
 
 
 script: export COVERAGE_PROCESS_START=gluon/tests/coverage.ini; ./web2py.py --run_system_tests --with_coverage
 
 after_success:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coverage combine; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --config_file=gluon/tests/coverage.ini; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then codecov; fi
 
 notifications:
   email: true

CHANGELOG

@@ -1,3 +1,21 @@
+## 2.12.1
+
+- security fix: Validate for open redirect everywhere, not just in login()
+- allow to pack invidual apps and selected files as packed exe files
+- allow bulk user registration with default bulk_register_enabled=False
+- allow unsorted multiword query in grid search
+- better MongoDB support with newer pyDAL
+- enable <app>/appadmin/manage/auth by default for user admin
+- allow mail.settings.server='logging:filename' to log emails to a file
+- better caching logic
+- fixed order of confirm-password field
+- TLS support in ldap
+- prettydate can do UTC
+- jquery 1.11.3
+- bootstrap 3.3.5
+- moved to codecov and enabled appveyor
+- many bug fixes
+
 ## 2.11.1
 
 - Many small but significative improvements and bug fixes

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.11.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.12.3-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

README.markdown

@@ -38,9 +38,10 @@ PyDAL uses a separate stable release cycle to the rest of web2py. PyDAL releases
 
 ## Tests
 
-[![Build Status](https://img.shields.io/travis/web2py/web2py.svg?style=flat-square)](https://travis-ci.org/web2py/web2py)
+[![Build Status](https://img.shields.io/travis/web2py/web2py/master.svg?style=flat-square&label=Travis-CI)](https://travis-ci.org/web2py/web2py)
+[![MS Build Status](https://img.shields.io/appveyor/ci/web2py/web2py/master.svg?style=flat-square&label=Appveyor-CI)](https://ci.appveyor.com/project/web2py/web2py)
+[![Coverage Status](https://img.shields.io/codecov/c/github/web2py/web2py.svg?style=flat-square)](https://codecov.io/github/web2py/web2py)
 
-[![Coverage Status](https://img.shields.io/coveralls/web2py/web2py.svg?style=flat-square)](https://coveralls.io/r/web2py/web2py)
 
 ## Installation Instructions
 
@@ -63,7 +64,7 @@ That's it!!!
             packages/              > web2py submodules
               dal/
             contrib/               > third party libraries
-            tests/                 > unittests  
+            tests/                 > unittests
         applications/              > are the apps
             admin/                 > web based IDE
                 ...

VERSION

@@ -1 +1 @@
-Version 2.11.2-stable+timestamp.2015.05.30.11.29.46
+Version 2.12.3-stable+timestamp.2015.08.18.19.14.07

applications/admin/controllers/appadmin.py

@@ -49,7 +49,8 @@ if request.function == 'manage':
                                       auth.table_group(),
                                       auth.table_permission()])
     manager_role = manager_action.get('role', None) if manager_action else None
-    auth.requires_membership(manager_role)(lambda: None)()
+    if not (gluon.fileutils.check_credentials(request) or auth.has_membership(manager_role)):
+        raise HTTP(403, "Not authorized")
     menu = False
 elif (request.application == 'admin' and not session.authorized) or \
         (request.application != 'admin' and not gluon.fileutils.check_credentials(request)):
@@ -80,7 +81,6 @@ if False and request.tickets_db:
 def get_databases(request):
     dbs = {}
     for (key, value) in global_env.items():
-        cond = False
         try:
             cond = isinstance(value, GQLDB)
         except:
@@ -420,7 +420,7 @@ def ccache():
         'oldest': time.time(),
         'keys': []
     }
-
+    
     disk = copy.copy(ram)
     total = copy.copy(ram)
     disk['keys'] = []
@@ -445,30 +445,31 @@ def ccache():
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
+        # get ram stats directly from the cache object
+        ram_stats = cache.ram.stats[request.application]
+        ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
+        ram['misses'] = ram_stats['misses']
+        try:
+            ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
+        except (KeyError, ZeroDivisionError):
+            ram['ratio'] = 0
+
         for key, value in cache.ram.storage.iteritems():
-            if isinstance(value, dict):
-                ram['hits'] = value['hit_total'] - value['misses']
-                ram['misses'] = value['misses']
-                try:
-                    ram['ratio'] = ram['hits'] * 100 / value['hit_total']
-                except (KeyError, ZeroDivisionError):
-                    ram['ratio'] = 0
-            else:
-                if hp:
-                    ram['bytes'] += hp.iso(value[1]).size
-                    ram['objects'] += hp.iso(value[1]).count
-                ram['entries'] += 1
-                if value[0] < ram['oldest']:
-                    ram['oldest'] = value[0]
-                ram['keys'].append((key, GetInHMS(time.time() - value[0])))
+            if hp:
+                ram['bytes'] += hp.iso(value[1]).size
+                ram['objects'] += hp.iso(value[1]).count
+            ram['entries'] += 1
+            if value[0] < ram['oldest']:
+                ram['oldest'] = value[0]
+            ram['keys'].append((key, GetInHMS(time.time() - value[0])))
 
         for key in cache.disk.storage:
             value = cache.disk.storage[key]
-            if isinstance(value, dict):
-                disk['hits'] = value['hit_total'] - value['misses']
-                disk['misses'] = value['misses']
+            if isinstance(value[1], dict):
+                disk['hits'] = value[1]['hit_total'] - value[1]['misses']
+                disk['misses'] = value[1]['misses']
                 try:
-                    disk['ratio'] = disk['hits'] * 100 / value['hit_total']
+                    disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:
@@ -480,12 +481,12 @@ def ccache():
                     disk['oldest'] = value[0]
                 disk['keys'].append((key, GetInHMS(time.time() - value[0])))
 
-        total['entries'] = ram['entries'] + disk['entries']
-        total['bytes'] = ram['bytes'] + disk['bytes']
-        total['objects'] = ram['objects'] + disk['objects']
-        total['hits'] = ram['hits'] + disk['hits']
-        total['misses'] = ram['misses'] + disk['misses']
-        total['keys'] = ram['keys'] + disk['keys']
+        ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
+        ram_keys.remove('ratio')
+        ram_keys.remove('oldest')
+        for key in ram_keys:
+            total[key] = ram[key] + disk[key]
+
         try:
             total['ratio'] = total['hits'] * 100 / (total['hits'] +
                                                 total['misses'])
@@ -577,9 +578,7 @@ def bg_graph_model():
         group = meta_graphmodel['group'].replace(' ', '')
         if not subgraphs.has_key(group):
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
-            subgraphs[group]['tables'].append(tablename)
-        else:
-            subgraphs[group]['tables'].append(tablename)
+        subgraphs[group]['tables'].append(tablename)
 
         graph.add_node(tablename, name=tablename, shape='plaintext',
                        label=table_template(tablename))

applications/admin/controllers/debug.py

@@ -220,7 +220,7 @@ def list_breakpoints():
     "Return a list of linenumbers for current breakpoints"
 
     breakpoints = []
-    ok = None
+    ok = False
     try:
         filename = os.path.join(request.env['applications_parent'],
                                 'applications', request.vars.filename)
@@ -235,5 +235,4 @@ def list_breakpoints():
         ok = True
     except Exception, e:
         session.flash = str(e)
-        ok = False
     return response.json({'ok': ok, 'breakpoints': breakpoints})

applications/admin/controllers/default.py

@@ -292,9 +292,6 @@ def site():
             log_progress(appname)
             session.flash = T(msg, dict(appname=appname,
                                         digest=md5_hash(installed)))
-        elif f and form_update.vars.overwrite:
-            msg = 'unable to install application "%(appname)s"'
-            session.flash = T(msg, dict(appname=form_update.vars.name))
         else:
             msg = 'unable to install application "%(appname)s"'
             session.flash = T(msg, dict(appname=form_update.vars.name))
@@ -370,25 +367,56 @@ def pack_plugin():
         session.flash = T('internal error')
         redirect(URL('plugin', args=request.args))
 
+
+
+def pack_exe(app, base, filenames=None):
+    import urllib
+    import zipfile
+    from cStringIO import StringIO
+    # Download latest web2py_win and open it with zipfile
+    download_url = 'http://www.web2py.com/examples/static/web2py_win.zip'
+    out = StringIO()
+    out.write(urllib.urlopen(download_url).read())
+    web2py_win = zipfile.ZipFile(out, mode='a')
+    # Write routes.py with the application as default
+    routes = u'# -*- coding: utf-8 -*-\nrouters = dict(BASE=dict(default_application="%s"))' % app
+    web2py_win.writestr('web2py/routes.py', routes.encode('utf-8'))
+    # Copy the application into the zipfile
+    common_root = os.path.dirname(base)
+    for filename in filenames:
+        fname = os.path.join(base, filename)
+        arcname = os.path.join('web2py/applications', app, filename)
+        web2py_win.write(fname, arcname)
+    web2py_win.close()
+    response.headers['Content-Type'] = 'application/zip'
+    response.headers['Content-Disposition'] = 'attachment; filename=web2py.app.%s.zip' % app
+    out.seek(0)
+    return response.stream(out)
+
+
 def pack_custom():
     app = get_app()
     base = apath(app, r=request)
     if request.post_vars.file:
+        
         files = request.post_vars.file
         files = [files] if not isinstance(files,list) else files
-        fname = 'web2py.app.%s.w2p' % app
-        try:
-            filename = app_pack(app, request, raise_ex=True, filenames=files)
-        except Exception, e:
-            filename = None
-        if filename:
-            response.headers['Content-Type'] = 'application/w2p'
-            disposition = 'attachment; filename=%s' % fname
-            response.headers['Content-Disposition'] = disposition
-            return safe_read(filename, 'rb')
+        if request.post_vars.doexe is None:
+            fname = 'web2py.app.%s.w2p' % app
+            try:
+                filename = app_pack(app, request, raise_ex=True, filenames=files)
+            except Exception, e:
+                filename = None
+            if filename:
+                response.headers['Content-Type'] = 'application/w2p'
+                disposition = 'attachment; filename=%s' % fname
+                response.headers['Content-Disposition'] = disposition
+                return safe_read(filename, 'rb')
+            else:
+                session.flash = T('internal error: %s', e)
+                redirect(URL(args=request.args))
         else:
-            session.flash = T('internal error: %s', e)
-            redirect(URL(args=request.args))
+            return pack_exe(app, base, files)
     def ignore(fs):
         return [f for f in fs if not (
                 f[:1] in '#' or f.endswith('~') or f.endswith('.bak'))]
@@ -867,13 +895,9 @@ def resolve():
 
     def getclass(item):
         """ Determine item class """
-
-        if item[0] == ' ':
-            return 'normal'
-        if item[0] == '+':
-            return 'plus'
-        if item[0] == '-':
-            return 'minus'
+        operators = {' ':'normal', '+':'plus', '-':'minus'}
+        
+        return operators[item[0]]
 
     if request.vars:
         c = '\n'.join([item[2:].rstrip() for (i, item) in enumerate(d) if item[0]
@@ -1510,7 +1534,7 @@ def upload_file():
         if filename:
             d = dict(filename=filename[len(path):])
         else:
-            d = dict(filename='unkown')
+            d = dict(filename='unknown')
         session.flash = T('cannot upload file "%(filename)s"', d)
 
     redirect(request.vars.sender)

applications/admin/views/appadmin.html

@@ -137,9 +137,9 @@
     <h4>{{=T("Overview")}}</h4>
     <p>{{=T.M("Number of entries: **%s**", total['entries'])}}</p>
     {{if total['entries'] > 0:}}
-      <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses})",
-              dict(ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
-     </p>
+    <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses)})",
+             dict( ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
+    </p>
     <p>
       {{=T("Size of cache:")}}
       {{if object_stats:}}
@@ -155,7 +155,7 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle();')}}
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="all_keys">
       {{=total['keys']}}
     </div>
@@ -183,7 +183,7 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle();')}}
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
@@ -212,7 +212,7 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle();')}}
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
       <div class="hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>

applications/admin/views/default/design.html

@@ -1,5 +1,7 @@
 {{extend 'layout.html'}}
 {{
+import re
+regex_space = re.compile('\s+')
 def all(items):
     return reduce(lambda a,b:a and b,items,True)
 def peekfile(path,file,vars={},title=None):
@@ -304,7 +306,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
             while path!=file_path:
                 if len(file_path)>=len(path) and all([v==file_path[k] for k,v in enumerate(path)]):
                     path.append(file_path[len(path)])
-                    thispath='static__'+'__'.join(path)
+                    thispath = regex_space.sub('-', 'static__'+'__'.join(path))
     }}
         <li class="folder"><i>&nbsp;</i>
             <a href="javascript:collapse('{{=thispath}}');" class="file">{{=path[-1]}}/</a>

applications/admin/views/default/pack_custom.html

@@ -25,6 +25,7 @@
 <form action="{{=URL(args=request.args)}}" method="POST">
 <h2>{{=T('Select Files to Package')}}</h2>
 <input type="submit" value="{{=T('Download .w2p')}}" class="btn"/>
+<input type="submit" name="doexe" value="{{=T('Download as .exe')}}" class="btn"/>
 <div style="margin-top:20px">
 {{tree(base)}}
 </div>

applications/examples/controllers/appadmin.py

@@ -49,7 +49,8 @@ if request.function == 'manage':
                                       auth.table_group(),
                                       auth.table_permission()])
     manager_role = manager_action.get('role', None) if manager_action else None
-    auth.requires_membership(manager_role)(lambda: None)()
+    if not (gluon.fileutils.check_credentials(request) or auth.has_membership(manager_role)):
+        raise HTTP(403, "Not authorized")
     menu = False
 elif (request.application == 'admin' and not session.authorized) or \
         (request.application != 'admin' and not gluon.fileutils.check_credentials(request)):
@@ -80,7 +81,6 @@ if False and request.tickets_db:
 def get_databases(request):
     dbs = {}
     for (key, value) in global_env.items():
-        cond = False
         try:
             cond = isinstance(value, GQLDB)
         except:
@@ -420,7 +420,7 @@ def ccache():
         'oldest': time.time(),
         'keys': []
     }
-
+    
     disk = copy.copy(ram)
     total = copy.copy(ram)
     disk['keys'] = []
@@ -445,30 +445,31 @@ def ccache():
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
+        # get ram stats directly from the cache object
+        ram_stats = cache.ram.stats[request.application]
+        ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
+        ram['misses'] = ram_stats['misses']
+        try:
+            ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
+        except (KeyError, ZeroDivisionError):
+            ram['ratio'] = 0
+
         for key, value in cache.ram.storage.iteritems():
-            if isinstance(value, dict):
-                ram['hits'] = value['hit_total'] - value['misses']
-                ram['misses'] = value['misses']
-                try:
-                    ram['ratio'] = ram['hits'] * 100 / value['hit_total']
-                except (KeyError, ZeroDivisionError):
-                    ram['ratio'] = 0
-            else:
-                if hp:
-                    ram['bytes'] += hp.iso(value[1]).size
-                    ram['objects'] += hp.iso(value[1]).count
-                ram['entries'] += 1
-                if value[0] < ram['oldest']:
-                    ram['oldest'] = value[0]
-                ram['keys'].append((key, GetInHMS(time.time() - value[0])))
+            if hp:
+                ram['bytes'] += hp.iso(value[1]).size
+                ram['objects'] += hp.iso(value[1]).count
+            ram['entries'] += 1
+            if value[0] < ram['oldest']:
+                ram['oldest'] = value[0]
+            ram['keys'].append((key, GetInHMS(time.time() - value[0])))
 
         for key in cache.disk.storage:
             value = cache.disk.storage[key]
-            if isinstance(value, dict):
-                disk['hits'] = value['hit_total'] - value['misses']
-                disk['misses'] = value['misses']
+            if isinstance(value[1], dict):
+                disk['hits'] = value[1]['hit_total'] - value[1]['misses']
+                disk['misses'] = value[1]['misses']
                 try:
-                    disk['ratio'] = disk['hits'] * 100 / value['hit_total']
+                    disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:
@@ -480,12 +481,12 @@ def ccache():
                     disk['oldest'] = value[0]
                 disk['keys'].append((key, GetInHMS(time.time() - value[0])))
 
-        total['entries'] = ram['entries'] + disk['entries']
-        total['bytes'] = ram['bytes'] + disk['bytes']
-        total['objects'] = ram['objects'] + disk['objects']
-        total['hits'] = ram['hits'] + disk['hits']
-        total['misses'] = ram['misses'] + disk['misses']
-        total['keys'] = ram['keys'] + disk['keys']
+        ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
+        ram_keys.remove('ratio')
+        ram_keys.remove('oldest')
+        for key in ram_keys:
+            total[key] = ram[key] + disk[key]
+
         try:
             total['ratio'] = total['hits'] * 100 / (total['hits'] +
                                                 total['misses'])
@@ -577,9 +578,7 @@ def bg_graph_model():
         group = meta_graphmodel['group'].replace(' ', '')
         if not subgraphs.has_key(group):
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
-            subgraphs[group]['tables'].append(tablename)
-        else:
-            subgraphs[group]['tables'].append(tablename)
+        subgraphs[group]['tables'].append(tablename)
 
         graph.add_node(tablename, name=tablename, shape='plaintext',
                        label=table_template(tablename))

applications/examples/private/content/en/default/documentation/more.markmin

@@ -8,6 +8,7 @@
 
 #### Learning and Demos
 - [[Intro video http://www.youtube.com/watch?v=BXzqmHx6edY]] and [[code examples https://github.com/mjhea0/web2py]]
+- [[Step by step tutorial https://milesm.pythonanywhere.com/wiki]]
 - [[Killer Web Development Tutorial http://killer-web-development.com/]]
 - [[Real Python for the Web http://www.realpython.com]] (web development with web2py and more!)
 - [[Admin Demo http://www.web2py.com/demo_admin popup]] (web-based IDE)

applications/examples/views/appadmin.html

@@ -137,9 +137,9 @@
     <h4>{{=T("Overview")}}</h4>
     <p>{{=T.M("Number of entries: **%s**", total['entries'])}}</p>
     {{if total['entries'] > 0:}}
-      <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses})",
-              dict(ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
-     </p>
+    <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses)})",
+             dict( ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
+    </p>
     <p>
       {{=T("Size of cache:")}}
       {{if object_stats:}}
@@ -155,7 +155,7 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle();')}}
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="all_keys">
       {{=total['keys']}}
     </div>
@@ -183,7 +183,7 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle();')}}
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
@@ -212,7 +212,7 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle();')}}
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
       <div class="hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>

applications/examples/views/default/support.html

@@ -17,15 +17,19 @@
     <ul>
       <li><a target="_blank" href="http://experts4solutions.com">Experts4Soutions</a> (worldwide)</li>
       <li><a target="_blank" href="http://www.planethost.com">PlanetHost</a> (USA)</li>
+      <li><a target="_blank" href="http://www.10biosystems.com">10BioSystems</a> (USA)</li>
       <li><a target="_blank" href="http://www.formatics.nl">Formatics</a> (Netherlands)</li>
+      <li><a target="_blank" href="http://www.corebyte.nl">Corebyte</a> (Netherlands)</li>
+      <li><a target="_blank" href="http://www.dutveul.nl">Dutveul</a> (Netherlands)</li>
       <li><a target="_blank" href="http://www.onemewebservices.com">OneMeWebServices</a> (Canada)</li>
       <li><a target="_blank" href="http://www.budgetbytes.nl">BudgetBytes</a> (The Netherlands)</li>
       <li><a target="_blank" href="http://www.androsoft.pl">ANDROSoft</a> (Poland)</li>
 
-      <li><a target="_blank" href="www.sonnetech.com.br">Sonne Tech</a> (Brazil)</li>
+      <li><a target="_blank" href="http://www.sonnetech.com.br">Sonne Tech</a> (Brazil)</li>
       <li><a target="_blank" href="http://www.nrg.com.br">NRG Internet Solutions</a> (Brazil)</li>
       <li><a target="_blank" href="http://itjp.net.br/">ITJP</a> (Brazil)</li>
       <li><a target="_blank" href="http://i-am.pt">I am Consultoria</a> (Portugal)</li>
+      <li><a target="_blank" href="http://www.definescope.com/">DefineScope</a> (Portugal)</li>
       <li><a target="_blank" href="http://lpfx.com.br">LPFX</a> (Brazil)</li>
       <li><a target="_blank" href="http://emotionull.com">Emotionull</a> (Greece and Cyprus)</li>
       <li><a target="_blank" href="http://www.vsa-services.com/">VSA Services</a> (Singapore)</li>

applications/welcome/controllers/appadmin.py

@@ -49,7 +49,8 @@ if request.function == 'manage':
                                       auth.table_group(),
                                       auth.table_permission()])
     manager_role = manager_action.get('role', None) if manager_action else None
-    auth.requires_membership(manager_role)(lambda: None)()
+    if not (gluon.fileutils.check_credentials(request) or auth.has_membership(manager_role)):
+        raise HTTP(403, "Not authorized")
     menu = False
 elif (request.application == 'admin' and not session.authorized) or \
         (request.application != 'admin' and not gluon.fileutils.check_credentials(request)):
@@ -80,7 +81,6 @@ if False and request.tickets_db:
 def get_databases(request):
     dbs = {}
     for (key, value) in global_env.items():
-        cond = False
         try:
             cond = isinstance(value, GQLDB)
         except:
@@ -420,7 +420,7 @@ def ccache():
         'oldest': time.time(),
         'keys': []
     }
-
+    
     disk = copy.copy(ram)
     total = copy.copy(ram)
     disk['keys'] = []
@@ -445,30 +445,31 @@ def ccache():
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
+        # get ram stats directly from the cache object
+        ram_stats = cache.ram.stats[request.application]
+        ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
+        ram['misses'] = ram_stats['misses']
+        try:
+            ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
+        except (KeyError, ZeroDivisionError):
+            ram['ratio'] = 0
+
         for key, value in cache.ram.storage.iteritems():
-            if isinstance(value, dict):
-                ram['hits'] = value['hit_total'] - value['misses']
-                ram['misses'] = value['misses']
-                try:
-                    ram['ratio'] = ram['hits'] * 100 / value['hit_total']
-                except (KeyError, ZeroDivisionError):
-                    ram['ratio'] = 0
-            else:
-                if hp:
-                    ram['bytes'] += hp.iso(value[1]).size
-                    ram['objects'] += hp.iso(value[1]).count
-                ram['entries'] += 1
-                if value[0] < ram['oldest']:
-                    ram['oldest'] = value[0]
-                ram['keys'].append((key, GetInHMS(time.time() - value[0])))
+            if hp:
+                ram['bytes'] += hp.iso(value[1]).size
+                ram['objects'] += hp.iso(value[1]).count
+            ram['entries'] += 1
+            if value[0] < ram['oldest']:
+                ram['oldest'] = value[0]
+            ram['keys'].append((key, GetInHMS(time.time() - value[0])))
 
         for key in cache.disk.storage:
             value = cache.disk.storage[key]
-            if isinstance(value, dict):
-                disk['hits'] = value['hit_total'] - value['misses']
-                disk['misses'] = value['misses']
+            if isinstance(value[1], dict):
+                disk['hits'] = value[1]['hit_total'] - value[1]['misses']
+                disk['misses'] = value[1]['misses']
                 try:
-                    disk['ratio'] = disk['hits'] * 100 / value['hit_total']
+                    disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:
@@ -480,12 +481,12 @@ def ccache():
                     disk['oldest'] = value[0]
                 disk['keys'].append((key, GetInHMS(time.time() - value[0])))
 
-        total['entries'] = ram['entries'] + disk['entries']
-        total['bytes'] = ram['bytes'] + disk['bytes']
-        total['objects'] = ram['objects'] + disk['objects']
-        total['hits'] = ram['hits'] + disk['hits']
-        total['misses'] = ram['misses'] + disk['misses']
-        total['keys'] = ram['keys'] + disk['keys']
+        ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
+        ram_keys.remove('ratio')
+        ram_keys.remove('oldest')
+        for key in ram_keys:
+            total[key] = ram[key] + disk[key]
+
         try:
             total['ratio'] = total['hits'] * 100 / (total['hits'] +
                                                 total['misses'])
@@ -577,9 +578,7 @@ def bg_graph_model():
         group = meta_graphmodel['group'].replace(' ', '')
         if not subgraphs.has_key(group):
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
-            subgraphs[group]['tables'].append(tablename)
-        else:
-            subgraphs[group]['tables'].append(tablename)
+        subgraphs[group]['tables'].append(tablename)
 
         graph.add_node(tablename, name=tablename, shape='plaintext',
                        label=table_template(tablename))

applications/welcome/controllers/default.py

@@ -30,6 +30,7 @@ def user():
     http://..../[app]/default/user/retrieve_password
     http://..../[app]/default/user/change_password
     http://..../[app]/default/user/manage_users (requires membership in
+    http://..../[app]/default/user/bulk_register
     use @auth.requires_login()
         @auth.requires_membership('group name')
         @auth.requires_permission('read','table name',record_id)

applications/welcome/static/css/web2py-bootstrap3.css

@@ -24,7 +24,9 @@ div.flash.alert:hover {
 .ie-lte8 div.flash:hover {
     filter: alpha(opacity=25);
 }
-
+.main-container {
+    margin-top: 20px;
+}
 
 div.error {
     width: auto;
@@ -283,6 +285,7 @@ li.w2p_grid_breadcrumb_elem {
 .web2py_console .form-control {
     width: 20%;
     display: inline;
+    height: 100%;
 }
 .web2py_console #w2p_keywords {
     width: 50%;

applications/welcome/views/appadmin.html

@@ -137,9 +137,9 @@
     <h4>{{=T("Overview")}}</h4>
     <p>{{=T.M("Number of entries: **%s**", total['entries'])}}</p>
     {{if total['entries'] > 0:}}
-      <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses})",
-              dict(ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
-     </p>
+    <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses)})",
+             dict( ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
+    </p>
     <p>
       {{=T("Size of cache:")}}
       {{if object_stats:}}
@@ -155,7 +155,7 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle();')}}
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="all_keys">
       {{=total['keys']}}
     </div>
@@ -183,7 +183,7 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle();')}}
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
@@ -212,7 +212,7 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle();')}}
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
       <div class="hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>

applications/welcome/views/layout.html

@@ -75,7 +75,7 @@
     {{end}}
     <!-- Main ========================================= -->
     <!-- Begin page content -->
-    <div class="container-fluid">
+    <div class="container-fluid main-container">
       {{if left_sidebar_enabled:}}
         <div class="col-md-3 left-sidebar">
             {{block left_sidebar}}

appveyor.yml

@@ -0,0 +1,25 @@
+build: false
+
+environment:
+  matrix:
+    - PYTHON: "C:/Python27"
+      COVERAGE_PROCESS_START: gluon/tests/coverage.ini
+
+clone_depth: 50
+
+init:
+  - "ECHO %PYTHON%"
+  - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%
+
+install:
+  - ps: Start-FileDownload https://bootstrap.pypa.io/get-pip.py
+  - python get-pip.py
+  - pip install codecov
+  - git submodule update --init --recursive
+
+test_script:
+  - python web2py.py --run_system_tests --with_coverage
+
+after_test:
+  - coverage combine
+  - codecov

gluon/admin.py

@@ -59,6 +59,8 @@ def app_pack(app, request, raise_ex=False, filenames=None):
         w2p_pack(filename, apath(app, request), filenames=filenames)
         return filename
     except Exception, e:
+        import traceback
+        print traceback.format_exc()
         if raise_ex:
             raise
         return False
@@ -118,10 +120,9 @@ def app_cleanup(app, request):
                 r = False
 
     # Remove cache files
-    path = apath('%s/cache/' % app, request)
-    CacheOnDisk(folder=path).clear()
-    
+    path = apath('%s/cache/' % app, request)    
     if os.path.exists(path):
+        CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
                 if f[:1] != '.': recursive_unlink(os.path.join(path, f))

gluon/compileapp.py

@@ -261,7 +261,7 @@ class LoadFactory(object):
         import globals
         target = target or 'c' + str(random.random())[2:]
         attr['_id'] = target
-        request = self.environment['request']
+        request = current.request
         if '.' in f:
             f, extension = f.rsplit('.', 1)
         if url or ajax:
@@ -532,10 +532,11 @@ def run_models_in(environment):
     It tries pre-compiled models first before compiling them.
     """
 
-    folder = environment['request'].folder
-    c = environment['request'].controller
+    request = current.request
+    folder = request.folder
+    c = request.controller
     #f = environment['request'].function
-    response = environment['response']
+    response = current.response
 
     path = pjoin(folder, 'models')
     cpath = pjoin(folder, 'compiled')
@@ -577,7 +578,7 @@ def run_controller_in(controller, function, environment):
     """
 
     # if compiled should run compiled!
-    folder = environment['request'].folder
+    folder = current.request.folder
     path = pjoin(folder, 'compiled')
     badc = 'invalid controller (%s/%s)' % (controller, function)
     badf = 'invalid function (%s/%s)' % (controller, function)
@@ -631,7 +632,7 @@ def run_controller_in(controller, function, environment):
             layer = filename + ':' + function
             code = getcfs(layer, filename, lambda: compile2(code, layer))
         restricted(code, environment, filename)
-    response = environment['response']
+    response = current.response
     vars = response._vars
     if response.postprocessing:
         vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
@@ -649,8 +650,8 @@ def run_view_in(environment):
     or `view/generic.extension`
     It tries the pre-compiled views_controller_function.pyc before compiling it.
     """
-    request = environment['request']
-    response = environment['response']
+    request = current.request
+    response = current.response
     view = response.view
     folder = request.folder
     path = pjoin(folder, 'compiled')

gluon/contrib/imageutils.py

@@ -6,15 +6,17 @@
 #
 # Given the model
 #
-# db.define_table("table_name", Field("picture", "upload"), Field("thumbnail", "upload"))
+# db.define_table("table_name", Field("picture", "upload"),
+#                 Field("thumbnail", "upload"))
 #
-# # to resize the picture on upload
+# to resize the picture on upload
 #
 # from images import RESIZE
 #
 # db.table_name.picture.requires = RESIZE(200, 200)
 #
-# # to store original image in picture and create a thumbnail in 'thumbnail' field
+# to store original image in picture and create a thumbnail
+# in 'thumbnail' field
 #
 # from images import THUMB
 # db.table_name.thumbnail.compute = lambda row: THUMB(row.picture, 200, 200)
@@ -24,8 +26,11 @@ from gluon import current
 
 
 class RESIZE(object):
-    def __init__(self, nx=160, ny=80, error_message=' image resize'):
-        (self.nx, self.ny, self.error_message) = (nx, ny, error_message)
+
+    def __init__(self, nx=160, ny=80, quality=100,
+                 error_message=' image resize'):
+        (self.nx, self.ny, self.quality, self.error_message) = (
+            nx, ny, quality, error_message)
 
     def __call__(self, value):
         if isinstance(value, str) and len(value) == 0:
@@ -36,7 +41,7 @@ class RESIZE(object):
             img = Image.open(value.file)
             img.thumbnail((self.nx, self.ny), Image.ANTIALIAS)
             s = cStringIO.StringIO()
-            img.save(s, 'JPEG', quality=100)
+            img.save(s, 'JPEG', quality=self.quality)
             s.seek(0)
             value.file = s
         except:
@@ -51,7 +56,7 @@ def THUMB(image, nx=120, ny=120, gae=False, name='thumb'):
             request = current.request
             from PIL import Image
             import os
-            img = Image.open(os.path.join(request.folder,'uploads',image))
+            img = Image.open(os.path.join(request.folder, 'uploads', image))
             img.thumbnail((nx, ny), Image.ANTIALIAS)
             root, ext = os.path.splitext(image)
             thumb = '%s_%s%s' % (root, name, ext)

gluon/contrib/login_methods/ldap_auth.py

@@ -33,6 +33,7 @@ def ldap_auth(server='ldap', port=None,
               group_name_attrib='cn',
               group_member_attrib='memberUid',
               group_filterstr='objectClass=*',
+              tls=False,
               logging_level='error'):
 
     """
@@ -80,6 +81,13 @@ def ldap_auth(server='ldap', port=None,
     If ldap is using GnuTLS then you need cert_file="..." instead cert_path
     because cert_path isn't implemented in GnuTLS :(
 
+    To enable TLS, set tls=True:
+
+        auth.settings.login_methods.append(ldap_auth(
+            server='my.ldap.server',
+            base_dn='ou=Users,dc=domain,dc=com',
+            tls=True))
+
     If you need to bind to the directory with an admin account in order to
     search it then specify bind_dn & bind_pw to use for this.
     - currently only implemented for Active Directory
@@ -610,6 +618,8 @@ def ldap_auth(server='ldap', port=None,
                 ldap_port = 389
             con = ldap.initialize(
                 "ldap://" + ldap_server + ":" + str(ldap_port))
+        if tls:
+            con.start_tls_s()
         return con
 
     def get_user_groups_from_ldap(username,

gluon/contrib/login_methods/saml2_auth.py

@@ -13,6 +13,7 @@ Include in your model (eg db.py)::
 
     auth.define_tables(username=True)
     from gluon.contrib.login_methods.saml2_auth import Saml2Auth
+    import os
     auth.settings.login_form=Saml2Auth(
     config_file = os.path.join(request.folder,'private','sp_conf'),
     maps=dict(
@@ -20,10 +21,59 @@ Include in your model (eg db.py)::
         email=lambda v: v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
         user_id=lambda v: v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0]))
         
-you must have private/sp_conf.py, the pysaml2 sp configuration file
+you must have private/sp_conf.py, the pysaml2 sp configuration file. For example:
+
+
+    #!/usr/bin/env python
+    # -*- coding: utf-8 -*-
+
+    from saml2 import BINDING_HTTP_POST, BINDING_HTTP_REDIRECT
+    import os.path
+    import requests
+    import tempfile
+
+    BASEDIR = os.path.abspath(os.path.dirname(__file__))
+
+    # Web2py SP url and application name
+    HOST = 'http://127.0.0.1:8000'
+    APP = 'sp'
+
+    # To load the IDP metadata...
+    IDP_METADATA = 'http://127.0.0.1:8088/metadata'
+
+    def full_path(local_file):
+        return os.path.join(BASEDIR, local_file)
+
+    CONFIG = {                
+        # your entity id, usually your subdomain plus the url to the metadata view.
+        'entityid': '%s/%s/default/metadata' % (HOST, APP),
+        'service': {
+            'sp' : {
+                'name': 'MYSP',  
+                'endpoints': {     
+                    'assertion_consumer_service': [
+                        ('%s/%s/default/user/login' % (HOST, APP), BINDING_HTTP_REDIRECT),
+                        ('%s/%s/default/user/login' % (HOST, APP), BINDING_HTTP_POST),       
+                        ],
+                    },
+                },
+            },
+        # Your private and public key.
+        'key_file': full_path('pki/mykey.pem'),
+        'cert_file': full_path('pki/mycert.pem'),
+        
+        # where the remote metadata is stored
+        'metadata': {
+            "remote": [{
+                "url": IDP_METADATA,
+                "cert":full_path('pki/mycert.pem')
+                }]
+            },
+    }
+
 """
 
-from saml2 import BINDING_HTTP_REDIRECT
+from saml2 import BINDING_HTTP_REDIRECT, BINDING_HTTP_POST
 from saml2.client import Saml2Client
 from gluon.utils import web2py_uuid
 from gluon import current, redirect, URL
@@ -59,10 +109,13 @@ def saml2_handler(session, request, config_filename = None):
     client = Saml2Client(config_file = config_filename)
     idps = client.metadata.with_descriptor("idpsso")
     entityid = idps.keys()[0]
-    bindings = [BINDING_HTTP_REDIRECT]
+    bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST]
     binding, destination = client.pick_binding(
         "single_sign_on_service", bindings, "idpsso", entity_id=entityid)
-    binding = BINDING_HTTP_REDIRECT 
+    if request.env.request_method == 'GET':
+        binding = BINDING_HTTP_REDIRECT 
+    elif request.env.request_method == 'POST':
+        binding = BINDING_HTTP_POST
     if not request.vars.SAMLResponse:
         req_id, req = client.create_authn_request(destination, binding=binding)
         relay_state = web2py_uuid().replace('-','')

gluon/contrib/simplejsonrpc.py

@@ -34,7 +34,7 @@ except ImportError:
 class JSONRPCError(RuntimeError):
     "Error object for remote procedure call fail"
     def __init__(self, code, message, data=None):
-        value = "%s: %s\n%s" % (code, message, '\n'.join(data))
+        value = "%s: %s\n%s" % (code, message, '\n'.join(data or ''))
         RuntimeError.__init__(self, value)
         self.code = code
         self.message = message

gluon/contrib/websocket_messaging.py

@@ -145,6 +145,10 @@ class TokenHandler(tornado.web.RequestHandler):
 
 
 class DistributeHandler(tornado.websocket.WebSocketHandler):
+   
+   def check_origin(self, origin):
+      return True
+
     def open(self, params):
         group, token, name = params.split('/') + [None, None]
         self.group = group or 'default'

gluon/portalocker.py

@@ -119,7 +119,7 @@ class LockedFile(object):
             lock(self.file, LOCK_EX)
             if not 'a' in mode:
                 self.file.seek(0)
-                self.file.truncate()
+                self.file.truncate(0)
         else:
             raise RuntimeError("invalid LockedFile(...,mode)")
 

gluon/sanitizer.py

@@ -66,14 +66,15 @@ class XssCleaner(HTMLParser):
 
         #to strip or escape disallowed tags?
         self.strip_disallowed = strip_disallowed
-        self.in_disallowed = False
+        # there might be data after final closing tag, that is to be ignored
+        self.in_disallowed = [False]
 
     def handle_data(self, data):
-        if data and not self.in_disallowed:
+        if data and not self.in_disallowed[-1]:
             self.result += xssescape(data)
 
     def handle_charref(self, ref):
-        if self.in_disallowed:
+        if self.in_disallowed[-1]:
             return
         elif len(ref) < 7 and (ref.isdigit() or ref == 'x27'): # x27 is a special case for apostrophe
             self.result += '&#%s;' % ref
@@ -81,7 +82,7 @@ class XssCleaner(HTMLParser):
             self.result += xssescape('&#%s' % ref)
 
     def handle_entityref(self, ref):
-        if self.in_disallowed:
+        if self.in_disallowed[-1]:
             return
         elif ref in entitydefs:
             self.result += '&%s;' % ref
@@ -89,7 +90,7 @@ class XssCleaner(HTMLParser):
             self.result += xssescape('&%s' % ref)
 
     def handle_comment(self, comment):
-        if self.in_disallowed:
+        if self.in_disallowed[-1]:
             return
         elif comment:
             self.result += xssescape('<!--%s-->' % comment)
@@ -100,11 +101,11 @@ class XssCleaner(HTMLParser):
         attrs
     ):
         if tag not in self.permitted_tags:
-            if self.strip_disallowed:
-                self.in_disallowed = True
-            else:
+            self.in_disallowed.append(True)
+            if (not self.strip_disallowed):
                 self.result += xssescape('<%s>' % tag)
         else:
+            self.in_disallowed.append(False)
             bt = '<' + tag
             if tag in self.allowed_attributes:
                 attrs = dict(attrs)
@@ -119,6 +120,7 @@ class XssCleaner(HTMLParser):
                     else:
                         bt += ' %s=%s' % (xssescape(attribute),
                                           quoteattr(attrs[attribute]))
+            # deal with <a> without href and <img> without src
             if bt == '<a' or bt == '<img':
                 return
             if tag in self.requires_no_close:
@@ -129,10 +131,9 @@ class XssCleaner(HTMLParser):
 
     def handle_endtag(self, tag):
         bracketed = '</%s>' % tag
+        self.in_disallowed.pop()
         if tag not in self.permitted_tags:
-            if self.strip_disallowed:
-                self.in_disallowed = False
-            else:
+            if (not self.strip_disallowed):
                 self.result += xssescape(bracketed)
         elif tag in self.open_tags:
             self.result += bracketed
@@ -143,10 +144,13 @@ class XssCleaner(HTMLParser):
         Accepts relative, absolute, and mailto urls
         """
 
-        parsed = urlparse(url)
-        return (parsed[0] in self.allowed_schemes and '.' in parsed[1]) \
-            or (parsed[0] in self.allowed_schemes and '@' in parsed[2]) \
-            or (parsed[0] == '' and parsed[2].startswith('/'))
+        if url.startswith('#'):
+            return True
+        else:
+            parsed = urlparse(url)
+            return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or
+                    (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or
+                    (parsed[0] == '' and parsed[2].startswith('/')))
 
     def strip(self, rawstring, escape=True):
         """

gluon/sqlhtml.py

@@ -29,7 +29,7 @@ from gluon.html import URL, FIELDSET, P, DEFAULT_PASSWORD_DISPLAY
 from pydal.base import DEFAULT
 from pydal.objects import Table, Row, Expression, Field
 from pydal.adapters.base import CALLABLETYPES
-from pydal.helpers.methods import smart_query, bar_encode
+from pydal.helpers.methods import smart_query, bar_encode,  _repr_ref
 from pydal.helpers.classes import Reference, SQLCustomType
 from gluon.storage import Storage
 from gluon.utils import md5_hash
@@ -71,6 +71,26 @@ def represent(field, value, record):
     else:
         raise RuntimeError("field representation must take 1 or 2 args")
 
+class CacheRepresenter(object):
+    def __init__(self):
+        self.cache = {}
+    def __call__(self, field, value, row):
+        cache = self.cache
+        if field not in cache:
+            cache[field] = {}
+        try:
+            nvalue = cache[field][value]
+        except KeyError:
+            try:
+                nvalue = field.represent(value, row)
+            except KeyError:
+                try:
+                    nvalue = field.represent(value, row[field.tablename])
+                except KeyError:
+                    nvalue = None
+            if isinstance(field, _repr_ref):
+                cache[field][value] = nvalue
+        return nvalue
 
 def safe_int(x):
     try:
@@ -626,13 +646,12 @@ class AutocompleteWidget(object):
     def __init__(self, request, field, id_field=None, db=None,
                  orderby=None, limitby=(0, 10), distinct=False,
                  keyword='_autocomplete_%(tablename)s_%(fieldname)s',
-                 min_length=2, help_fields=None, help_string=None):
+                 min_length=2, help_fields=None, help_string=None, at_beginning = True):
 
         self.help_fields = help_fields or []
         self.help_string = help_string
         if self.help_fields and not self.help_string:
-            self.help_string = ' '.join('%%(%s)s' % f.name
-                                        for f in self.help_fields)
+            self.help_string = ' '.join('%%(%s)s' % f.name for f in self.help_fields)
 
         self.request = request
         self.keyword = keyword % dict(tablename=field.tablename,
@@ -642,6 +661,7 @@ class AutocompleteWidget(object):
         self.limitby = limitby
         self.distinct = distinct
         self.min_length = min_length
+        self.at_beginning = at_beginning
         self.fields = [field]
         if id_field:
             self.is_reference = True
@@ -659,8 +679,10 @@ class AutocompleteWidget(object):
             field = self.fields[0]
             if settings and settings.global_settings.web2py_runtime_gae:
                 rows = self.db(field.__ge__(self.request.vars[self.keyword]) & field.__lt__(self.request.vars[self.keyword] + u'\ufffd')).select(orderby=self.orderby, limitby=self.limitby, *(self.fields+self.help_fields))
-            else:
+            elif self.at_beginning:
                 rows = self.db(field.like(self.request.vars[self.keyword] + '%', case_sensitive=False)).select(orderby=self.orderby, limitby=self.limitby, distinct=self.distinct, *(self.fields+self.help_fields))
+            else:
+                rows = self.db(field.contains(self.request.vars[self.keyword], case_sensitive=False)).select(orderby=self.orderby, limitby=self.limitby, distinct=self.distinct, *(self.fields+self.help_fields))
             if rows:
                 if self.is_reference:
                     id_field = self.fields[1]
@@ -859,14 +881,14 @@ def formstyle_bootstrap3_stacked(form, fields):
                 label = ''
             elif isinstance(controls, (SELECT, TEXTAREA)):
                 controls.add_class('form-control')
-                            
+
         elif isinstance(controls, SPAN):
             _controls = P(controls.components)
 
         elif isinstance(controls, UL):
             for e in controls.elements("input"):
                 e.add_class('form-control')
-                
+
         if isinstance(label, LABEL):
             label['_class'] = 'control-label'
 
@@ -909,9 +931,9 @@ def formstyle_bootstrap3_inline_factory(col_label_size=3):
                     label = ''
                 elif isinstance(controls, (SELECT, TEXTAREA)):
                     controls.add_class('form-control')
-                
+
             elif isinstance(controls, SPAN):
-                _controls = P(controls.components, 
+                _controls = P(controls.components,
                               _class="form-control-static %s" % col_class)
             elif isinstance(controls, UL):
                 for e in controls.elements("input"):
@@ -1126,7 +1148,8 @@ class SQLFORM(FORM):
         extra_fields = extra_fields or []
         self.extra_fields = {}
         for extra_field in extra_fields:
-            self.fields.append(extra_field.name)
+            if not extra_field.name in self.fields:
+                self.fields.append(extra_field.name)
             self.extra_fields[extra_field.name] = extra_field
             extra_field.db = table._db
             extra_field.table = table
@@ -1678,7 +1701,7 @@ class SQLFORM(FORM):
                         self.vars.update(pk)
                     else:
                         ret = False
-            else:
+            elif self.table._db._uri:
                 if record_id:
                     self.vars.id = self.record[self.id_field_name]
                     if fields:
@@ -1691,6 +1714,7 @@ class SQLFORM(FORM):
 
     AUTOTYPES = {
         type(''): ('string', None),
+        type(u''): ('string',None),
         type(True): ('boolean', None),
         type(1): ('integer', IS_INT_IN_RANGE(-1e12, +1e12)),
         type(1.0): ('double', IS_FLOAT_IN_RANGE()),
@@ -1755,10 +1779,16 @@ class SQLFORM(FORM):
             keywords = keywords[0]
             request.vars.keywords = keywords
         key = keywords.strip()
-        if key and ' ' not in key and not '"' in key and not "'" in key:
+        if key and not '"' in key:
             SEARCHABLE_TYPES = ('string', 'text', 'list:string')
-            parts = [field.contains(
-                key) for field in fields if field.type in SEARCHABLE_TYPES]
+            sfields = [field for field in fields if field.type in SEARCHABLE_TYPES]
+            if settings.global_settings.web2py_runtime_gae:
+                return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields])
+            else:
+                return reduce(lambda a,b:a&b,[
+                        reduce(lambda a,b: a|b, [
+                                field.contains(k) for field in sfields]
+                               ) for k in key.split()])
 
             # from https://groups.google.com/forum/#!topic/web2py/hKe6lI25Bv4
             # needs testing...
@@ -1772,10 +1802,6 @@ class SQLFORM(FORM):
             #        filters.append(reduce(lambda a, b: (a & b), all_words_filters))
             #parts = filters
 
-        else:
-            parts = None
-        if parts:
-            return reduce(lambda a, b: a | b, parts)
         else:
             return smart_query(fields, key)
 
@@ -1838,15 +1864,19 @@ class SQLFORM(FORM):
                 operators = SELECT(*[OPTION(T(option), _value=option) for option in options], _class='form-control')
                 _id = "%s_%s" % (value_id, name)
                 if field_type in ['boolean', 'double', 'time', 'integer']:
-                    value_input = SQLFORM.widgets[field_type].widget(field, field.default, _id=_id, _class='form-control')
+                    widget_ = SQLFORM.widgets[field_type]
+                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control')
                 elif field_type == 'date':
-                    iso_format = {'_data-w2p_date_format' : '%Y-%m-%d'}
-                    value_input = SQLFORM.widgets.date.widget(field, field.default, _id=_id, _class='form-control', **iso_format)
+                    iso_format = {'_data-w2p_date_format': '%Y-%m-%d'}
+                    widget_ = SQLFORM.widgets.date
+                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control', **iso_format)
                 elif field_type == 'datetime':
-                    iso_format = {'_data-w2p_datetime_format' : '%Y-%m-%d %H:%M:%S'}
-                    value_input = SQLFORM.widgets.datetime.widget(field, field.default, _id=_id, _class='form-control', **iso_format)
+                    iso_format = {'_data-w2p_datetime_format': '%Y-%m-%d %H:%M:%S'}
+                    widget_ = SQLFORM.widgets.datetime
+                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control', **iso_format)
                 elif (field_type.startswith('reference ') or
                       field_type.startswith('list:reference ')) and \
+                      hasattr(field.requires, 'options') or \
                       hasattr(field.requires, 'options'):
                     value_input = SELECT(
                         *[OPTION(v, _value=k)
@@ -1856,7 +1886,8 @@ class SQLFORM(FORM):
                 elif field_type.startswith('reference ') or \
                      field_type.startswith('list:integer') or \
                      field_type.startswith('list:reference '):
-                    value_input = SQLFORM.widgets.integer.widget(field, field.default, _id=_id, _class='form-control')
+                    widget_ = SQLFORM.widgets.integer
+                    value_input = widget_.widget(field, field.default, _id=_id, _class=widget_._class + ' form-control')
                 else:
                     value_input = INPUT(
                         _type='text', _id=_id,
@@ -2106,10 +2137,8 @@ class SQLFORM(FORM):
         # - url has valid signature (vars are not signed, only path_info)
         # = url does not contain 'create','delete','edit' (readonly)
         if user_signature:
-            if not (
-                '/'.join(str(a) for a in args) == '/'.join(request.args) or
-                URL.verify(request, user_signature=user_signature,
-                           hash_vars=False) or
+            if not ('/'.join(map(str,args)) == '/'.join(map(str,request.args)) or
+                    URL.verify(request, user_signature=user_signature, hash_vars=False) or
                     (request.args(len(args)) == 'view' and not logged)):
                 session.flash = T('not authorized')
                 redirect(referrer)
@@ -2662,7 +2691,7 @@ class SQLFORM(FORM):
             htmltable = TABLE(COLGROUP(*cols), THEAD(head))
             tbody = TBODY()
             numrec = 0
-            repr_cache = {}
+            repr_cache = CacheRepresenter()
             for row in rows:
                 trcols = []
                 id = row[field_id]
@@ -2675,31 +2704,20 @@ class SQLFORM(FORM):
                         continue
                     if field.type == 'blob':
                         continue
-                    value = row[str(field)]
+                    if isinstance(field, Field.Virtual) and field.tablename in row:
+                        value = dbset.db[field.tablename][row[field.tablename][field_id]][field.name]
+                    else:
+                        value = row[str(field)]
                     maxlength = maxtextlengths.get(str(field), maxtextlength)
                     if field.represent:
                         if field.type.startswith('reference'):
-                            if field not in repr_cache:
-                                repr_cache[field] = {}
-                            try:
-                                nvalue = repr_cache[field][value]
-                            except KeyError:
-                                try:
-                                    nvalue = field.represent(value, row)
-                                except KeyError:
-                                    try:
-                                        nvalue = field.represent(
-                                            value, row[field.tablename])
-                                    except KeyError:
-                                        nvalue = None
-                                repr_cache[field][value] = nvalue
+                            nvalue = repr_cache(field, value, row)
                         else:
                             try:
                                 nvalue = field.represent(value, row)
                             except KeyError:
                                 try:
-                                    nvalue = field.represent(
-                                        value, row[field.tablename])
+                                    nvalue = field.represent(value, row[field.tablename])
                                 except KeyError:
                                     nvalue = None
                         value = nvalue

gluon/tests/coverage.ini

@@ -26,6 +26,7 @@ exclude_lines =
 ignore_errors = True
 omit = gluon/contrib/*
        gluon/tests/*
+       gluon/packages/*
 
 [html]
 directory = coverage_html_report

gluon/tools.py

@@ -765,10 +765,13 @@ class Mail(object):
         result = {}
         try:
             if self.settings.server == 'logging':
-                logger.warn('email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' %
-                            ('-' * 40, sender,
-                                ', '.join(to), subject,
-                                text or html, '-' * 40))
+                entry = 'email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' % \
+                    ('-' * 40, sender, ', '.join(to), subject, text or html, '-' * 40)
+                logger.warn(entry)
+            elif self.settings.server.startswith('logging:'):
+                entry = 'email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' % \
+                    ('-' * 40, sender, ', '.join(to), subject, text or html, '-' * 40)
+                open(self.settings.server[8:], 'a').write(entry)
             elif self.settings.server == 'gae':
                 xcc = dict()
                 if cc:
@@ -1146,6 +1149,7 @@ class Auth(object):
         reset_password_requires_verification=False,
         registration_requires_verification=False,
         registration_requires_approval=False,
+        bulk_register_enabled=False,
         login_after_registration=False,
         login_after_password_change=True,
         alternate_requires_registration=False,
@@ -1179,6 +1183,7 @@ class Auth(object):
         table_permission_name='auth_permission',
         table_event_name='auth_event',
         table_cas_name='auth_cas',
+        table_token_name='auth_token',
         table_user=None,
         table_group=None,
         table_membership=None,
@@ -1247,6 +1252,8 @@ class Auth(object):
         retrieve_password_subject='Password retrieve',
         reset_password='Click on the link %(link)s to reset your password',
         reset_password_subject='Password reset',
+        bulk_invite_subject='Invitation to join%(site)s',
+        bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete the process',
         invalid_reset_password='Invalid reset password',
         profile_updated='Profile updated',
         new_password='New password',
@@ -1460,6 +1467,7 @@ class Auth(object):
         settings.update(Auth.default_settings)
         settings.update(
             cas_domains=[request.env.http_host],
+            enable_tokens=False,
             cas_provider=cas_provider,
             cas_actions=dict(login='login',
                              validate='validate',
@@ -1499,6 +1507,8 @@ class Auth(object):
             change_password_onvalidation = [],
             change_password_onaccept = [],
             retrieve_password_onvalidation = [],
+            request_reset_password_onvalidation = [],
+            request_reset_password_onaccept = [],
             reset_password_onvalidation = [],
             reset_password_onaccept = [],
             hmac_key = hmac_key,
@@ -1534,6 +1544,12 @@ class Auth(object):
         next = current.request.vars._next
         if isinstance(next, (list, tuple)):
             next = next[0]
+        if next and self.settings.prevent_open_redirect_attacks:
+            # Prevent an attacker from adding an arbitrary url after the
+            # _next variable in the request.
+            items = next.split('/')
+            if '//' in next and items[2] != current.request.env.http_host:
+                next = None            
         return next
 
     def _get_user_id(self):
@@ -1560,6 +1576,9 @@ class Auth(object):
     def table_cas(self):
         return self.db[self.settings.table_cas_name]
 
+    def table_token(self):
+        return self.db[self.settings.table_token_name]
+
     def _HTTP(self, *a, **b):
         """
         only used in lambda: self._HTTP(404)
@@ -1587,7 +1606,8 @@ class Auth(object):
                        'retrieve_username', 'retrieve_password',
                        'reset_password', 'request_reset_password',
                        'change_password', 'profile', 'groups',
-                       'impersonate', 'not_authorized'):
+                       'impersonate', 'not_authorized', 'confirm_registration', 
+                       'bulk_register','manage_tokens'):
             if len(request.args) >= 2 and args[0] == 'impersonate':
                 return getattr(self, args[0])(request.args[1])
             else:
@@ -1914,7 +1934,7 @@ class Auth(object):
                   writable=False, readable=False,
                   label=T('Modified By'),  ondelete=ondelete))
 
-    def define_tables(self, username=None, signature=None,
+    def define_tables(self, username=None, signature=None, enable_tokens=False,
                       migrate=None, fake_migrate=None):
         """
         To be called unless tables are defined manually
@@ -1941,6 +1961,7 @@ class Auth(object):
             username = settings.use_username
         else:
             settings.use_username = username
+        settings.enable_tokens = enable_tokens
         if not self.signature:
             self.define_signature()
         if signature == True:
@@ -2124,6 +2145,21 @@ class Auth(object):
                         migrate=self.__get_migrate(
                             settings.table_cas_name, migrate),
                         fake_migrate=fake_migrate))
+        if settings.enable_tokens:
+            extra_fields = settings.extra_fields.get(
+                settings.table_token_name, []) + signature_list
+            if not settings.table_token_name in db.tables:
+                db.define_table(
+                    settings.table_token_name,
+                    Field('user_id', reference_table_user, default=None,
+                          label=self.messages.label_user_id),
+                    Field('expires_on', 'datetime', default=datetime.datetime(2999,12,31)),
+                    Field('token',writable=False,default=web2py_uuid(),unique=True),
+                    *extra_fields,
+                    **dict(
+                        migrate=self.__get_migrate(
+                            settings.table_token_name, migrate),
+                        fake_migrate=fake_migrate))
         if not db._lazy_tables:
             settings.table_user = db[settings.table_user_name]
             settings.table_group = db[settings.table_group_name]
@@ -2323,8 +2359,8 @@ class Auth(object):
             # user not in database try other login methods
             for login_method in self.settings.login_methods:
                 if login_method != self and login_method(username, password):
-                    self.user = username
-                    return username
+                    self.user = user
+                    return user
         return False
 
     def register_bare(self, **fields):
@@ -2333,14 +2369,16 @@ class Auth(object):
         and a raw password.
         """
         settings = self._get_login_settings()
-        if not fields.get(settings.passfield):
-            raise ValueError("register_bare: " +
-                             "password not provided or invalid")
-        elif not fields.get(settings.userfield):
+        # users can register_bare even if no password is provided, 
+        # in this case they will have to reset their password to login
+        if fields.get(settings.passfield):
+            fields[settings.passfield] = \
+                settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
+        if not fields.get(settings.userfield):
             raise ValueError("register_bare: " +
                              "userfield not provided or invalid")
-        fields[settings.passfield] = settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
-        user = self.get_or_create_user(fields, login=False, get=False, update_fields=self.settings.update_fields)
+        user = self.get_or_create_user(fields, login=False, get=False, 
+                                       update_fields=self.settings.update_fields)
         if not user:
             # get or create did not create a user (it ignores duplicate records)
             return False
@@ -2484,10 +2522,6 @@ class Auth(object):
 
         ### use session for federated login
         snext = self.get_vars_next()
-        if snext and self.settings.prevent_open_redirect_attacks:
-            items = snext.split('/')
-            if '//' in snext and items[2] != request.env.http_host:
-                snext = None
 
         if snext:
             session._auth_next = snext
@@ -2860,14 +2894,18 @@ class Auth(object):
 
         passfield = self.settings.password_field
         formstyle = self.settings.formstyle
-        if self.settings.register_verify_password:
+        if self.settings.register_verify_password:            
+            if self.settings.register_fields == None:
+                self.settings.register_fields = [f.name for f in table_user if f.writable]
+                k = self.settings.register_fields.index("password")
+                self.settings.register_fields.insert(k+1, "password_two")
             extra_fields = [
                 Field("password_two", "password", requires=IS_EQUAL_TO(
                         request.post_vars.get(passfield, None),
                         error_message=self.messages.mismatched_password),
                         label=current.T("Confirm Password"))]
         else:
-            extra_fields = []
+            extra_fields = []        
         form = SQLFORM(table_user,
                        fields=self.settings.register_fields,
                        hidden=dict(_next=next),
@@ -3136,6 +3174,147 @@ class Auth(object):
         table_user.email.requires = old_requires
         return form
 
+    def confirm_registration(
+        self,
+        next=DEFAULT,
+        onvalidation=DEFAULT,
+        onaccept=DEFAULT,
+        log=DEFAULT,
+        ):
+        """
+        Returns a form to confirm user registration
+        """
+
+        table_user = self.table_user()
+        request = current.request
+        # response = current.response
+        session = current.session
+
+        if next is DEFAULT:
+            next = self.get_vars_next() or self.settings.reset_password_next
+
+        if self.settings.prevent_password_reset_attacks:
+            key = request.vars.key
+            if not key and len(request.args)>1:
+                key = request.args[-1]
+            if key:
+                session._reset_password_key = key
+                redirect(self.url(args='confirm_registration'))
+            else:
+                key = session._reset_password_key
+        else:
+            key = request.vars.key or getarg(-1)
+        try:
+            t0 = int(key.split('-')[0])
+            if time.time() - t0 > 60 * 60 * 24:
+                raise Exception
+            user = table_user(reset_password_key=key)
+            if not user:
+                raise Exception
+        except Exception as e:
+            session.flash = self.messages.invalid_reset_password
+            redirect(self.url('login', vars=dict(test=e)))
+            redirect(next, client_side=self.settings.client_side)
+        passfield = self.settings.password_field
+        form = SQLFORM.factory(
+            Field('first_name',
+                  label='First Name',
+                  required=True),
+            Field('last_name',
+                  label='Last Name',
+                  required=True),
+            Field('new_password', 'password',
+                  label=self.messages.new_password,
+                  requires=self.table_user()[passfield].requires),
+            Field('new_password2', 'password',
+                  label=self.messages.verify_password,
+                  requires=[IS_EXPR(
+                        'value==%s' % repr(request.vars.new_password),
+                        self.messages.mismatched_password)]),
+            submit_button='Confirm Registration',
+            hidden=dict(_next=next),
+            formstyle=self.settings.formstyle,
+            separator=self.settings.label_separator
+        )
+        if form.process().accepted:
+            user.update_record(
+                **{passfield: str(form.vars.new_password),
+                   'first_name': str(form.vars.first_name),
+                   'last_name': str(form.vars.last_name),
+                   'registration_key': '',
+                   'reset_password_key': ''})
+            session.flash = self.messages.password_changed
+            if self.settings.login_after_password_change:
+                self.login_user(user)
+            redirect(next, client_side=self.settings.client_side)
+        return form
+
+    def email_registration(self, subject, body, user):
+        """
+        Sends and email invitation to a user informing they have been registered with the application
+        """
+        reset_password_key = str(int(time.time())) + '-' + web2py_uuid()
+        link = self.url(self.settings.function,
+                        args=('confirm_registration',), vars={'key': reset_password_key},
+                        scheme=True)
+        d = dict(user)
+        d.update(dict(key=reset_password_key, link=link, site=current.request.env.http_host))
+        if self.settings.mailer and self.settings.mailer.send(
+            to=user.email,
+            subject=subject % d,
+            message=body % d):
+            user.update_record(reset_password_key=reset_password_key)
+            return True
+        return False
+
+    def bulk_register(self, max_emails=100):
+        """
+        Creates a form for ther user to send invites to other users to join
+        """
+        if not self.user:
+            redirect(self.settings.login_url)
+        if not self.setting.bulk_register_enabled:
+            return HTTP(404)
+
+        form = SQLFORM.factory(
+            Field('subject','string',default=self.messages.bulk_invite_subject,requires=IS_NOT_EMPTY()),
+            Field('emails','text',requires=IS_NOT_EMPTY()),
+            Field('message','text',default=self.messages.bulk_invite_body,requires=IS_NOT_EMPTY()),
+            formstyle=self.settings.formstyle)
+
+        if form.process().accepted:
+            emails = re.compile('[^\s\'"@<>,;:]+\@[^\s\'"@<>,;:]+').findall(form.vars.emails)
+            # send the invitations            
+            emails_sent = []
+            emails_fail = []
+            emails_exist = []
+            for email in emails[:max_emails]:
+                if self.table_user()(email=email):
+                    emails_exist.append(email)
+                else:
+                    user = self.register_bare(email=email)
+                    if self.email_registration(form.vars.subject, form.vars.message, user):
+                        emails_sent.append(email)
+                    else:
+                        emails_fail.append(email)
+            emails_fail += emails[max_emails:]
+            form = DIV(H4('Emails sent'),UL(*[A(x,_href='mailto:'+x) for x in emails_sent]),
+                       H4('Emails failed'),UL(*[A(x,_href='mailto:'+x) for x in emails_fail]),
+                       H4('Emails existing'),UL(*[A(x,_href='mailto:'+x) for x in emails_exist]))
+        return form
+
+    def manage_tokens(self):
+        if not self.user:
+            redirect(self.settings.login_url)        
+        table_token =self.table_token()
+        table_token.user_id.writable = False
+        table_token.user_id.default = self.user.id
+        table_token.token.writable = False
+        if current.request.args(1) == 'new':
+            table_token.token.readable = False
+        form = SQLFORM.grid(table_token, args=['manage_tokens'])
+        return form
+
     def reset_password(self,
                        next=DEFAULT,
                        onvalidation=DEFAULT,
@@ -3173,6 +3352,12 @@ class Auth(object):
         except Exception:
             session.flash = self.messages.invalid_reset_password
             redirect(next, client_side=self.settings.client_side)
+
+        if onvalidation is DEFAULT:
+            onvalidation = self.settings.reset_password_onvalidation
+        if onaccept is DEFAULT:
+            onaccept = self.settings.reset_password_onaccept
+
         passfield = self.settings.password_field
         form = SQLFORM.factory(
             Field('new_password', 'password',
@@ -3188,7 +3373,7 @@ class Auth(object):
             formstyle=self.settings.formstyle,
             separator=self.settings.label_separator
         )
-        if form.accepts(request, session,
+        if form.accepts(request, session, onvalidation=onvalidation,
                         hideerror=self.settings.hideerror):
             user.update_record(
                 **{passfield: str(form.vars.new_password),
@@ -3197,6 +3382,7 @@ class Auth(object):
             session.flash = self.messages.password_changed
             if self.settings.login_after_password_change:
                 self.login_user(user)
+            callback(onaccept, form)
             redirect(next, client_side=self.settings.client_side)
         return form
 
@@ -3222,9 +3408,9 @@ class Auth(object):
             response.flash = self.messages.function_disabled
             return ''
         if onvalidation is DEFAULT:
-            onvalidation = self.settings.reset_password_onvalidation
+            onvalidation = self.settings.request_reset_password_onvalidation
         if onaccept is DEFAULT:
-            onaccept = self.settings.reset_password_onaccept
+            onaccept = self.settings.request_reset_password_onaccept
         if log is DEFAULT:
             log = self.messages['reset_password_log']
         userfield = self.settings.login_userfield or 'username' \
@@ -3553,7 +3739,12 @@ class Auth(object):
 
                 basic_allowed, basic_accepted, user = self.basic()
                 user = user or self.user
-                if requires_login:
+
+                login_required = requires_login
+                if callable(login_required):
+                    login_required = login_required()
+
+                if login_required:
                     if not user:
                         if current.request.ajax:
                             raise HTTP(401, self.messages.ajax_failed_authentication)
@@ -3594,6 +3785,26 @@ class Auth(object):
         """
         return self.requires(True, otherwise=otherwise)
 
+    def requires_login_or_token(self, otherwise=None):
+        if self.settings.enable_tokens == True:
+            user = None
+            request = current.request
+            token = request.env.http_web2py_user_token or request.vars._token
+            table_token = self.table_token()
+            table_user = self.table_user()
+            from gluon.settings import global_settings
+            if global_settings.web2py_runtime_gae:
+                row = table_token(token=token)
+                if row:
+                    user = table_user(row.user_id)
+            else:
+                row = self.db(table_token.token==token)(table_user.id==table_token.user_id).select().first()
+                if row:
+                    user = row[table_user._tablename]
+            if user:
+                self.login_user(user)
+        return self.requires(True, otherwise=otherwise)
+
     def requires_membership(self, role=None, group_id=None, otherwise=None):
         """
         Decorator that prevents access to action if not logged in or
@@ -5317,11 +5528,12 @@ def completion(callback):
     return _completion
 
 
-def prettydate(d, T=lambda x: x):
+def prettydate(d, T=lambda x: x, utc=False):
+    now = datetime.datetime.utcnow() if utc else datetime.datetime.now()
     if isinstance(d, datetime.datetime):
-        dt = datetime.datetime.now() - d
+        dt = now - d
     elif isinstance(d, datetime.date):
-        dt = datetime.date.today() - d
+        dt = now.date() - d
     elif not d:
         return ''
     else:
@@ -6192,7 +6404,7 @@ class Wiki(object):
             count = db.wiki_tag.wiki_page.count()
             fields = [db.wiki_page.id, db.wiki_page.slug,
                       db.wiki_page.title, db.wiki_page.tags,
-                      db.wiki_page.can_read]
+                      db.wiki_page.can_read, db.wiki+page.can_edit]
             if preview:
                 fields.append(db.wiki_page.body)
             if query is None:

gluon/validators.py

@@ -200,8 +200,11 @@ class IS_MATCH(Validator):
         self.is_unicode = is_unicode
 
     def __call__(self, value):
-        if self.is_unicode and not isinstance(value, unicode):
-            match = self.regex.search(str(value).decode('utf8'))
+        if self.is_unicode:
+            if isinstance(value,unicode):
+                match = self.regex.search(value)
+            else:
+                match = self.regex.search(str(value).decode('utf8'))
         else:
             match = self.regex.search(str(value))
         if match is not None:
@@ -3479,7 +3482,7 @@ class IS_IPV6(Validator):
             from gluon.contrib import ipaddr as ipaddress
 
         try:
-            ip = ipaddress.IPv6Address(value)
+            ip = ipaddress.IPv6Address(value.decode('utf-8'))
             ok = True
         except ipaddress.AddressValueError:
             return (value, translate(self.error_message))
@@ -3491,7 +3494,7 @@ class IS_IPV6(Validator):
                 self.subnets = [self.subnets]
             for network in self.subnets:
                 try:
-                    ipnet = ipaddress.IPv6Network(network)
+                    ipnet = ipaddress.IPv6Network(network.decode('utf-8'))
                 except (ipaddress.NetmaskValueError, ipaddress.AddressValueError):
                     return (value, translate('invalid subnet provided'))
                 if ip in ipnet:
@@ -3700,20 +3703,22 @@ class IS_IPADDRESS(Validator):
 
     def __call__(self, value):
         try:
-            import ipaddress
+            from ipaddress import ip_address as IPAddress
+            from ipaddress import IPv6Address, IPv4Address
         except ImportError:
-            from gluon.contrib import ipaddr as ipaddress
+            from gluon.contrib.ipaddr import (IPAddress, IPv4Address,
+                                              IPv6Address)
 
         try:
-            ip = ipaddress.IPAddress(value)
-        except ValueError, e:
+            ip = IPAddress(value.decode('utf-8'))
+        except ValueError:
             return (value, translate(self.error_message))
 
-        if self.is_ipv4 and isinstance(ip, ipaddress.IPv6Address):
+        if self.is_ipv4 and isinstance(ip, IPv6Address):
             retval = (value, translate(self.error_message))
-        elif self.is_ipv6 and isinstance(ip, ipaddress.IPv4Address):
+        elif self.is_ipv6 and isinstance(ip, IPv4Address):
             retval = (value, translate(self.error_message))
-        elif self.is_ipv4 or isinstance(ip, ipaddress.IPv4Address):
+        elif self.is_ipv4 or isinstance(ip, IPv4Address):
             retval = IS_IPV4(
                 minip=self.minip,
                 maxip=self.maxip,
@@ -3723,7 +3728,7 @@ class IS_IPADDRESS(Validator):
                 is_automatic=self.is_automatic,
                 error_message=self.error_message
                 )(value)
-        elif self.is_ipv6 or isinstance(ip, ipaddress.IPv6Address):
+        elif self.is_ipv6 or isinstance(ip, IPv6Address):
             retval = IS_IPV6(
                 is_private=self.is_private,
                 is_link_local=self.is_link_local,

gluon/widget.py

@@ -1058,6 +1058,11 @@ def start_schedulers(options):
         print 'starting single-scheduler for "%s"...' % app_
         run(app_, True, True, None, False, code)
         return
+
+    # Work around OS X problem: http://bugs.python.org/issue9405
+    import urllib
+    urllib.getproxies()
+
     for app in apps:
         app_, code = get_code_for_scheduler(app, options)
         if not app_:

scripts/setup-web2py-centos7.sh

@@ -195,7 +195,7 @@ NameVirtualHost *:80
 NameVirtualHost *:443
 
 <VirtualHost *:80>
-  WSGIDaemonProcess web2py user=apache group=apache processes=1 threads=1
+  WSGIDaemonProcess web2py user=apache group=apache
   WSGIProcessGroup web2py
   WSGIScriptAlias / /opt/web-apps/web2py/wsgihandler.py
   WSGIPassAuthorization On

scripts/setup-web2py-fedora-ami.sh

@@ -299,7 +299,7 @@ NameVirtualHost *:80
 NameVirtualHost *:443
 
 <VirtualHost *:80>
-  WSGIDaemonProcess web2py user=apache group=apache processes=1 threads=1
+  WSGIDaemonProcess web2py user=apache group=apache
   WSGIProcessGroup web2py
   WSGIScriptAlias / /opt/web-apps/web2py/wsgihandler.py
 

scripts/setup-web2py-fedora.sh

@@ -301,7 +301,7 @@ NameVirtualHost *:80
 NameVirtualHost *:443
 
 <VirtualHost *:80>
-  WSGIDaemonProcess web2py user=apache group=apache processes=1 threads=1
+  WSGIDaemonProcess web2py user=apache group=apache
   WSGIProcessGroup web2py
   WSGIScriptAlias / /opt/web-apps/web2py/wsgihandler.py
   WSGIPassAuthorization On

scripts/setup-web2py-nginx-uwsgi-centos7.sh

@@ -0,0 +1,235 @@
+#!/bin/bash
+
+# This script will install web2py with nginx+uwsgi on centos 7
+# This script is based on excellent tutorial by Justin Ellingwood on 
+# https://www.digitalocean.com/community/tutorials/how-to-deploy-web2py-python-applications-with-uwsgi-and-nginx-on-centos-7
+
+#
+# Phase 1: First, let's ask a few things
+#
+
+read -p "Enter username under which web2py will be installed [web2py]: " USERNAME
+USERNAME=${USERNAME:-web2py}
+
+read -p "Enter path where web2py will be installed [/opt/web2py_apps]: " WEB2PY_PATH
+WEB2PY_PATH=${WEB2PY_PATH:-/opt/web2py_apps}
+
+read -p "Web2py subdirectory will be called: [web2py]: " WEB2PY_APP
+WEB2PY_APP=${WEB2PY_APP:-web2py}
+
+read -p "Enter your web2py admin password: " WEB2PY_PASS
+
+read -p "Enter your domain name: " YOUR_SERVER_DOMAIN
+
+#  open new user
+useradd -d $WEB2PY_PATH $USERNAME
+
+# if it's not already open, let's create a directory for web2py
+mkdir -p $WEB2PY_PATH
+
+# now let's create a self signed certificate
+cd $WEB2PY_PATH
+
+openssl req -x509 -new -newkey rsa:4096 -days 3652 -nodes -keyout $WEB2PY_APP.key -out $WEB2PY_APP.crt
+
+#
+# phase 2: That was all the input that we needed so let's install the components
+#
+
+echo "Installing necessary components"
+
+# Verify packages are up to date
+yum -y upgrade
+
+# Install required packages
+yum install -y epel-release
+yum install -y python-devel python-pip gcc nginx wget unzip python-psycopg2 MySQL-python
+
+# download and unzip web2py
+
+echo "Downloading web2py"
+
+cd $WEB2PY_PATH
+wget http://web2py.com/examples/static/web2py_src.zip
+unzip web2py_src.zip
+rm web2py_src.zip
+
+# preparing wsgihandler
+chown -R $USERNAME.$USERNAME $WEB2PY_PATH/$WEB2PY_APP
+mv $WEB2PY_PATH/$WEB2PY_APP/handlers/wsgihandler.py $WEB2PY_PATH/$WEB2PY_APP
+
+# now let's install uwsgi
+
+pip install uwsgi
+
+# preparing directories
+mkdir -p /etc/uwsgi/sites
+mkdir -p /var/log/uwsgi
+mkdir -p /etc/nginx/ssl/
+
+#
+#  Phase 3: Ok, everything is installed now so we'll configure things
+#
+
+# Create configuration file for uwsgi in /etc/uwsgi/$WEB2PY_APP.ini
+echo '[uwsgi]
+chdir = WEB2PY_PATH_PLACEHOLDER/WEB2PY_APP_PLACEHOLDER
+module = wsgihandler:application
+
+master = true
+processes = 5
+
+uid = USERNAME_PLACEHOLDER
+socket = /run/uwsgi/WEB2PY_APP_PLACEHOLDER.sock
+chown-socket = USERNAME_PLACEHOLDER:nginx
+chmod-socket = 660
+vacuum = true
+' >/etc/uwsgi/sites/$WEB2PY_APP.ini
+
+sed -i "s@WEB2PY_PATH_PLACEHOLDER@$WEB2PY_PATH@" /etc/uwsgi/sites/$WEB2PY_APP.ini
+sed -i "s@WEB2PY_APP_PLACEHOLDER@$WEB2PY_APP@" /etc/uwsgi/sites/$WEB2PY_APP.ini
+sed -i "s@USERNAME_PLACEHOLDER@$USERNAME@" /etc/uwsgi/sites/$WEB2PY_APP.ini
+
+# Create a daemon configuration file for uwsgi
+cat  > /etc/systemd/system/uwsgi.service <<EOF
+[Unit]
+Description=uWSGI Emperor service
+
+[Service]
+ExecStartPre=/usr/bin/bash -c 'mkdir -p /run/uwsgi; chown USERNAME_PLACEHOLDER:nginx /run/uwsgi'
+ExecStart=/usr/bin/uwsgi --emperor /etc/uwsgi/sites
+Restart=always
+KillSignal=SIGQUIT
+Type=notify
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+sed -i "s@USERNAME_PLACEHOLDER@$USERNAME@" /etc/systemd/system/uwsgi.service
+
+#chmod 777 /etc/systemd/system/uwsgi.service
+
+# create a nginx configuration file
+cat  > /etc/nginx/nginx.conf <<EOF
+# For more information on configuration, see:
+#   * Official English Documentation: http://nginx.org/en/docs/
+#   * Official Russian Documentation: http://nginx.org/ru/docs/
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format  main  '\$remote_addr - \$remote_user [\$time_local] "\$request" '
+                      '\$status \$body_bytes_sent "\$http_referer" '
+                      '"\$http_user_agent" "\$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    # Load modular configuration files from the /etc/nginx/conf.d directory.
+    # See http://nginx.org/en/docs/ngx_core_module.html#include
+    # for more information.
+    include /etc/nginx/conf.d/*.conf;
+
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  YOUR_SERVER_DOMAIN_PLACEHOLDER;
+        root         /usr/share/nginx/html;
+
+        # Load configuration files for the default server block.
+        include /etc/nginx/default.d/*.conf;
+
+        location ~* /(\w+)/static/ {
+            root WEB2PY_PATH_PLACEHOLDER/WEB2PY_APP_PLACEHOLDER/applications/;
+        }
+
+        location / {
+            include uwsgi_params;
+            uwsgi_pass unix:/run/uwsgi/WEB2PY_APP_PLACEHOLDER.sock;
+        }
+
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+    
+    server {
+        listen 443;
+        server_name YOUR_SERVER_DOMAIN_PLACEHOLDER;
+        
+        ssl on;
+        ssl_certificate /etc/nginx/ssl/WEB2PY_APP_PLACEHOLDER.crt;
+        ssl_certificate_key /etc/nginx/ssl/WEB2PY_APP_PLACEHOLDER.key;
+        
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
+        ssl_prefer_server_ciphers on;
+        
+        location / {
+            include uwsgi_params;
+            uwsgi_pass unix:/run/uwsgi/WEB2PY_APP_PLACEHOLDER.sock;
+        }
+    }
+}
+EOF
+
+sed -i "s@YOUR_SERVER_DOMAIN_PLACEHOLDER@$YOUR_SERVER_DOMAIN@" /etc/nginx/nginx.conf
+sed -i "s@WEB2PY_PATH_PLACEHOLDER@$WEB2PY_PATH@" /etc/nginx/nginx.conf
+sed -i "s@WEB2PY_APP_PLACEHOLDER@$WEB2PY_APP@" /etc/nginx/nginx.conf
+
+#
+# Phase 4: everything is configured now, just a few final touches
+#
+
+# copying certificates to nginx directory
+mv $WEB2PY_PATH/$WEB2PY_APP.crt* /etc/nginx/ssl
+mv $WEB2PY_PATH/$WEB2PY_APP.key* /etc/nginx/ssl
+
+# creating web2py admin password
+cd $WEB2PY_PATH/$WEB2PY_APP
+python -c "from gluon.main import save_password; save_password('$WEB2PY_PASS',443)"
+chown -R $USERNAME.$USERNAME $WEB2PY_PATH/$WEB2PY_APP
+
+# taking care of permissions
+chmod 700 /etc/nginx/ssl
+usermod -a -G $USERNAME nginx
+chmod 710 $WEB2PY_PATH
+
+# enabling daemons
+systemctl start nginx
+systemctl start uwsgi
+systemctl enable nginx
+systemctl enable uwsgi
+
+# If firewall is active make sure these ports are open
+
+firewall-cmd --zone=public --add-port=80/tcp --permanent
+firewall-cmd --zone=public --add-port=443/tcp --permanent
+firewall-cmd --zone=public --add-port=22/tcp --permanent
+firewall-cmd --reload
+
+echo
+echo 'Web2py is now installed on this server!'
+echo
+

scripts/setup-web2py-ubuntu.sh

@@ -1,7 +1,7 @@
 echo "This script will:
 1) install all modules need to run web2py on Ubuntu 14.04
 2) install web2py in /home/www-data/
-3) create a self signed sll certificate
+3) create a self signed ssl certificate
 4) setup web2py with mod_wsgi
 5) overwrite /etc/apache2/sites-available/default
 6) restart apache.
@@ -84,7 +84,7 @@ openssl x509 -noout -fingerprint -text < /etc/apache2/ssl/self_signed.cert > /et
 echo "rewriting your apache config file to use mod_wsgi"
 echo "================================================="
 echo '
-WSGIDaemonProcess web2py user=www-data group=www-data processes=1 threads=1
+WSGIDaemonProcess web2py user=www-data group=www-data
 
 <VirtualHost *:80>
 

scripts/web2py-scheduler.conf

@@ -0,0 +1,21 @@
+description "web2py task scheduler"
+
+# INSTRUCTIONS:
+# COPY THIS FILE IN:
+#    /etc/init/web2py-scheduler.con
+#
+# To start/stop the scheduler, use
+#   "sudo start web2py-scheduler"
+#   "sudo stop web2py-scheduler"
+#   "sudo status web2py-scheduler"
+#
+# YOU MAY HAVE TO EDIT PATH TO WEB2PY BELOW
+
+start on (local-filesystems and net-device-up IFACE=eth0)
+stop  on shutdown
+
+# Give up if restart occurs 8 times in 60 seconds.                              
+respawn limit 8 60
+
+exec sudo -u www-data python /home/www-data/web2py/web2py.py -K parking > /tmp/scheduler.out
+respawn