fixed compileapp problem in appveyor test (2nd attempt)

Tracking pyDAL v15.12

.travis.yml

@@ -18,6 +18,10 @@ before_script:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install --download-cache $HOME/.pip-cache unittest2; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache coverage; fi;
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache codecov; fi
+  - mysql -e 'create database pydal;'
+  - psql -c 'create database pydal;' -U postgres
+  - psql -c 'create extension postgis;' -U postgres -d pydal;
+  - psql -c 'SHOW SERVER_VERSION' -U postgres
 
 
 script: export COVERAGE_PROCESS_START=gluon/tests/coverage.ini; ./web2py.py --run_system_tests --with_coverage
@@ -28,3 +32,6 @@ after_success:
 
 notifications:
   email: true
+
+addons:
+  postgresql: "9.4"

CHANGELOG

@@ -1,4 +1,18 @@
-## 2.12.1
+## 2.13.1
+
+- added fabfile.py
+- fixed oauth2 renew token, thanks dokime7
+- fixed add_membership, del_membership, add_membership IntegrityError (when auth.enable_record_versioning) 
+- allow passing unicode to template render
+- allow IS_NOT_IN_DB to work with custom primarykey, thanks timmyborg
+- allow HttpOnly cookies
+- french pluralizaiton rules, thanks Mathieu Clabaut
+- fixed bug in redirect to cas service, thanks Fernando González
+- allow deploying to pythonanywhere from the web2py admin that you're running locally, thanks Leonel
+- better tests
+- many more bug fixes
+
+## 2.12.1-3
 
 - security fix: Validate for open redirect everywhere, not just in login()
 - allow to pack invidual apps and selected files as packed exe files

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.12.1-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.13.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

VERSION

@@ -1 +1 @@
-Version 2.12.1-stable+timestamp.2015.08.07.02.10.57
+Version 2.12.3-stable+timestamp.2015.08.18.19.14.07

applications/admin/controllers/appadmin.py

@@ -445,15 +445,16 @@ def ccache():
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
-        for key, value in cache.ram.storage.iteritems():
+        # get ram stats directly from the cache object
-            if isinstance(value, dict):
+        ram_stats = cache.ram.stats[request.application]
-                ram['hits'] = value['hit_total'] - value['misses']
+        ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
-                ram['misses'] = value['misses']
+        ram['misses'] = ram_stats['misses']
         try:
-                    ram['ratio'] = ram['hits'] * 100 / value['hit_total']
+            ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
         except (KeyError, ZeroDivisionError):
             ram['ratio'] = 0
-            else:
+
+        for key, value in cache.ram.storage.iteritems():
             if hp:
                 ram['bytes'] += hp.iso(value[1]).size
                 ram['objects'] += hp.iso(value[1]).count
@@ -464,11 +465,11 @@ def ccache():
 
         for key in cache.disk.storage:
             value = cache.disk.storage[key]
-            if isinstance(value, dict):
+            if isinstance(value[1], dict):
-                disk['hits'] = value['hit_total'] - value['misses']
+                disk['hits'] = value[1]['hit_total'] - value[1]['misses']
-                disk['misses'] = value['misses']
+                disk['misses'] = value[1]['misses']
                 try:
-                    disk['ratio'] = disk['hits'] * 100 / value['hit_total']
+                    disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:

applications/admin/controllers/default.py

@@ -484,9 +484,15 @@ def cleanup():
 
 def compile_app():
     app = get_app()
-    c = app_compile(app, request)
+    c = app_compile(app, request,
+        skip_failed_views = (request.args(1) == 'skip_failed_views'))
     if not c:
         session.flash = T('application compiled')
+    elif isinstance(c, list):
+        session.flash = DIV(*[T('application compiled'), BR(), BR(),
+                              T('WARNING: The following views could not be compiled:'), BR()] +
+                             [CAT(BR(), view) for view in c] +
+                             [BR(), BR(), T('DO NOT use the "Pack compiled" feature.')])
     else:
         session.flash = DIV(T('Cannot compile: there are errors in your app:'),
                             CODE(c))

applications/admin/controllers/pythonanywhere.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+import base64
+import os
+import re
+import gzip
+import tarfile
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+from xmlrpclib import ProtocolError
+from gluon.contrib.simplejsonrpc import ServerProxy
+
+
+def deploy():
+    response.title = T('Deploy to pythonanywhere')
+    return {}
+
+
+def create_account():
+    """ Create a PythonAnywhere account """
+    if not request.vars:
+        raise HTTP(400)
+    
+    if request.vars.username and request.vars.web2py_admin_password:
+        # Check if web2py is already there otherwise we get an error 500 too.
+        client = ServerProxy('https://%(username)s:%(web2py_admin_password)s@%(username)s.pythonanywhere.com/admin/webservices/call/jsonrpc' % request.vars)
+        try:
+            if client.login() is True:
+                return response.json({'status': 'ok'})
+        except ProtocolError as error:
+            pass
+
+    import urllib, urllib2
+    url = 'https://www.pythonanywhere.com/api/web2py/create_account'
+    data = urllib.urlencode(request.vars)
+    req = urllib2.Request(url, data)
+    
+    try:
+        reply = urllib2.urlopen(req)
+    except urllib2.HTTPError as error:
+        if error.code == 400:
+            reply = error
+        elif error.code == 500:
+            return response.json({'status':'error', 'errors':{'username': ['An App other than web2py is installed in the domain %(username)s.pythonanywhere.com' % request.vars]}})
+        else:
+            raise
+    response.headers['Content-Type'] = 'application/json'
+    return reply.read()
+
+
+def list_apps():
+    """ Get a list of apps both remote and local """
+    if not request.vars.username or not request.vars.password:
+        raise HTTP(400)
+    client = ServerProxy('https://%(username)s:%(password)s@%(username)s.pythonanywhere.com/admin/webservices/call/jsonrpc' % request.vars)
+    regex = re.compile('^\w+$')
+    local = [f for f in os.listdir(apath(r=request)) if regex.match(f)]
+    try:
+        pythonanywhere = client.list_apps()
+    except ProtocolError as error:
+        raise HTTP(error.errcode)
+    return response.json({'local': local, 'pythonanywhere': pythonanywhere})
+
+
+def bulk_install():
+    """ Install a list of apps """
+
+    def b64pack(app):
+        """
+        Given an app's name, return the base64 representation of its packed version.
+        """
+        folder = apath(app, r=request)
+        tmpfile = StringIO()
+        tar = tarfile.TarFile(fileobj=tmpfile, mode='w')
+        try:
+            filenames = listdir(folder, '^[\w\.\-]+$', add_dirs=True, 
+                                exclude_content_from=['cache', 'sessions', 'errors'])
+            for fname in filenames:
+                tar.add(os.path.join(folder, fname), fname, False)
+        finally:
+            tar.close()
+        tmpfile.seek(0)
+        gzfile = StringIO()
+        w2pfp = gzip.GzipFile(fileobj=gzfile, mode='wb')
+        w2pfp.write(tmpfile.read())
+        w2pfp.close()
+        gzfile.seek(0)
+        return base64.b64encode(gzfile.read())
+
+    request.vars.apps = request.vars['apps[]']
+    if not request.vars.apps or not request.vars.username or not request.vars.password:
+        raise HTTP(400)
+    if not isinstance(request.vars.apps, list):
+        request.vars.apps = [request.vars.apps]  # Only one app selected
+
+    client = ServerProxy('https://%(username)s:%(password)s@%(username)s.pythonanywhere.com/admin/webservices/call/jsonrpc' % request.vars)
+
+    for app in request.vars.apps:
+        try:
+            client.install(app, app+'.w2p', b64pack(app))
+        except ProtocolError as error:
+            raise HTTP(error.errcode)
+
+    return response.json({'status': 'ok'}) 

applications/admin/languages/pt.py

@@ -17,6 +17,7 @@
 'A new version of web2py is available: %s': 'Está disponível uma nova versão do web2py: %s',
 'About': 'sobre',
 'About application': 'Sobre a aplicação',
+'Accept Terms': 'Accept Terms',
 'additional code for your application': 'código adicional para sua aplicação',
 'Additional code for your application': 'Código adicional para a sua aplicação',
 'admin disabled because no admin password': ' admin desabilitado por falta de senha definida',
@@ -50,6 +51,7 @@
 'Autocomplete Python Code': 'Autocompletar Código Python',
 'Available databases and tables': 'Bancos de dados e tabelas disponíveis',
 'back': 'voltar',
+'Begin': 'Begin',
 'browse': 'buscar',
 'cache': 'cache',
 'cache, errors and sessions cleaned': 'cache, erros e sessões eliminadas',
@@ -112,11 +114,15 @@
 'Deploy': 'publicar',
 'Deploy on Google App Engine': 'Publicar no Google App Engine',
 'Deploy to OpenShift': 'Deploy to OpenShift',
+'Deploy to pythonanywhere': 'Deploy to pythonanywhere',
+'Deploy to PythonAnywhere': 'Deploy to PythonAnywhere',
+'Deployment Interface': 'Deployment Interface',
 'Description': 'Descrição',
 'design': 'modificar',
 'DESIGN': 'Projeto',
 'Design for': 'Projeto de',
 'Detailed traceback description': 'Detailed traceback description',
+'details': 'details',
 'direction: ltr': 'direção: ltr',
 'Disable': 'Disable',
 'docs': 'docs',
@@ -137,6 +143,7 @@
 'Editing file': 'Editando arquivo',
 'Editing file "%s"': 'Editando arquivo "%s"',
 'Editing Language file': 'Editando arquivo de linguagem',
+'Email Address': 'Email Address',
 'Enterprise Web Framework': 'Framework web empresarial',
 'Error': 'Erro',
 'Error logs for "%(app)s"': 'Logs de erro para "%(app)s"',
@@ -148,6 +155,7 @@
 'Expand Abbreviation (html files only)': 'Expandir Abreviação (só para ficheiros html)',
 'export as csv file': 'exportar como arquivo CSV',
 'exposes': 'expõe',
+'exposes:': 'exposes:',
 'extends': 'estende',
 'failed to reload module': 'Falha ao recarregar o módulo',
 'failed to reload module because:': 'falha ao recarregar o módulo por:',
@@ -165,6 +173,7 @@
 'Find Next': 'Localizar Seguinte',
 'Find Previous': 'Localizar Anterior',
 'First name': 'Nome',
+'Form has errors': 'Form has errors',
 'Frames': 'Frames',
 'Functions with no doctests will result in [passed] tests.': 'Funções sem doctests resultarão em testes [aceitos].',
 'graph model': 'graph model',
@@ -174,6 +183,7 @@
 'Hide/Show Translated strings': '',
 'htmledit': 'htmledit',
 'If the report above contains a ticket number it indicates a failure in executing the controller, before any attempt to execute the doctests. This is usually due to an indentation error or an error outside function code.\nA green title indicates that all tests (if defined) passed. In this case test results are not shown.': 'Se o relatório acima contém um número de ticket, isso indica uma falha no controlador em execução, antes de tantar executar os doctests. Isto acontece geralmente por erro de endentação ou erro fora do código da função.\r\nO titulo em verde indica que os testes (se definidos) passaram. Neste caso os testes não são mostrados.',
+'if your application uses a database other than sqlite you will then have to configure its DAL in pythonanywhere.': 'if your application uses a database other than sqlite you will then have to configure its DAL in pythonanywhere.',
 'Import/Export': 'Importar/Exportar',
 'includes': 'inclui',
 'insert new': 'inserir novo',
@@ -198,22 +208,28 @@
 'Last name': 'Sobrenome',
 'Last saved on:': 'Salvo em:',
 'License for': 'Licença para',
+'lists by ticket': 'lists by ticket',
+'Loading...': 'Loading...',
 'loading...': 'carregando...',
+'Local Apps': 'Local Apps',
 'locals': 'locals',
 'Login': 'Entrar',
 'login': 'inicio de sessão',
+'Login successful': 'Login successful',
 'Login to the Administrative Interface': 'Entrar na interface adminitrativa',
+'Login/Register': 'Login/Register',
 'Logout': 'finalizar sessão',
 'Lost Password': 'Senha perdida',
-'Manage': 'Manage',
 'manage': 'gerenciar',
+'Manage': 'Manage',
 'merge': 'juntar',
-'Models': 'Modelos',
 'models': 'modelos',
+'Models': 'Modelos',
 'Modules': 'Módulos',
 'modules': 'módulos',
 'Name': 'Nome',
 'new application "%s" created': 'nova aplicação "%s" criada',
+'New Application Wizard': 'New Application Wizard',
 'New application wizard': 'Assistente para novas aplicações ',
 'new plugin installed': 'novo plugin instalado',
 'New Record': 'Novo registro',
@@ -224,6 +240,7 @@
 'No databases in this application': 'Não existem bancos de dados nesta aplicação',
 'no match': 'não encontrado',
 'no package selected': 'nenhum pacote selecionado',
+'No ticket_storage.txt found under /private folder': 'No ticket_storage.txt found under /private folder',
 'online designer': 'online designer',
 'or alternatively': 'or alternatively',
 'Or Get from URL:': 'Ou Obtenha do URL:',
@@ -241,6 +258,7 @@
 'Password': 'Senha',
 'password changed': 'senha alterada',
 'Peeking at file': 'Visualizando arquivo',
+'Please wait, giving pythonanywhere a moment...': 'Please wait, giving pythonanywhere a moment...',
 'plugin "%(plugin)s" deleted': 'plugin "%(plugin)s" eliminado',
 'Plugin "%s" in application': 'Plugin "%s" na aplicação',
 'plugins': 'plugins',
@@ -250,8 +268,11 @@
 'previous 100 rows': '100 registros anteriores',
 'Private files': 'Private files',
 'private files': 'private files',
+'PythonAnywhere Apps': 'PythonAnywhere Apps',
+'PythonAnywhere Password': 'PythonAnywhere Password',
 'Query:': 'Consulta:',
 'Rapid Search': 'Rapid Search',
+'Read': 'Read',
 'record': 'registro',
 'record does not exist': 'o registro não existe',
 'record id': 'id do registro',
@@ -263,6 +284,7 @@
 'Replace': 'Substituir',
 'Replace All': 'Substituir Tudo',
 'request': 'request',
+'requires python-git, but not installed': 'requires python-git, but not installed',
 'Resolve Conflict file': 'Arquivo de resolução de conflito',
 'response': 'response',
 'restore': 'restaurar',
@@ -285,6 +307,9 @@
 'shell': 'Terminal',
 'Site': 'site',
 'some files could not be removed': 'alguns arquicos não puderam ser removidos',
+'Something went wrong please wait a few minutes before retrying': 'Something went wrong please wait a few minutes before retrying',
+'source : filesystem': 'source : filesystem',
+'Start a new app': 'Start a new app',
 'Start searching': 'Start searching',
 'Start wizard': 'iniciar assistente',
 'state': 'estado',
@@ -294,6 +319,7 @@
 'Submit': 'Submit',
 'submit': 'enviar',
 'Sure you want to delete this object?': 'Tem certeza que deseja apaagr este objeto?',
+'switch to : db': 'switch to : db',
 'table': 'tabela',
 'Table name': 'Nome da tabela',
 'test': 'testar',
@@ -348,6 +374,7 @@
 'update': 'atualizar',
 'update all languages': 'atualizar todas as linguagens',
 'Update:': 'Atualizar:',
+'upgrade now to %s': 'upgrade now to %s',
 'upgrade web2py now': 'atualize o web2py agora',
 'upload': 'upload',
 'Upload': 'Upload',
@@ -361,6 +388,7 @@
 'Use (...)&(...) for AND, (...)|(...) for OR, and ~(...)  for NOT to build more complex queries.': 'Use (...)&(...) para AND, (...)|(...) para OR, y ~(...) para NOT, para criar consultas mais complexas.',
 'Use an url:': 'Use uma url:',
 'User ID': 'ID do Usuario',
+'Username': 'Username',
 'variables': 'variáveis',
 'Version': 'Versão',
 'versioning': 'versionamento',
@@ -368,10 +396,13 @@
 'view': 'visão',
 'Views': 'Visões',
 'views': 'visões',
+'Warning!': 'Warning!',
 'Web Framework': 'Web Framework',
+'web2py Admin Password': 'web2py Admin Password',
 'web2py is up to date': 'web2py está atualizado',
 'web2py Recent Tweets': 'Tweets Recentes de @web2py',
 'web2py upgraded; please restart it': 'web2py atualizado; favor reiniciar',
 'Welcome to web2py': 'Bem-vindo ao web2py',
 'YES': 'SIM',
+'You only need these if you have already registered': 'You only need these if you have already registered',
 }

applications/admin/static/js/web2py.js

@@ -75,8 +75,8 @@
          * this over and over... all will be bound to the document
          */
         /*adds btn class to buttons*/
-        $('button', target).addClass('btn');
+        $('button:not([class^="btn"])', target).addClass('btn');
-        $('form input[type="submit"], form input[type="button"]', target).addClass('btn');
+        $('form input[type="submit"]:not([class^="btn"]), form input[type="button"]:not([class^="btn"])', target).addClass('btn');
         /* javascript for PasswordWidget*/
         $('input[type=password][data-w2p_entropy]', target).each(function() {
           web2py.validate_entropy($(this));
@@ -133,7 +133,7 @@
       },
       ajax_init: function(target) {
         /*called whenever a fragment gets loaded */
-        $('.hidden', target).hide();
+        $('.w2p_hidden', target).hide();
         web2py.manage_errors(target);
         web2py.ajax_fields(target);
         web2py.show_if_handler(target);
@@ -161,7 +161,7 @@
          * and require no dom manipulations
          */
         var doc = $(document);
-        doc.on('click', '.flash', function(e) {
+        doc.on('click', '.w2p_flash', function(e) {
           var t = $(this);
           if(t.css('top') == '0px') t.slideUp('slow');
           else t.fadeOut();
@@ -241,7 +241,7 @@
           /*personally I don't like it.
            *if there's an error it it flashed and can be removed
            *as any other message
-           *doc.off('click', '.flash')
+           *doc.off('click', '.w2p_flash')
            */
           switch(xhr.status) {
             case 500:
@@ -257,12 +257,20 @@
           if(form.hasClass('no_trap')) {
             return;
           }
+
+          var w2p_target = $(this).attr('data-w2p_target');
+          if (typeof w2p_target === typeof undefined || w2p_target === false) {
             form.attr('data-w2p_target', target);
+          } else {
+            target = w2p_target;
+          }
+
           var url = form.attr('action');
           if((url === "") || (url === "#") || (typeof url === 'undefined')) {
             /* form has no action. Use component url. */
             url = action;
           }
+
           form.submit(function(e) {
             web2py.disableElement(form.find(web2py.formInputClickSelector));
             web2py.hide_flash();
@@ -530,13 +538,13 @@
       },
       /*helper for flash messages*/
       flash: function(message, status) {
-        var flash = $('.flash');
+        var flash = $('.w2p_flash');
         web2py.hide_flash();
         flash.html(message).addClass(status);
         if(flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
       },
       hide_flash: function() {
-        $('.flash').fadeOut(0).html('');
+        $('.w2p_flash').fadeOut(0).html('');
       },
       show_if_handler: function(target) {
         var triggers = {};
@@ -692,7 +700,7 @@
         $.web2py.component_handler(target);
       },
       main_hook: function() {
-        var flash = $('.flash');
+        var flash = $('.w2p_flash');
         flash.hide();
         if(flash.html()) web2py.flash(flash.html());
         web2py.ajax_init(document);

applications/admin/views/appadmin.html

@@ -137,7 +137,7 @@
     <h4>{{=T("Overview")}}</h4>
     <p>{{=T.M("Number of entries: **%s**", total['entries'])}}</p>
     {{if total['entries'] > 0:}}
-      <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses})",
+    <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses)})",
              dict( ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
     </p>
     <p>
@@ -155,7 +155,7 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle();')}}
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="all_keys">
       {{=total['keys']}}
     </div>
@@ -183,7 +183,7 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle();')}}
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
@@ -212,7 +212,7 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle();')}}
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
       <div class="hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>

applications/admin/views/default/site.html

@@ -27,7 +27,9 @@
           {{buttons.append((URL('pack',args=a), T("Pack all")))}}
           {{buttons.append((URL('pack_custom',args=a), T("Pack custom")))}}
           {{if not os.path.exists('applications/%s/compiled' % a):}}
-                {{buttons.append((URL('compile_app',args=a), T("Compile")))}}
+                {{buttons.append((URL('compile_app',args=[a, 'skip_failed_views']),
+                                 T("Compile (skip failed views)")))}}
+                {{buttons.append((URL('compile_app',args=a), T("Compile (all or nothing)")))}}
           {{else:}}
         {{buttons.append((URL('pack',args=(a, 'compiled')), T("Pack compiled")))}}
         {{if glob.glob('applications/%s/controllers/*.py' % a):}}
@@ -138,6 +140,7 @@
     <p class="row-buttons">
       {{=button(URL('gae','deploy'), T('Deploy on Google App Engine'))}}
       {{=button(URL('openshift','deploy'),T('Deploy to OpenShift'))}}
+      {{=button(URL('pythonanywhere','deploy'), T('Deploy to PythonAnywhere'))}}
     </p>
       </div> <!-- /DEPLOY ON GAE -->
       <!-- APP WIZARD -->

applications/admin/views/layout.html

@@ -77,6 +77,7 @@
         <script type="text/javascript">
             jQuery(document).ready(function(){
                 jQuery("[rel=tooltip]").tooltip();
+                jQuery(":input").attr("autocomplete","off");
             });
         </script>
         <script>

applications/admin/views/pythonanywhere/deploy.html

@@ -0,0 +1,176 @@
+{{extend 'layout.html'}}
+<h2><span style="color:#139FD7">python</span>anywhere {{=T('Deployment Interface')}}</h2>
+
+
+<div id="register_form">
+    <h3>{{=T('Login/Register')}}</h3>
+    <form class="form-horizontal" id="palogin">
+
+    <div class="control-group" id="username__row">
+        <label class="control-label" for="username">{{=T('Username')}}</label>
+        <div class="controls">
+            <input type="text" name="username" id="username"><span class="help-inline">*</span>
+            <span class="help-block"></span>
+        </div>
+    </div>
+
+    <div class="control-group" id="email_address__row">
+        <label class="control-label" for="email_address">{{=T('Email Address')}}</label>
+        <div class="controls">
+            <input type="text" name="email_address" id="email_address">
+            <span class="help-block"></span>
+        </div>
+    </div>
+
+    <div class="control-group" id="pythonanywhere_password__row">
+        <label class="control-label" for="pythonanywhere_password">{{=T('PythonAnywhere Password')}}</label>
+        <div class="controls">
+            <input type="password" name="pythonanywhere_password" id="pythonanywhere_password">
+            <span class="help-block"></span>
+        </div>
+    </div>
+
+    <div class="control-group" id="web2py_admin_password__row">
+        <label class="control-label" for="web2py_admin_password">{{=T('web2py Admin Password')}}</label>
+        <div class="controls">
+            <input type="password" name="web2py_admin_password" id="web2py_admin_password"><span class="help-inline">*</span>
+            <span class="help-block"></span>
+        </div>
+    </div>
+
+    <div class="control-group" id="accepts_terms__row">
+        <div class="controls">
+            <label class="checkbox">
+                <input type="checkbox" name="accepts_terms" id="accepts_terms"><a target="_blank" href="https://www.pythonanywhere.com/terms/">{{=T('Accept Terms')}}</a>
+            </label>
+            <span class="help-block"></span>
+        </div>
+    </div>
+
+    <div class="control-group">
+        <div class="controls">
+            <button type="submit" class="btn btn-primary" id="submit_palogin">{{=T('Submit')}}</button>
+        </div>
+    </div>
+
+    </form>
+    <p>* {{=T('You only need these if you have already registered')}}</p>
+</div>
+
+<div class="row-fluid" id="app_manager" style="display:none;">
+    <div class="span6">
+        <h3>{{=T('Local Apps')}}</h3>
+        <form id="apppicker">
+        <select name="apps" class="form-control" id="local" multiple>
+            <option>{{=T('Loading...')}}</option>
+        </select>
+        <input type="submit" value="Deploy" id="deploy_button" class="btn btn-primary">
+        </form>
+
+        <div class="alert alert-info">
+            <strong>{{=T('Warning!')}}</strong> {{=T('if your application uses a database other than sqlite you will then have to configure its DAL in pythonanywhere.')}}
+        </div>
+    </div>
+    <div class="span6">
+        <h3>{{=T('PythonAnywhere Apps')}}</h3>
+        <ul id="pythonanywhere">
+            <li>{{=T('Loading...')}}</li>
+        </ul>
+    </div>
+</div>
+
+<script>
+
+$(document).ready(function() {
+
+    $('#palogin').off('submit');
+    $('#palogin').submit(function(event) {
+        var data = $('#palogin').serialize(); 
+        $.web2py.disableElement($('#submit_palogin'));
+        $.web2py.disableFormElements($('#palogin'));
+        $.ajax({
+                url: '{{=URL("pythonanywhere", "create_account")}}',
+                type: 'POST',
+                data: data,
+                dataType: 'json',
+        }).done(function(data, textStatus, jqXHR) {
+                $('#palogin .error').removeClass('error');
+                $('#palogin .help-block').text('');
+                if(data.status == 'error') {
+                    for(var error in data.errors) {
+                        $('#' + error + '__row').addClass('error');
+                        $('#' + error + '__row .help-block').text(data.errors[error][0]);
+                    }
+                    $.web2py.enableElement($('#submit_palogin'));
+                    $.web2py.enableFormElements($('#palogin'));
+                    $.web2py.flash("{{=T('Form has errors')}}");
+                } else {
+                    $.web2py.flash("{{=T('Login successful')}}");
+                    $('#register_form').hide();
+                    $('#app_manager').show();
+                    refresh_apps();
+                }
+        }).fail(function(){
+            $.web2py.flash("{{=T('Something went wrong please wait a few minutes before retrying')}}");
+            $.web2py.enableElement($('#submit_palogin'));
+            $.web2py.enableFormElements($('#palogin'));
+        });
+        event.preventDefault();
+    });
+
+    $('#apppicker').off('submit');
+    $('#apppicker').submit(function(event) {
+        var data = $('#apppicker').serialize(); 
+        $.web2py.disableElement($('#deploy_button'));
+        $.ajax({
+                url: '{{=URL("pythonanywhere", "bulk_install")}}',
+                type: 'POST',
+                data: {username: $('#username').val(), password: $('#web2py_admin_password').val(), apps: $('#local').val()},
+                dataType: 'json',
+        }).done(function(data, textStatus, jqXHR) {
+                refresh_apps();
+                $.web2py.enableElement($('#deploy_button'));
+        }).fail(function(){
+            $.web2py.flash("{{=T('Something went wrong please wait a few minutes before retrying')}}");
+            $.web2py.enableElement($('#deploy_button'));
+        });
+        event.preventDefault();
+    });
+});
+
+function refresh_apps() {
+    // Refresh List of Apps
+    $('#deploy_button').prop('disabled', true); 
+    $.ajax({
+            url: '{{=URL("pythonanywhere", "list_apps")}}',
+            type: 'GET',
+            data: {username: $('#username').val(), password: $('#web2py_admin_password').val()},
+            dataType: 'json',
+    }).done(function(data, textStatus, jqXHR) {
+            var i = 0;
+            $('#local').html('')
+            for(i = 0; i < data.local.length; i++) {
+                $('#local').append($('<option>', {
+                    value: data.local[i],
+                    text: data.local[i]
+                }));
+            }
+            $('#local').multiSelect('refresh');
+            $('#pythonanywhere').html('')
+            for(i = 0; i < data.pythonanywhere.length; i++) {
+                $('#pythonanywhere').append($('<li>', {
+                    text: data.pythonanywhere[i]
+                }));
+            }
+            $('#deploy_button').prop('disabled', false);
+            $.web2py.hide_flash();
+    }).fail(function(){
+        // Mostly this happens if it's a new account, just waiting a bit should be enough.
+        $.get('http://' + $('#username').val() + '.pythonanywhere.com'); // Kickstart the instance
+        $.web2py.flash("{{=T('Please wait, giving pythonanywhere a moment...')}}");
+        setTimeout(refresh_apps, 30000);
+    });
+}
+
+</script>
+

applications/examples/controllers/appadmin.py

@@ -445,15 +445,16 @@ def ccache():
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
-        for key, value in cache.ram.storage.iteritems():
+        # get ram stats directly from the cache object
-            if isinstance(value, dict):
+        ram_stats = cache.ram.stats[request.application]
-                ram['hits'] = value['hit_total'] - value['misses']
+        ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
-                ram['misses'] = value['misses']
+        ram['misses'] = ram_stats['misses']
         try:
-                    ram['ratio'] = ram['hits'] * 100 / value['hit_total']
+            ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
         except (KeyError, ZeroDivisionError):
             ram['ratio'] = 0
-            else:
+
+        for key, value in cache.ram.storage.iteritems():
             if hp:
                 ram['bytes'] += hp.iso(value[1]).size
                 ram['objects'] += hp.iso(value[1]).count
@@ -464,11 +465,11 @@ def ccache():
 
         for key in cache.disk.storage:
             value = cache.disk.storage[key]
-            if isinstance(value, dict):
+            if isinstance(value[1], dict):
-                disk['hits'] = value['hit_total'] - value['misses']
+                disk['hits'] = value[1]['hit_total'] - value[1]['misses']
-                disk['misses'] = value['misses']
+                disk['misses'] = value[1]['misses']
                 try:
-                    disk['ratio'] = disk['hits'] * 100 / value['hit_total']
+                    disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:

applications/examples/private/content/en/default/documentation/more.markmin

@@ -9,6 +9,7 @@
 #### Learning and Demos
 - [[Intro video http://www.youtube.com/watch?v=BXzqmHx6edY]] and [[code examples https://github.com/mjhea0/web2py]]
 - [[Step by step tutorial https://milesm.pythonanywhere.com/wiki]]
+- [[web2py Reference Project http://www.web2pyref.com/]]
 - [[Killer Web Development Tutorial http://killer-web-development.com/]]
 - [[Real Python for the Web http://www.realpython.com]] (web development with web2py and more!)
 - [[Admin Demo http://www.web2py.com/demo_admin popup]] (web-based IDE)

applications/examples/static/css/web2py.css

@@ -320,3 +320,10 @@ li.w2p_grid_breadcrumb_elem {
 .ie-lte8 div.flash{ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#222222', endColorstr='#000000', GradientType=0 ); }
 .ie-lte8 div.flash:hover {filter:alpha(opacity=25);}
 .ie9 #w2p_query_panel {padding-bottom:2px}
+.control-label.readonly{
+   padding-top:0px !important;
+   padding-right:0px !important;
+}
+
+
+

applications/examples/static/js/web2py.js

@@ -75,8 +75,8 @@
          * this over and over... all will be bound to the document
          */
         /*adds btn class to buttons*/
-        $('button', target).addClass('btn');
+        $('button:not([class^="btn"])', target).addClass('btn');
-        $('form input[type="submit"], form input[type="button"]', target).addClass('btn');
+        $('form input[type="submit"]:not([class^="btn"]), form input[type="button"]:not([class^="btn"])', target).addClass('btn');
         /* javascript for PasswordWidget*/
         $('input[type=password][data-w2p_entropy]', target).each(function() {
           web2py.validate_entropy($(this));
@@ -133,7 +133,7 @@
       },
       ajax_init: function(target) {
         /*called whenever a fragment gets loaded */
-        $('.hidden', target).hide();
+        $('.w2p_hidden', target).hide();
         web2py.manage_errors(target);
         web2py.ajax_fields(target);
         web2py.show_if_handler(target);
@@ -161,7 +161,7 @@
          * and require no dom manipulations
          */
         var doc = $(document);
-        doc.on('click', '.flash', function(e) {
+        doc.on('click', '.w2p_flash', function(e) {
           var t = $(this);
           if(t.css('top') == '0px') t.slideUp('slow');
           else t.fadeOut();
@@ -241,7 +241,7 @@
           /*personally I don't like it.
            *if there's an error it it flashed and can be removed
            *as any other message
-           *doc.off('click', '.flash')
+           *doc.off('click', '.w2p_flash')
            */
           switch(xhr.status) {
             case 500:
@@ -257,12 +257,20 @@
           if(form.hasClass('no_trap')) {
             return;
           }
+
+          var w2p_target = $(this).attr('data-w2p_target');
+          if (typeof w2p_target === typeof undefined || w2p_target === false) {
             form.attr('data-w2p_target', target);
+          } else {
+            target = w2p_target;
+          }
+
           var url = form.attr('action');
           if((url === "") || (url === "#") || (typeof url === 'undefined')) {
             /* form has no action. Use component url. */
             url = action;
           }
+
           form.submit(function(e) {
             web2py.disableElement(form.find(web2py.formInputClickSelector));
             web2py.hide_flash();
@@ -530,13 +538,13 @@
       },
       /*helper for flash messages*/
       flash: function(message, status) {
-        var flash = $('.flash');
+        var flash = $('.w2p_flash');
         web2py.hide_flash();
         flash.html(message).addClass(status);
         if(flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
       },
       hide_flash: function() {
-        $('.flash').fadeOut(0).html('');
+        $('.w2p_flash').fadeOut(0).html('');
       },
       show_if_handler: function(target) {
         var triggers = {};
@@ -692,7 +700,7 @@
         $.web2py.component_handler(target);
       },
       main_hook: function() {
-        var flash = $('.flash');
+        var flash = $('.w2p_flash');
         flash.hide();
         if(flash.html()) web2py.flash(flash.html());
         web2py.ajax_init(document);

applications/examples/views/appadmin.html

@@ -137,7 +137,7 @@
     <h4>{{=T("Overview")}}</h4>
     <p>{{=T.M("Number of entries: **%s**", total['entries'])}}</p>
     {{if total['entries'] > 0:}}
-      <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses})",
+    <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses)})",
              dict( ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
     </p>
     <p>
@@ -155,7 +155,7 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle();')}}
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="all_keys">
       {{=total['keys']}}
     </div>
@@ -183,7 +183,7 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle();')}}
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
     <div class="hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
@@ -212,7 +212,7 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle();')}}
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
       <div class="hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>

applications/welcome/controllers/appadmin.py

@@ -445,15 +445,16 @@ def ccache():
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
-        for key, value in cache.ram.storage.iteritems():
+        # get ram stats directly from the cache object
-            if isinstance(value, dict):
+        ram_stats = cache.ram.stats[request.application]
-                ram['hits'] = value['hit_total'] - value['misses']
+        ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
-                ram['misses'] = value['misses']
+        ram['misses'] = ram_stats['misses']
         try:
-                    ram['ratio'] = ram['hits'] * 100 / value['hit_total']
+            ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
         except (KeyError, ZeroDivisionError):
             ram['ratio'] = 0
-            else:
+
+        for key, value in cache.ram.storage.iteritems():
             if hp:
                 ram['bytes'] += hp.iso(value[1]).size
                 ram['objects'] += hp.iso(value[1]).count
@@ -464,11 +465,11 @@ def ccache():
 
         for key in cache.disk.storage:
             value = cache.disk.storage[key]
-            if isinstance(value, dict):
+            if isinstance(value[1], dict):
-                disk['hits'] = value['hit_total'] - value['misses']
+                disk['hits'] = value[1]['hit_total'] - value[1]['misses']
-                disk['misses'] = value['misses']
+                disk['misses'] = value[1]['misses']
                 try:
-                    disk['ratio'] = disk['hits'] * 100 / value['hit_total']
+                    disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:

applications/welcome/controllers/default.py

@@ -29,12 +29,12 @@ def user():
     http://..../[app]/default/user/profile
     http://..../[app]/default/user/retrieve_password
     http://..../[app]/default/user/change_password
-    http://..../[app]/default/user/manage_users (requires membership in
     http://..../[app]/default/user/bulk_register
     use @auth.requires_login()
         @auth.requires_membership('group name')
         @auth.requires_permission('read','table name',record_id)
     to decorate functions that need access control
+    also notice there is http://..../[app]/appadmin/manage/auth to allow administrator to manage users
     """
     return dict(form=auth())
 

applications/welcome/static/css/web2py-bootstrap3.css

@@ -1,4 +1,4 @@
-div.flash {
+div.w2p_flash {
     background-image: none;
     border-radius: 4px;
     -o-border-radius: 4px;
@@ -15,13 +15,13 @@ div.flash {
     margin: 0 0 20px;
     padding: 15px 35px 15px 15px;
 }
-div.flash.alert:hover {
+div.w2p_flash.alert:hover {
     opacity: 1;
 }
-.ie-lte8 div.flash {
+.ie-lte8 div.w2p_flash {
     filter: progid: DXImageTransform.Microsoft.gradient(startColorstr='#222222', endColorstr='#000000', GradientType=0);
 }
-.ie-lte8 div.flash:hover {
+.ie-lte8 div.w2p_flash:hover {
     filter: alpha(opacity=25);
 }
 .main-container {
@@ -37,7 +37,7 @@ div.error {
     display: inline-block;
     padding: 5px;
 }
-div.flash.alert {
+div.w2p_flash.alert {
     display: none;
     position: fixed;
     top: 70px;
@@ -136,7 +136,7 @@ header h1 {
 header .jumbotron {
     background-color: transparent;
 }
-.flash {
+.w2p_flash {
     opacity: 0.9!important;
     right: 100px;
 }
@@ -314,6 +314,3 @@ td.w2p_fc,
 input[type=checkbox], input[type=radio] {
     margin: 4px 4px 0 0;
 }
-.btn {
-    margin-right: 4px;
-}

applications/welcome/static/css/web2py.css

@@ -33,7 +33,7 @@ audio {width:200px}
 [type="text"], [type="password"], select {
   margin-right: 5px; width: 300px;
 }
-.hidden {display:none;visibility:visible}
+.w2p_hidden {display:none;visibility:visible}
 .right {float:right; text-align:right}
 .left {float:left; text-align:left}
 .center {width:100%; text-align:center; vertical-align:middle}
@@ -88,7 +88,7 @@ div.w2p_export_menu a, div.w2p_wiki_tags a, div.w2p_cloud a {margin-left:5px; pa
 #web2py_user_form td {vertical-align:top}
 
 /*********** web2py specific ***********/
-div.flash {
+div.w2p_flash {
     font-weight:bold;
     display:none;
     position:fixed;
@@ -117,11 +117,11 @@ div.flash {
     z-index:2000;
 }
 
-div.flash #closeflash{color:inherit; float:right; margin-left:15px;}
+div.w2p_flash #closeflash{color:inherit; float:right; margin-left:15px;}
 .ie-lte7  div.flash #closeflash
 {color:expression(this.parentNode.currentStyle['color']);float:none;position:absolute;right:4px;}
 
-div.flash:hover { opacity:0.25; }
+div.w2p_flash:hover { opacity:0.25; }
 
 div.error_wrapper {display:block}
 div.error {
@@ -304,8 +304,8 @@ li.w2p_grid_breadcrumb_elem {
 /* fix some IE problems */
 
 .ie-lte7 .topbar .container {z-index:2}
-.ie-lte8 div.flash{ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#222222', endColorstr='#000000', GradientType=0 ); }
+.ie-lte8 div.w2p_flash{ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#222222', endColorstr='#000000', GradientType=0 ); }
-.ie-lte8 div.flash:hover {filter:alpha(opacity=25);}
+.ie-lte8 div.w2p_flash:hover {filter:alpha(opacity=25);}
 .ie9 #w2p_query_panel {padding-bottom:2px}
 
 .web2py_console .form-control {width: 20%; display: inline;}

applications/welcome/static/js/web2py-bootstrap3.js

@@ -6,9 +6,9 @@
      * this over and over... all will be bound to the document
      */
     /*adds btn class to buttons*/
-    $('button', target).addClass('btn btn-default');
+    $('button:not([class^="btn"])', target).addClass('btn btn-default');
     $("p.w2p-autocomplete-widget input").addClass('form-control');
-    $('form input[type="submit"], form input[type="button"]', target).addClass('btn btn-default');
+    $('form input[type="submit"]:not([class^="btn"]), form input[type="button"]:not([class^="btn"])', target).addClass('btn btn-default');
     /* javascript for PasswordWidget*/
     $('input[type=password][data-w2p_entropy]', target).each(function() {
       web2py.validate_entropy($(this));
@@ -18,9 +18,9 @@
       function pe(ul, e) {
         var new_line = ml(ul);
         rel(ul);
-        if ($(e.target).parent().is(':visible')) {
+        if ($(e.target).closest('li').is(':visible')) {
           /* make sure we didn't delete the element before we insert after */
-          new_line.insertAfter($(e.target).parent());
+          new_line.insertAfter($(e.target).closest('li'));
         } else {
           /* the line we clicked on was deleted, just add to end of list */
           new_line.appendTo(ul);
@@ -30,9 +30,9 @@
       }
 
       function rl(ul, e) {
-        if ($(ul).children().length > 1) {
+        if ($(ul).find('li').length > 1) {
           /* only remove if we have more than 1 item so the list is never empty */
-          $(e.target).parent().remove();
+          $(e.target).closest('li').remove();
         }
       }
 
@@ -46,13 +46,13 @@
       function rel(ul) {
         /* keep only as many as needed*/
         $(ul).find("li").each(function() {
-          var trimmed = $.trim($(this.firstChild).val());
+          var trimmed = $.trim($(this).find(":text").val());
           if (trimmed == '') $(this).remove();
-          else $(this.firstChild).val(trimmed);
+          else $(this).find(":text").val(trimmed);
         });
       }
       var ul = this;
-      $(ul).find(":text").after('<a class="btn btn-default" href="#">+</a>&nbsp;<a class="btn btn-default" href="#">-</a>').keypress(function(e) {
+      $(ul).find(":text").addClass('form-control').wrap("<div class='input-group'></div>").after('<div class="input-group-addon"><i class="glyphicon glyphicon-plus"></i></div><div class="input-group-addon"><i class="glyphicon glyphicon-minus"></i></div>').keypress(function(e) {
         return (e.which == 13) ? pe(ul, e) : true;
       }).next().click(function(e) {
         pe(ul, e);

applications/welcome/static/js/web2py.js

@@ -75,8 +75,8 @@
          * this over and over... all will be bound to the document
          */
         /*adds btn class to buttons*/
-        $('button', target).addClass('btn');
+        $('button:not([class^="btn"])', target).addClass('btn');
-        $('form input[type="submit"], form input[type="button"]', target).addClass('btn');
+        $('form input[type="submit"]:not([class^="btn"]), form input[type="button"]:not([class^="btn"])', target).addClass('btn');
         /* javascript for PasswordWidget*/
         $('input[type=password][data-w2p_entropy]', target).each(function() {
           web2py.validate_entropy($(this));
@@ -133,7 +133,7 @@
       },
       ajax_init: function(target) {
         /*called whenever a fragment gets loaded */
-        $('.hidden', target).hide();
+        $('.w2p_hidden', target).hide();
         web2py.manage_errors(target);
         web2py.ajax_fields(target);
         web2py.show_if_handler(target);
@@ -161,7 +161,7 @@
          * and require no dom manipulations
          */
         var doc = $(document);
-        doc.on('click', '.flash', function(e) {
+        doc.on('click', '.w2p_flash', function(e) {
           var t = $(this);
           if(t.css('top') == '0px') t.slideUp('slow');
           else t.fadeOut();
@@ -241,7 +241,7 @@
           /*personally I don't like it.
            *if there's an error it it flashed and can be removed
            *as any other message
-           *doc.off('click', '.flash')
+           *doc.off('click', '.w2p_flash')
            */
           switch(xhr.status) {
             case 500:
@@ -257,12 +257,20 @@
           if(form.hasClass('no_trap')) {
             return;
           }
+
+          var w2p_target = $(this).attr('data-w2p_target');
+          if (typeof w2p_target === typeof undefined || w2p_target === false) {
             form.attr('data-w2p_target', target);
+          } else {
+            target = w2p_target;
+          }
+
           var url = form.attr('action');
           if((url === "") || (url === "#") || (typeof url === 'undefined')) {
             /* form has no action. Use component url. */
             url = action;
           }
+
           form.submit(function(e) {
             web2py.disableElement(form.find(web2py.formInputClickSelector));
             web2py.hide_flash();
@@ -530,13 +538,13 @@
       },
       /*helper for flash messages*/
       flash: function(message, status) {
-        var flash = $('.flash');
+        var flash = $('.w2p_flash');
         web2py.hide_flash();
         flash.html(message).addClass(status);
         if(flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
       },
       hide_flash: function() {
-        $('.flash').fadeOut(0).html('');
+        $('.w2p_flash').fadeOut(0).html('');
       },
       show_if_handler: function(target) {
         var triggers = {};
@@ -692,7 +700,7 @@
         $.web2py.component_handler(target);
       },
       main_hook: function() {
-        var flash = $('.flash');
+        var flash = $('.w2p_flash');
         flash.hide();
         if(flash.html()) web2py.flash(flash.html());
         web2py.ajax_init(document);

applications/welcome/views/appadmin.html

@@ -137,7 +137,7 @@
     <h4>{{=T("Overview")}}</h4>
     <p>{{=T.M("Number of entries: **%s**", total['entries'])}}</p>
     {{if total['entries'] > 0:}}
-      <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses})",
+    <p>{{=T.M("Hit Ratio: **%(ratio)s%%** (**%(hits)s** %%{hit(hits)} and **%(misses)s** %%{miss(misses)})",
              dict( ratio=total['ratio'], hits=total['hits'], misses=total['misses']))}}
     </p>
     <p>
@@ -155,8 +155,8 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle();')}}
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "w2p_hidden" );')}}
-    <div class="hidden" id="all_keys">
+    <div class="w2p_hidden" id="all_keys">
       {{=total['keys']}}
     </div>
     <br />
@@ -183,8 +183,8 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle();')}}
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "w2p_hidden" );')}}
-    <div class="hidden" id="ram_keys">
+    <div class="w2p_hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
     <br />
@@ -212,8 +212,8 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle();')}}
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "w2p_hidden" );')}}
-      <div class="hidden" id="disk_keys">
+      <div class="w2p_hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>
       <br />

applications/welcome/views/layout.html

@@ -21,7 +21,6 @@
     <meta name="google-site-verification" content="">
     <!-- include stylesheets -->
     <link rel="stylesheet" href="{{=URL('static','css/bootstrap.min.css')}}"/>
-    <link rel="stylesheet" href="{{=URL('static','css/bootstrap-theme.min.css')}}"/>
     <link rel="stylesheet" href="{{=URL('static','css/web2py-bootstrap3.css')}}"/>
     <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
     <link rel="apple-touch-icon" href="{{=URL('static','images/favicon.png')}}">
@@ -47,9 +46,9 @@
   </head>
   <body>
     <!--[if lt IE 8]><p class="browserupgrade">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p><![endif]-->
-    <div class="flash alert alert-dismissable">{{=response.flash or ''}}</div>
+    <div class="w2p_flash alert alert-dismissable">{{=response.flash or ''}}</div>
     <!-- Navbar ======================================= -->
-    <nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+    <nav class="navbar navbar-default navbar-fixed-top" role="navigation">
       <div class="container-fluid">
         <div class="navbar-header">
           <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">

fabfile.py

@@ -0,0 +1,150 @@
+from fabric.api import *
+from fabric.operations import put, get
+from fabric.contrib.files import exists
+import os
+import datetime
+import getpass
+
+env.hosts = env.hosts or raw_input('hostname (example.com):').split(',')
+env.user = env.user or raw_input('username              :')
+
+INSTALL_SCRIPT = "setup-web2py-nginx-uwsgi-ubuntu.sh"
+now =  datetime.datetime.now()
+applications = '/home/www-data/web2py/applications'
+
+def create_user(username):
+    """fab -H root@host create_user:username"""
+    password = getpass.getpass(name+' password for %s> ' % username)
+    run('useradd -m %s' % username)
+    run('usermod --password %s %s' % (crypt.crypt(password, 'salt'), username))
+    run('mkdir -p ~%s/.ssh' % username)
+    run('cp /etc/sudoers /tmp/sudoers.new')
+    append('/tmp/sudoers.new', '%s ALL=NOPASSWD: ALL' % username, use_sudo=True)
+    run('visudo -c -f /tmp/sudoers.new')
+    run('EDITOR="cp /tmp/sudoers.new" visudo')
+    uncomment('~%s/.bashrc' % username, '#force_color_prompt=yes')
+    local('ssh-copy-id %s' % env.hosts[0])
+
+def install_web2py():        
+    """fab -H username@host install_web2py"""
+    sudo('wget https://raw.githubusercontent.com/web2py/web2py/master/scripts/%s' % INSTALL_SCRIPT)
+    sudo('chmod +x %s' % INSTALL_SCRIPT)
+    sudo('./'+INSTALL_SCRIPT)
+
+def start_webserver():
+    sudo('service nginx start')
+    sudo('start uwsgi-emperor')
+    sudo('start web2py-scheduler')
+
+def stop_webserver():
+    sudo('stop uwsgi-emperor')
+    sudo('service nginx stop')
+    sudo('stop web2py-scheduler')
+
+def restart_webserver():
+    stop_webserver()
+    start_webserver()
+
+def notify(appname=None):
+    """fab -H username@host notify:appname"""
+    appname = appname or os.path.split(os.getcwd())[-1]
+    appfolder = applications+'/'+appname
+    with cd(appfolder):
+        sudo('echo "response.flash = \'System Going Down For Maintenance\'" > models/flash_goingdown.py')
+
+def down(appname=None):
+    """fab -H username@host down:appname"""
+    appname = appname or os.path.split(os.getcwd())[-1]
+    appfolder = applications+'/'+appname
+    with cd(appfolder):
+        sudo('echo `date` > DISABLED')
+        sudo('rm -rf sessions/* || true')
+
+def up(appname=None):
+    """fab -H username@host up:appname"""
+    appname = appname or os.path.split(os.getcwd())[-1]
+    appfolder = applications+'/'+appname
+    with cd(appfolder):
+        if exists('modules/flash_goingdown.py'):
+            sudo('rm modules/flash_goingdown.py')
+        sudo('rm DISABLED')
+
+def mkdir_or_backup(appname):
+    appfolder = applications+'/'+appname
+    if not exists(appfolder):
+        sudo('mkdir %s' % appfolder)
+        sudo('chown -R www-data:www-data %s' % appfolder)
+        backup = None
+    else:
+        dt = now.strftime('%y-%m-%d-%h-%m')
+        backup = '%s.%s.zip' % (appname, dt)
+        with cd(applications):
+            sudo('zip -r %s %s' % (backup, appname))
+    return backup
+
+def git_deploy(appname, repo):
+    """fab -H username@host git_deploy:appname,username/remoname"""
+    appfolder = applications+'/'+appname
+    backup = mkdir_or_backup(appfolder)
+
+    if exists(appfolder):
+        with cd(appfolder):
+            sudo('git pull origin master')
+            sudo('chown -R www-data:www-data *')
+    else:
+        with cd(applications):
+            sudo('git clone git@github.com/%s %s' % (repo, name))
+            sudo('chown -R www-data:www-data %s' % name)
+    
+def retrieve(appname=None):
+    """fab -H username@host retrieve:appname"""
+    appname = appname or os.path.split(os.getcwd())[-1]
+    appfolder = applications+'/'+appname
+    filename = '%s.zip' % appname
+    with cd(appfolder):
+        sudo('zip -r /tmp/%s *' % filename)
+    get('/tmp/%s' % filename, filename)
+    sudo('rm /tmp/%s' % filename)
+    local('unzip %s' % filename)
+    local('rm %s' % filename)
+
+def deploy(appname=None, all=False):
+    """fab -H username@host deploy:appname,all"""
+    appname = appname or os.path.split(os.getcwd())[-1]
+    appfolder = applications+'/'+appname
+    if os.path.exists('_update.zip'):
+        os.unlink('_update.zip')
+
+    backup = mkdir_or_backup(appfolder)
+            
+    if all=='all' or not backup:
+        local('zip -r _update.zip * -x *~ -x .* -x \#* -x *.bak -x *.bak2')
+    else:        
+        local('zip -r _update.zip */*.py views/*.html views/*/*.html static/*')
+    put('_update.zip','/tmp/_update.zip')
+
+    with cd(appfolder):
+        sudo('unzip -o /tmp/_update.zip')
+        sudo('chown -R www-data:www-data *')
+        sudo('echo "%s" > DATE_DEPLOYMENT' % now)
+               
+    if backup:
+        print 'TO RESTORE: fab restore:%s' % backup
+             
+def restore(backup):
+    """fab -H username@host restore:backupfilename"""
+    appname = backup.split('/')[-1].split('.')[0]
+    appfolder = applications + '/' + appname
+    with cd(appfolder):
+        sudo('rm -r *')
+    with cd(applications):
+        sudo('unzip %s' % backup)
+        sudo('chown -R www-data:www-data %s' % appname)
+
+def cleanup(appname):
+    appname = appname or os.path.split(os.getcwd())[-1]
+    appfolder = applications + '/' + appname
+    with cd(appfolder):
+        sudo('rm -rf sessions/* || true')
+        sudo('rm -rf errors/* || true')
+        sudo('rm -rf cache/* || true')

gluon/admin.py

@@ -59,6 +59,8 @@ def app_pack(app, request, raise_ex=False, filenames=None):
         w2p_pack(filename, apath(app, request), filenames=filenames)
         return filename
     except Exception, e:
+        import traceback
+        print traceback.format_exc()
         if raise_ex:
             raise
         return False
@@ -119,9 +121,8 @@ def app_cleanup(app, request):
 
     # Remove cache files
     path = apath('%s/cache/' % app, request)
-    CacheOnDisk(folder=path).clear()
-    
     if os.path.exists(path):
+        CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
                 if f[:1] != '.': recursive_unlink(os.path.join(path, f))
@@ -130,7 +131,7 @@ def app_cleanup(app, request):
     return r
 
 
-def app_compile(app, request):
+def app_compile(app, request, skip_failed_views=False):
     """Compiles the application
 
     Args:
@@ -144,8 +145,8 @@ def app_compile(app, request):
     from compileapp import compile_application, remove_compiled_application
     folder = apath(app, request)
     try:
-        compile_application(folder)
+        failed_views = compile_application(folder, skip_failed_views)
-        return None
+        return failed_views
     except (Exception, RestrictedError):
         tb = traceback.format_exc(sys.exc_info)
         remove_compiled_application(folder)

gluon/cache.py

@@ -606,8 +606,7 @@ class Cache(object):
             def wrapped_f():
                 if current.request.env.request_method != 'GET':
                     return func()
-                if time_expire:
+
-                    cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire)
                 if quick:
                     session_ = True if 'S' in quick else False
                     vars_ = True if 'V' in quick else False
@@ -615,13 +614,18 @@ class Cache(object):
                     user_agent_ = True if 'U' in quick else False
                     public_ = True if 'P' in quick else False
                 else:
-                        session_, vars_, lang_, user_agent_, public_ = session, vars, lang, user_agent, public
+                    (session_, vars_, lang_, user_agent_, public_) = \
+                        (session, vars, lang, user_agent, public)
+
+                if time_expire:
+                    cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire)
                     if not session_ and public_:
                         cache_control += ', public'
                         expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)).strftime('%a, %d %b %Y %H:%M:%S GMT')
                     else:
                         cache_control += ', private'
                         expires = 'Fri, 01 Jan 1990 00:00:00 GMT'
+
                 if cache_model:
                     #figure out the correct cache key
                     cache_key = [current.request.env.path_info, current.response.view]

gluon/compileapp.py

@@ -464,22 +464,28 @@ def read_pyc(filename):
     return marshal.loads(data[8:])
 
 
-def compile_views(folder):
+def compile_views(folder, skip_failed_views=False):
     """
     Compiles all the views in the application specified by `folder`
     """
 
     path = pjoin(folder, 'views')
+    failed_views = []
     for fname in listdir(path, '^[\w/\-]+(\.\w+)*$'):
         try:
             data = parse_template(fname, path)
         except Exception, e:
+            if skip_failed_views:
+                failed_views.append(fname)
+            else:
                 raise Exception("%s in %s" % (e, fname))
-        filename = 'views.%s.py' % fname.replace(os.path.sep, '.')
+        else:
+            filename = ('views/%s.py' % fname).replace('/', '_').replace('\\', '_')
             filename = pjoin(folder, 'compiled', filename)
             write_file(filename, data)
             save_pyc(filename)
             os.unlink(filename)
+    return failed_views if failed_views else None
 
 
 def compile_models(folder):
@@ -652,7 +658,7 @@ def run_view_in(environment):
     """
     request = current.request
     response = current.response
-    view = response.view
+    view = environment['response'].view
     folder = request.folder
     path = pjoin(folder, 'compiled')
     badv = 'invalid view (%s)' % view
@@ -667,32 +673,28 @@ def run_view_in(environment):
         ccode = parse_template(view, pjoin(folder, 'views'),
                                context=environment)
         restricted(ccode, environment, 'file stream')
-    elif os.path.exists(path):
+    else:
-        x = view.replace('/', '.')
+        filename = pjoin(folder, 'views', view)
-        files = ['views.%s.pyc' % x]
+        if os.path.exists(path): # compiled views
-        if allow_generic:
-            files.append('views.generic.%s.pyc' % request.extension)
-        # for backward compatibility
             x = view.replace('/', '_')
-        files.append('views_%s.pyc' % x)
+            files = ['views_%s.pyc' % x]
+            is_compiled = os.path.exists(pjoin(path, files[0]))
+            # Don't use a generic view if the non-compiled view exists.
+            if is_compiled or (not is_compiled and not os.path.exists(filename)):
                 if allow_generic:
                     files.append('views_generic.%s.pyc' % request.extension)
+                # for backward compatibility
                 if request.extension == 'html':
                     files.append('views_%s.pyc' % x[:-5])
                     if allow_generic:
                         files.append('views_generic.pyc')
                 # end backward compatibility code
                 for f in files:
-            filename = pjoin(path, f)
+                    compiled = pjoin(path, f)
-            if os.path.exists(filename):
+                    if os.path.exists(compiled):
-                code = read_pyc(filename)
+                        code = read_pyc(compiled)
-                restricted(code, environment, layer=filename)
+                        restricted(code, environment, layer=compiled)
                         return
-        raise HTTP(404,
-                   rewrite.THREAD_LOCAL.routes.error_message % badv,
-                   web2py_error=badv)
-    else:
-        filename = pjoin(folder, 'views', view)
         if not os.path.exists(filename) and allow_generic:
             view = 'generic.' + request.extension
             filename = pjoin(folder, 'views', view)
@@ -726,7 +728,7 @@ def remove_compiled_application(folder):
         pass
 
 
-def compile_application(folder):
+def compile_application(folder, skip_failed_views=False):
     """
     Compiles all models, views, controller for the application in `folder`.
     """
@@ -734,7 +736,8 @@ def compile_application(folder):
     os.mkdir(pjoin(folder, 'compiled'))
     compile_models(folder)
     compile_controllers(folder)
-    compile_views(folder)
+    failed_views = compile_views(folder, skip_failed_views)
+    return failed_views
 
 
 def test():

gluon/contrib/autolinks.py

@@ -93,7 +93,7 @@ def video(url):
 
 
 def googledoc_viewer(url):
-    return '<iframe src="http://docs.google.com/viewer?url=%s&embedded=true" style="max-width:100%%"></iframe>' % urllib.quote(url)
+    return '<iframe src="https://docs.google.com/viewer?url=%s&embedded=true" style="max-width:100%%"></iframe>' % urllib.quote(url)
 
 
 def web2py_component(url):

gluon/contrib/imageutils.py

@@ -6,15 +6,17 @@
 #
 # Given the model
 #
-# db.define_table("table_name", Field("picture", "upload"), Field("thumbnail", "upload"))
+# db.define_table("table_name", Field("picture", "upload"),
+#                 Field("thumbnail", "upload"))
 #
-# # to resize the picture on upload
+# to resize the picture on upload
 #
 # from images import RESIZE
 #
 # db.table_name.picture.requires = RESIZE(200, 200)
 #
-# # to store original image in picture and create a thumbnail in 'thumbnail' field
+# to store original image in picture and create a thumbnail
+# in 'thumbnail' field
 #
 # from images import THUMB
 # db.table_name.thumbnail.compute = lambda row: THUMB(row.picture, 200, 200)
@@ -24,8 +26,11 @@ from gluon import current
 
 
 class RESIZE(object):
-    def __init__(self, nx=160, ny=80, error_message=' image resize'):
+
-        (self.nx, self.ny, self.error_message) = (nx, ny, error_message)
+    def __init__(self, nx=160, ny=80, quality=100,
+                 error_message=' image resize'):
+        (self.nx, self.ny, self.quality, self.error_message) = (
+            nx, ny, quality, error_message)
 
     def __call__(self, value):
         if isinstance(value, str) and len(value) == 0:
@@ -36,7 +41,7 @@ class RESIZE(object):
             img = Image.open(value.file)
             img.thumbnail((self.nx, self.ny), Image.ANTIALIAS)
             s = cStringIO.StringIO()
-            img.save(s, 'JPEG', quality=100)
+            img.save(s, 'JPEG', quality=self.quality)
             s.seek(0)
             value.file = s
         except:

gluon/contrib/login_methods/ldap_auth.py

@@ -14,12 +14,20 @@ except Exception, e:
     raise e
 
 
-def ldap_auth(server='ldap', port=None,
+def ldap_auth(server='ldap',
+              port=None,
               base_dn='ou=users,dc=domain,dc=com',
-              mode='uid', secure=False, 
+              mode='uid',
-              cert_path=None, cert_file=None,              
+              secure=False,
-              cacert_path=None, cacert_file=None, key_file=None,
+              self_signed_certificate=None,  # See NOTE below
-              bind_dn=None, bind_pw=None, filterstr='objectClass=*',
+              cert_path=None,
+              cert_file=None,
+              cacert_path=None,
+              cacert_file=None,
+              key_file=None,
+              bind_dn=None,
+              bind_pw=None,
+              filterstr='objectClass=*',
               username_attrib='uid',
               custom_scope='subtree',
               allowed_groups=None,
@@ -159,6 +167,14 @@ def ldap_auth(server='ldap', port=None,
     You can set the logging level with the "logging_level" parameter, default
     is "error" and can be set to error, warning, info, debug.
     """
+
+    if self_signed_certificate:
+        # NOTE : If you have a self-signed SSL Certificate pointing over "port=686" and "secure=True" alone
+        #        will not work, you need also to set "self_signed_certificate=True".
+        # Ref1: https://onemoretech.wordpress.com/2015/06/25/connecting-to-ldap-over-self-signed-tls-with-python/
+        # Ref2: http://bneijt.nl/blog/post/connecting-to-ldaps-with-self-signed-cert-using-python/
+        ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+
     logger = logging.getLogger('web2py.auth.ldap_auth')
     if logging_level == 'error':
         logger.setLevel(logging.ERROR)
@@ -196,8 +212,7 @@ def ldap_auth(server='ldap', port=None,
             logger.warning('blank password not allowed')
             return False
         logger.debug('mode: [%s] manage_user: [%s] custom_scope: [%s]'
-                     ' manage_groups: [%s]' % (str(mode), str(manage_user),
+                     ' manage_groups: [%s]' % (str(mode), str(manage_user), str(custom_scope), str(manage_groups)))
-                     str(custom_scope), str(manage_groups)))
         if manage_user:
             if user_firstname_attrib.count(':') > 0:
                 (user_firstname_attrib,
@@ -246,14 +261,10 @@ def ldap_auth(server='ldap', port=None,
                 # in the ldap_basedn
                 requested_attrs = ['sAMAccountName']
                 if manage_user:
-                    requested_attrs.extend([user_firstname_attrib,
+                    requested_attrs.extend([user_firstname_attrib, user_lastname_attrib, user_mail_attrib])
-                                           user_lastname_attrib,
-                                           user_mail_attrib])
                 result = con.search_ext_s(
                     ldap_basedn, ldap.SCOPE_SUBTREE,
-                    "(&(sAMAccountName=%s)(%s))" % (
+                    "(&(sAMAccountName=%s)(%s))" % (ldap.filter.escape_filter_chars(username_bare), filterstr),
-                                ldap.filter.escape_filter_chars(username_bare),
-                                filterstr),
                     requested_attrs)[0][1]
                 if not isinstance(result, dict):
                     # result should be a dict in the form
@@ -286,9 +297,7 @@ def ldap_auth(server='ldap', port=None,
                 if manage_user:
                     result = con.search_s(dn, ldap.SCOPE_BASE,
                                           "(objectClass=*)",
-                                          [user_firstname_attrib,
+                                          [user_firstname_attrib, user_lastname_attrib, user_mail_attrib])[0][1]
-                                          user_lastname_attrib,
-                                          user_mail_attrib])[0][1]
 
             if ldap_mode == 'uid':
                 # OpenLDAP (UID)
@@ -302,9 +311,7 @@ def ldap_auth(server='ldap', port=None,
                 if manage_user:
                     result = con.search_s(dn, ldap.SCOPE_BASE,
                                           "(objectClass=*)",
-                                          [user_firstname_attrib,
+                                          [user_firstname_attrib, user_lastname_attrib, user_mail_attrib])[0][1]
-                                          user_lastname_attrib,
-                                          user_mail_attrib])[0][1]
 
             if ldap_mode == 'company':
                 # no DNs or password needed to search directory
@@ -319,9 +326,7 @@ def ldap_auth(server='ldap', port=None,
                 # find the uid
                 attrs = ['uid']
                 if manage_user:
-                    attrs.extend([user_firstname_attrib,
+                    attrs.extend([user_firstname_attrib, user_lastname_attrib, user_mail_attrib])
-                                  user_lastname_attrib,
-                                  user_mail_attrib])
                 # perform the actual search
                 company_search_result = con.search_s(ldap_basedn,
                                                      ldap.SCOPE_SUBTREE,
@@ -337,13 +342,11 @@ def ldap_auth(server='ldap', port=None,
                     basedns = ldap_basedn
                 else:
                     basedns = [ldap_basedn]
-                filter = '(&(uid=%s)(%s))' % (
+                filter = '(&(uid=%s)(%s))' % (ldap.filter.escape_filter_chars(username), filterstr)
-                    ldap.filter.escape_filter_chars(username), filterstr)
                 found = False
                 for basedn in basedns:
                     try:
-                        result = con.search_s(basedn, ldap.SCOPE_SUBTREE,
+                        result = con.search_s(basedn, ldap.SCOPE_SUBTREE, filter)
-                                              filter)
                         if result:
                             user_dn = result[0][0]
                             # Check the password
@@ -352,9 +355,10 @@ def ldap_auth(server='ldap', port=None,
                             break
                     except ldap.LDAPError, detail:
                         (exc_type, exc_value) = sys.exc_info()[:2]
-                        logger.warning(
+                        logger.warning("ldap_auth: searching %s for %s resulted in %s: %s\n" % (basedn,
-                        "ldap_auth: searching %s for %s resulted in %s: %s\n" %
+                                                                                                filter,
-                                       (basedn, filter, exc_type, exc_value)
+                                                                                                exc_type,
+                                                                                                exc_value)
                                        )
                 if not found:
                     logger.warning('User [%s] not found!' % username)
@@ -367,10 +371,7 @@ def ldap_auth(server='ldap', port=None,
                     basedns = ldap_basedn
                 else:
                     basedns = [ldap_basedn]
-                filter = '(&(%s=%s)(%s))' % (username_attrib,
+                filter = '(&(%s=%s)(%s))' % (username_attrib, ldap.filter.escape_filter_chars(username), filterstr)
-                                             ldap.filter.escape_filter_chars(
-                                                 username),
-                                             filterstr)
                 if custom_scope == 'subtree':
                     ldap_scope = ldap.SCOPE_SUBTREE
                 elif custom_scope == 'base':
@@ -389,9 +390,10 @@ def ldap_auth(server='ldap', port=None,
                             break
                     except ldap.LDAPError, detail:
                         (exc_type, exc_value) = sys.exc_info()[:2]
-                        logger.warning(
+                        logger.warning("ldap_auth: searching %s for %s resulted in %s: %s\n" % (basedn,
-                        "ldap_auth: searching %s for %s resulted in %s: %s\n" %
+                                                                                                filter,
-                                       (basedn, filter, exc_type, exc_value)
+                                                                                                exc_type,
+                                                                                                exc_value)
                                        )
                 if not found:
                     logger.warning('User [%s] not found!' % username)
@@ -401,16 +403,14 @@ def ldap_auth(server='ldap', port=None,
                 logger.info('[%s] Manage user data' % str(username))
                 try:
                     if user_firstname_part is not None:
-                        store_user_firstname = result[user_firstname_attrib][
+                        store_user_firstname = result[user_firstname_attrib][0].split(' ', 1)[user_firstname_part]
-                            0].split(' ', 1)[user_firstname_part]
                     else:
                         store_user_firstname = result[user_firstname_attrib][0]
                 except KeyError, e:
                     store_user_firstname = None
                 try:
                     if user_lastname_part is not None:
-                        store_user_lastname = result[user_lastname_attrib][
+                        store_user_lastname = result[user_lastname_attrib][0].split(' ', 1)[user_lastname_part]
-                            0].split(' ', 1)[user_lastname_part]
                     else:
                         store_user_lastname = result[user_lastname_attrib][0]
                 except KeyError, e:
@@ -419,32 +419,26 @@ def ldap_auth(server='ldap', port=None,
                     store_user_mail = result[user_mail_attrib][0]
                 except KeyError, e:
                     store_user_mail = None
-                try:
+                update_or_insert_values = {'first_name': store_user_firstname,
-                    #
+                                           'last_name': store_user_lastname,
+                                           'email': store_user_mail}
+                if '@' not in username:
                     # user as username
-                    # #################
+                    # ################
+                    fields = ['first_name', 'last_name', 'email']
                     user_in_db = db(db.auth_user.username == username)
-                    if user_in_db.count() > 0:
+                elif '@' in username:
-                        user_in_db.update(first_name=store_user_firstname,
-                                          last_name=store_user_lastname,
-                                          email=store_user_mail)
-                    else:
-                        db.auth_user.insert(first_name=store_user_firstname,
-                                            last_name=store_user_lastname,
-                                            email=store_user_mail,
-                                            username=username)
-                except:
-                    #
                     # user as email
-                    # ##############
+                    # #############
+                    fields = ['first_name', 'last_name']
                     user_in_db = db(db.auth_user.email == username)
+                update_or_insert_values = {f: update_or_insert_values[f] for f in fields}
                 if user_in_db.count() > 0:
-                        user_in_db.update(first_name=store_user_firstname,
+                    actual_values = user_in_db.select(*[db.auth_user[f] for f in fields]).first().as_dict()
-                                          last_name=store_user_lastname)
+                    if update_or_insert_values != actual_values:  # We don't update record if values are the same
+                        user_in_db.update(**update_or_insert_values)
                 else:
-                        db.auth_user.insert(first_name=store_user_firstname,
+                    db.auth_user.insert(**update_or_insert_values)
-                                            last_name=store_user_lastname,
-                                            email=username)
             con.unbind()
 
             if manage_groups:
@@ -486,9 +480,7 @@ def ldap_auth(server='ldap', port=None,
         # No match
         return False
 
-    def do_manage_groups(username,
+    def do_manage_groups(username, password=None, db=db):
-                         password=None,
-                         db=db):
         """
         Manage user groups
 
@@ -508,23 +500,19 @@ def ldap_auth(server='ldap', port=None,
             # Get all group name where the user is in actually in local db
             # #############################################################
             try:
-                db_user_id = db(db.auth_user.username == username).select(
+                db_user_id = db(db.auth_user.username == username).select(db.auth_user.id).first().id
-                    db.auth_user.id).first().id
             except:
                 try:
-                    db_user_id = db(db.auth_user.email == username).select(
+                    db_user_id = db(db.auth_user.email == username).select(db.auth_user.id).first().id
-                        db.auth_user.id).first().id
                 except AttributeError, e:
                     #
                     # There is no user in local db
                     # We create one
                     # ##############################
                     try:
-                        db_user_id = db.auth_user.insert(username=username,
+                        db_user_id = db.auth_user.insert(username=username, first_name=username)
-                                                         first_name=username)
                     except AttributeError, e:
-                        db_user_id = db.auth_user.insert(email=username,
+                        db_user_id = db.auth_user.insert(email=username, first_name=username)
-                                                         first_name=username)
             if not db_user_id:
                 logging.error(
                     'There is no username or email for %s!' % username)
@@ -532,27 +520,23 @@ def ldap_auth(server='ldap', port=None,
             # if old pydal version, assume this is a relational database which can do joins
             db_can_join = db.can_join() if hasattr(db, 'can_join') else True
             if db_can_join:
-                db_group_search = db(
+                db_group_search = \
-                    (db.auth_membership.user_id == db_user_id) &
+                    db((db.auth_membership.user_id == db_user_id) &
                        (db.auth_user.id == db.auth_membership.user_id) &
                        (db.auth_group.id == db.auth_membership.group_id))
             else:
                 # no joins on NoSQL databases, perform two queries
                 db_group_search = db(db.auth_membership.user_id == db_user_id)
-                group_ids = [x.group_id for x in db_group_search.select(
+                group_ids = [x.group_id for x in db_group_search.select(db.auth_membership.group_id, distinct=True)]
-                        db.auth_membership.group_id, distinct=True)]
                 db_group_search = db(db.auth_group.id.belongs(group_ids))
             db_groups_of_the_user = list()
             db_group_id = dict()
 
             if db_group_search.count() > 0:
-                for group in db_group_search.select(db.auth_group.id,
+                for group in db_group_search.select(db.auth_group.id, db.auth_group.role, distinct=True):
-                                                    db.auth_group.role,
-                                                    distinct=True):
                     db_group_id[group.role] = group.id
                     db_groups_of_the_user.append(group.role)
-            logging.debug('db groups of user %s: %s' %
+            logging.debug('db groups of user %s: %s' % (username, str(db_groups_of_the_user)))
-                          (username, str(db_groups_of_the_user)))
 
             #
             # Delete user membership from groups where user is not anymore
@@ -560,8 +544,7 @@ def ldap_auth(server='ldap', port=None,
             for group_to_del in db_groups_of_the_user:
                 if ldap_groups_of_the_user.count(group_to_del) == 0:
                     db((db.auth_membership.user_id == db_user_id) &
-                       (db.auth_membership.group_id == \
+                       (db.auth_membership.group_id == db_group_id[group_to_del])).delete()
-                         db_group_id[group_to_del])).delete()
 
             #
             # Create user membership in groups where user is not in already
@@ -569,16 +552,12 @@ def ldap_auth(server='ldap', port=None,
             for group_to_add in ldap_groups_of_the_user:
                 if db_groups_of_the_user.count(group_to_add) == 0:
                     if db(db.auth_group.role == group_to_add).count() == 0:
-                        gid = db.auth_group.insert(role=group_to_add,
+                        gid = db.auth_group.insert(role=group_to_add, description='Generated from LDAP')
-                                            description='Generated from LDAP')
                     else:
-                        gid = db(db.auth_group.role == group_to_add).select(
+                        gid = db(db.auth_group.role == group_to_add).select(db.auth_group.id).first().id
-                            db.auth_group.id).first().id
+                    db.auth_membership.insert(user_id=db_user_id, group_id=gid)
-                    db.auth_membership.insert(user_id=db_user_id,
-                                              group_id=gid)
         except:
-            logger.warning("[%s] Groups are not managed successfully!" %
+            logger.warning("[%s] Groups are not managed successfully!" % str(username))
-                           str(username))
             import traceback
             logger.debug(traceback.format_exc())
             return False
@@ -669,10 +648,12 @@ def ldap_auth(server='ldap', port=None,
                 con.simple_bind_s(username, password)
                 logger.debug('Ldap username connect...')
             # We have to use the full string
-            username = con.search_ext_s(base_dn, ldap.SCOPE_SUBTREE,
+            username = \
-                                        "(&(sAMAccountName=%s)(%s))" %
+                con.search_ext_s(base_dn,
-                            (ldap.filter.escape_filter_chars(username_bare),
+                                 ldap.SCOPE_SUBTREE,
-                            filterstr), ["cn"])[0][0]
+                                 "(&(sAMAccountName=%s)(%s))" % (ldap.filter.escape_filter_chars(username_bare),
+                                                                 filterstr),
+                                 ["cn"])[0][0]
         else:
             if ldap_binddn:
                 # need to search directory with an bind_dn account 1st
@@ -685,18 +666,14 @@ def ldap_auth(server='ldap', port=None,
         if username is None:
             return list()
         # search for groups where user is in
-        filter = '(&(%s=%s)(%s))' % (ldap.filter.escape_filter_chars(
+        filter = '(&(%s=%s)(%s))' % (ldap.filter.escape_filter_chars(group_member_attrib),
-                                                            group_member_attrib
-                                                            ),
                                      ldap.filter.escape_filter_chars(username),
                                      group_filterstr)
-        group_search_result = con.search_s(group_dn,
+        group_search_result = con.search_s(group_dn, ldap.SCOPE_SUBTREE, filter, [group_name_attrib])
-                                           ldap.SCOPE_SUBTREE,
-                                           filter, [group_name_attrib])
         ldap_groups_of_the_user = list()
         for group_row in group_search_result:
             group = group_row[1]
-            if type(group) == dict and group.has_key(group_name_attrib):
+            if type(group) == dict and group_name_attrib in group:
                 ldap_groups_of_the_user.extend(group[group_name_attrib])
 
         con.unbind()

gluon/contrib/login_methods/oauth20_account.py

@@ -139,24 +139,36 @@ server for requests.  It can be used for the optional"scope" parameters for Face
         Return the access token generated by the authenticating server.
 
         If token is already in the session that one will be used.
+        If token has expired refresh_token is used to get another token. 
         Otherwise the token is fetched from the auth server.
-
         """
+        refresh_token = None
         if current.session.token and 'expires' in current.session.token:
             expires = current.session.token['expires']
             # reuse token until expiration
             if expires == 0 or expires > time.time():
                 return current.session.token['access_token']
+            if 'refresh_token' in current.session.token:
+                refresh_token = current.session.token['refresh_token']
 
         code = current.request.vars.code
 
-        if code:
+        if code or refresh_token:
-            data = dict(client_id=self.client_id,
+            data = dict(
+                client_id=self.client_id,
                 client_secret=self.client_secret,
+            )
+            if code:
+                data.update(
                     redirect_uri=current.session.redirect_uri,
                     code=code,
                     grant_type='authorization_code'
                 )
+            elif refresh_token:
+                data.update(
+                    refresh_token=refresh_token,
+                    grant_type='refresh_token'
+                )
 
             open_url = None
             opener = self.__build_url_opener(self.token_url)

gluon/contrib/login_methods/saml2_auth.py

@@ -13,6 +13,7 @@ Include in your model (eg db.py)::
 
     auth.define_tables(username=True)
     from gluon.contrib.login_methods.saml2_auth import Saml2Auth
+    import os
     auth.settings.login_form=Saml2Auth(
     config_file = os.path.join(request.folder,'private','sp_conf'),
     maps=dict(
@@ -20,10 +21,59 @@ Include in your model (eg db.py)::
         email=lambda v: v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
         user_id=lambda v: v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0]))
         
-you must have private/sp_conf.py, the pysaml2 sp configuration file
+you must have private/sp_conf.py, the pysaml2 sp configuration file. For example:
+
+
+    #!/usr/bin/env python
+    # -*- coding: utf-8 -*-
+
+    from saml2 import BINDING_HTTP_POST, BINDING_HTTP_REDIRECT
+    import os.path
+    import requests
+    import tempfile
+
+    BASEDIR = os.path.abspath(os.path.dirname(__file__))
+
+    # Web2py SP url and application name
+    HOST = 'http://127.0.0.1:8000'
+    APP = 'sp'
+
+    # To load the IDP metadata...
+    IDP_METADATA = 'http://127.0.0.1:8088/metadata'
+
+    def full_path(local_file):
+        return os.path.join(BASEDIR, local_file)
+
+    CONFIG = {                
+        # your entity id, usually your subdomain plus the url to the metadata view.
+        'entityid': '%s/%s/default/metadata' % (HOST, APP),
+        'service': {
+            'sp' : {
+                'name': 'MYSP',  
+                'endpoints': {     
+                    'assertion_consumer_service': [
+                        ('%s/%s/default/user/login' % (HOST, APP), BINDING_HTTP_REDIRECT),
+                        ('%s/%s/default/user/login' % (HOST, APP), BINDING_HTTP_POST),       
+                        ],
+                    },
+                },
+            },
+        # Your private and public key.
+        'key_file': full_path('pki/mykey.pem'),
+        'cert_file': full_path('pki/mycert.pem'),
+        
+        # where the remote metadata is stored
+        'metadata': {
+            "remote": [{
+                "url": IDP_METADATA,
+                "cert":full_path('pki/mycert.pem')
+                }]
+            },
+    }
+
 """
 
-from saml2 import BINDING_HTTP_REDIRECT
+from saml2 import BINDING_HTTP_REDIRECT, BINDING_HTTP_POST
 from saml2.client import Saml2Client
 from gluon.utils import web2py_uuid
 from gluon import current, redirect, URL
@@ -59,10 +109,13 @@ def saml2_handler(session, request, config_filename = None):
     client = Saml2Client(config_file = config_filename)
     idps = client.metadata.with_descriptor("idpsso")
     entityid = idps.keys()[0]
-    bindings = [BINDING_HTTP_REDIRECT]
+    bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST]
     binding, destination = client.pick_binding(
         "single_sign_on_service", bindings, "idpsso", entity_id=entityid)
+    if request.env.request_method == 'GET':
         binding = BINDING_HTTP_REDIRECT 
+    elif request.env.request_method == 'POST':
+        binding = BINDING_HTTP_POST
     if not request.vars.SAMLResponse:
         req_id, req = client.create_authn_request(destination, binding=binding)
         relay_state = web2py_uuid().replace('-','')

gluon/contrib/plural_rules/fr.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# -*- coding: utf8 -*-
+# -*- coding: utf-8 -*-
 # Plural-Forms for fr (French))
 
 nplurals=2  # French language has 2 forms:
@@ -15,3 +15,55 @@ get_plural_id = lambda n: int(n != 1)
 # for words (or phrases) not found in plural_dict dictionary
 # construct_plural_form = lambda word, plural_id: (word + 'suffix')
 
+irregular={
+    'aïeul': 'aïeux',
+    'bonhomme': 'bonshommes',
+    'ciel': 'cieux',
+    'oeil': 'yeux',
+    'œil': 'yeux',
+    'madame': 'mesdames',
+    'mademoiselle': 'mesdemoiselles',
+    'monsieur': 'messieurs',
+    'bijou': 'bijoux',
+    'caillou': 'cailloux',
+    'chou': 'choux',
+    'genou': 'genoux',
+    'hibou': 'hiboux',
+    'joujou': 'joujoux',
+    'pou': 'poux',
+    'corail': ' coraux',
+    'émail': 'émaux',
+    'travail': 'travaux',
+    'vitrail': 'vitraux',
+    'soupirail': 'soupiraux',
+    'bail': 'baux',
+    'fermail': 'fermaux',
+    'ventail': 'ventaux',
+    'bleu': 'bleus',
+    'pneu': 'pneus',
+    'émeu': 'émeus',
+    'enfeu': 'enfeus',
+    #'lieu': 'lieus', # poisson
+
+}
+
+def construct_plural_form(word, plural_id):
+    u"""
+    >>> [construct_plural_form(x, 1) for x in \
+         [ 'bleu', 'nez', 'sex', 'bas', 'gruau', 'jeu', 'journal',\
+          'chose' ]]
+    ['bleus', 'nez', 'sex', 'bas', 'gruaux', 'jeux', 'journaux', 'choses']
+    """
+    if word in irregular:
+        return irregular[word]
+    if word[-1:] in ('s', 'x', 'z'):
+        return word
+    if word[-2:] in ('au', 'eu'):
+        return word + 'x'
+    if word[-2:] == 'al':
+        return word[0:-2] + 'aux'
+    return word + 's'
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod()

gluon/contrib/simplejsonrpc.py

@@ -33,7 +33,7 @@ except ImportError:
 
 class JSONRPCError(RuntimeError):
     "Error object for remote procedure call fail"
-    def __init__(self, code, message, data=None):
+    def __init__(self, code, message, data=''):
         value = "%s: %s\n%s" % (code, message, '\n'.join(data))
         RuntimeError.__init__(self, value)
         self.code = code

gluon/contrib/spreadsheet.py

@@ -16,7 +16,7 @@ def quote(text):
 
 class Node:
     def __init__(self, name, value, url='.', readonly=False, active=True,
-                 onchange=None, **kwarg):
+                 onchange=None, select=False, size=4, **kwarg):
         self.url = url
         self.name = name
         self.value = str(value)
@@ -26,10 +26,20 @@ class Node:
         self.readonly = readonly
         self.active = active
         self.onchange = onchange
-        self.size = 4
+        self.size = size
         self.locked = False
+        self.select = value if select and not isinstance(value, str) else False
 
     def xml(self):
+        if self.select:
+            selectAttributes = dict(_name=self.name,_id=self.name,_size=self.size,
+                                    _onblur="ajax('%s/blur',['%s']);"%(self.url,self.name))
+            #                        _onkeyup="ajax('%s/keyup',['%s'], ':eval');"%(self.url,self.name),
+            #                        _onfocus="ajax('%s/focus',['%s'], ':eval');"%(self.url,self.name),
+            for k,v in selectAttributes.items():
+                self.select[k] = v
+            return self.select.xml()
+        else:
             return """<input name="%s" id="%s" value="%s" size="%s"
         onkeyup="ajax('%s/keyup',['%s'], ':eval');"
         onfocus="ajax('%s/focus',['%s'], ':eval');"
@@ -391,7 +401,8 @@ class Sheet:
 
     def __init__(self, rows, cols, url='.', readonly=False,
                  active=True, onchange=None, value=None, data=None,
-                 headers=None, update_button="", **kwarg):
+                 headers=None, update_button="", c_headers=None,
+                 r_headers=None, **kwarg):
 
         """
         Arguments:
@@ -425,6 +436,9 @@ class Sheet:
         self.tr_attributes = {}
         self.td_attributes = {}
 
+        self.c_headers = c_headers
+        self.r_headers = r_headers
+
         self.data = data
         self.readonly = readonly
 
@@ -505,7 +519,7 @@ class Sheet:
         self.environment[name] = obj
 
     def cell(self, key, value, readonly=False, active=True,
-             onchange=None, **kwarg):
+             onchange=None, select=False, **kwarg):
         """
         key is the name of the cell
         value is the initial value of the cell. It can be a formula "=1+3"
@@ -528,7 +542,7 @@ class Sheet:
             value = value(r, c)
 
         node = Node(key, value, self.url, readonly, active,
-                    onchange, **kwarg)
+                    onchange, select=select, **kwarg)
         self.nodes[key] = node
         self[key] = value
 
@@ -781,11 +795,19 @@ class Sheet:
              gluon.html.TH, gluon.html.BR, gluon.html.SCRIPT)
         regex = re.compile('r\d+c\d+')
 
+        if not self.c_headers:
             header = TR(TH(), *[TH('c%s' % c)
                             for c in range(self.cols)])
+        else:
+            header = TR(TH(), *[TH('%s' % c)
+                            for c in self.c_headers])
+
         rows = []
         for r in range(self.rows):
+            if not self.r_headers:
                 tds = [TH('r%s' % r), ]
+            else:
+                tds = [TH('%s' % self.r_headers[r]), ]
             for c in range(self.cols):
                 key = 'r%sc%s' % (r, c)
                 attributes = {"_class": "w2p_spreadsheet_col_%s" %

gluon/contrib/web2py_jwt.py

@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+import datetime
+import uuid
+import time
+from gluon.serializers import json_parser
+import base64
+import hmac
+import hashlib
+from gluon.storage import Storage
+from gluon.utils import web2py_uuid
+from gluon import current
+from gluon.http import HTTP
+
+
+class Web2pyJwt(object):
+
+    """
+    If left externally, this needs the usual "singleton" approach.
+    Given I (we) don't know if to include in auth yet, let's stick to basics.
+
+    Args:
+     - secret_key: the secret. Without salting, an attacker knowing this can impersonate
+                   any user
+     - algorithm : uses as they are in the JWT specs, HS256, HS384 or HS512 basically means
+                   signing with HMAC with a 256, 284 or 512bit hash
+     - verify_expiration : verifies the expiration checking the exp claim
+     - leeway: allow n seconds of skew when checking for token expiration
+     - expiration : how many seconds a token may be valid
+     - allow_refresh: enable the machinery to get a refreshed token passing a not-already-expired
+                      token
+     - refresh_expiration_delta: to avoid continous refresh of the token
+     - header_prefix : self-explanatory. "JWT" and "Bearer" seems to be the emerging standards
+     - jwt_add_header: a dict holding additional mappings to the header. by default only alg and typ are filled
+     - user_param: the name of the parameter holding the username when requesting a token. Can be useful, e.g, for
+                   email-based authentication, with "email" as a parameter
+     - pass_param: same as above, but for the password
+     - realm: self-explanatory
+     - salt: can be static or a function that takes the payload as an argument.
+             Example:
+             def mysalt(payload):
+                return payload['hmac_key'].split('-')[0]
+     - additional_payload: can be a dict to merge with the payload or a function that takes
+                           the payload as input and returns the modified payload
+                           Example:
+                           def myadditional_payload(payload):
+                               payload['my_name_is'] = 'bond,james bond'
+                               return payload
+     - before_authorization: can be a callable that takes the deserialized token (a dict) as input.
+                             Gets called right after signature verification but before the actual
+                             authorization takes place. You can raise with HTTP a proper error message
+                             Example:
+                             def mybefore_authorization(tokend):
+                                 if not tokend['my_name_is'] == 'bond,james bond':
+                                     raise HTTP(400, u'Invalid JWT my_name_is claim')
+     - max_header_length: check max length to avoid load()ing unusually large tokens (could mean crafted, e.g. in a DDoS.)
+
+    Basic Usage:
+    in models (or the controller needing it)
+
+        myjwt = Web2pyJwt('secret', auth)
+
+    in the controller issuing tokens
+
+        def login_and_take_token():
+            return myjwt.jwt_token_manager()
+
+    A call then to /app/controller/login_and_take_token/auth with username and password returns the token
+    A call to /app/controller/login_and_take_token/refresh with the original token returns the refreshed token
+
+    To protect a function with JWT
+
+        @myjwt.requires_jwt()
+        @auth.requires_login()
+        def protected():
+            return '%s$%s' % (request.now, auth.user_id)
+
+    """
+
+    def __init__(self, secret_key,
+                 auth,
+                 algorithm='HS256',
+                 verify_expiration=True,
+                 leeway=30,
+                 expiration=60 * 5,
+                 allow_refresh=True,
+                 refresh_expiration_delta=60 * 60,
+                 header_prefix='Bearer',
+                 jwt_add_header=None,
+                 user_param='username',
+                 pass_param='password',
+                 realm='Login required',
+                 salt=None,
+                 additional_payload=None,
+                 before_authorization=None,
+                 max_header_length=4*1024,
+                 ):
+        self.secret_key = secret_key
+        self.auth = auth
+        self.algorithm = algorithm
+        if self.algorithm not in ('HS256', 'HS384', 'HS512'):
+            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
+        self.verify_expiration = verify_expiration
+        self.leeway = leeway
+        self.expiration = expiration
+        self.allow_refresh = allow_refresh
+        self.refresh_expiration_delta = refresh_expiration_delta
+        self.header_prefix = header_prefix
+        self.jwt_add_header = jwt_add_header or {}
+        base_header = {'alg': self.algorithm, 'typ': 'JWT'}
+        for k, v in self.jwt_add_header.iteritems():
+            base_header[k] = v
+        self.cached_b64h = self.jwt_b64e(json_parser.dumps(base_header))
+        digestmod_mapping = {
+            'HS256': hashlib.sha256,
+            'HS384': hashlib.sha384,
+            'HS512': hashlib.sha512
+        }
+        self.digestmod = digestmod_mapping[algorithm]
+        self.user_param = user_param
+        self.pass_param = pass_param
+        self.realm = realm
+        self.salt = salt
+        self.additional_payload = additional_payload
+        self.before_authorization = before_authorization
+        self.max_header_length = max_header_length
+        print 'initialized'
+
+    @staticmethod
+    def jwt_b64e(string):
+        if isinstance(string, unicode):
+            string = string.encode('uft-8', 'strict')
+        return base64.urlsafe_b64encode(string).strip(b'=')
+
+    @staticmethod
+    def jwt_b64d(string):
+        """base64 decodes a single bytestring (and is tolerant to getting
+        called with a unicode string).
+        The result is also a bytestring.
+        """
+        if isinstance(string, unicode):
+            string = string.encode('ascii', 'ignore')
+        return base64.urlsafe_b64decode(string + '=' * (-len(string) % 4))
+
+    def generate_token(self, payload):
+        secret = self.secret_key
+        if self.salt:
+            if callable(self.salt):
+                secret = "%s$%s" % (secret, self.salt(payload))
+            else:
+                secret = "%s$%s" % (secret, self.salt)
+            if isinstance(secret, unicode):
+                secret = secret.encode('ascii', 'ignore')
+        b64h = self.cached_b64h
+        b64p = self.jwt_b64e(json_parser.dumps(payload))
+        jbody = b64h + '.' + b64p
+        mauth = hmac.new(key=secret, msg=jbody, digestmod=self.digestmod)
+        jsign = self.jwt_b64e(mauth.digest())
+        return jbody + '.' + jsign
+
+    def verify_signature(self, body, signature, secret):
+        mauth = hmac.new(key=secret, msg=body, digestmod=self.digestmod)
+        return hmac.compare_digest(self.jwt_b64e(mauth.digest()), signature)
+
+    def load_token(self, token):
+        if isinstance(token, unicode):
+            token = token.encode('utf-8', 'strict')
+        body, sig = token.rsplit('.', 1)
+        b64h, b64b = body.split('.', 1)
+        if b64h != self.cached_b64h:
+            # header not the same
+            raise HTTP(400, u'Invalid JWT Header')
+        secret = self.secret_key
+        tokend = json_parser.loads(self.jwt_b64d(b64b))
+        if self.salt:
+            if callable(self.salt):
+                secret = "%s$%s" % (secret, self.salt(tokend))
+            else:
+                secret = "%s$%s" % (secret, self.salt)
+            if isinstance(secret, unicode):
+                secret = secret.encode('ascii', 'ignore')
+        if not self.verify_signature(body, sig, secret):
+            # signature verification failed
+            raise HTTP(400, u'Token signature is invalid')
+        if self.verify_expiration:
+            now = time.mktime(datetime.datetime.utcnow().timetuple())
+            if tokend['exp'] + self.leeway < now:
+                raise HTTP(400, u'Token is expired')
+        if callable(self.before_authorization):
+            self.before_authorization(tokend)
+        return tokend
+
+    def serialize_auth_session(self, session_auth):
+        """
+        As bad as it sounds, as long as this is rarely used (vs using the token)
+        this is the faster method, even if we ditch session in jwt_token_manager().
+        We (mis)use the heavy default auth mechanism to avoid any further computation,
+        while sticking to a somewhat-stable Auth API.
+        """
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
+        expires = now + self.expiration
+        payload = dict(
+            hmac_key=session_auth['hmac_key'],
+            user_groups=session_auth['user_groups'],
+            user=session_auth['user'].as_dict(),
+            iat=now,
+            exp=expires
+        )
+        return payload
+
+    def refresh_token(self, orig_payload):
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
+        if self.verify_expiration:
+            orig_exp = orig_payload['exp']
+            if orig_exp + self.leeway < now:
+                # token already expired, can't be used for refresh
+                raise HTTP(400, u'Token already expired')
+        orig_iat = orig_payload.get('orig_iat') or orig_payload['iat']
+        if orig_iat + self.refresh_expiration_delta < now:
+            # refreshed too long ago
+            raise HTTP(400, u'Token issued too long ago')
+        expires = now + self.refresh_expiration_delta
+        orig_payload.update(
+            orig_iat=orig_iat,
+            iat=now,
+            exp=expires,
+            hmac_key=web2py_uuid()
+        )
+        self.alter_payload(orig_payload)
+        return orig_payload
+
+    def alter_payload(self, payload):
+        if self.additional_payload:
+            if callable(self.additional_payload):
+                payload = self.additional_payload(payload)
+            elif isinstance(self.additional_payload, dict):
+                payload.update(self.additional_payload)
+        return payload
+
+    def jwt_token_manager(self):
+        """
+        The part that issues (and refreshes) tokens.
+        Used in a controller, given myjwt is the istantiated class, as
+
+            def api_auth():
+                return myjwt.jwt_token_manager()
+
+        Then, a call to /app/c/api_auth/auth with username and password
+        returns a token, while /app/c/api_auth/refresh with the current token
+        issues another token
+        """
+        request = current.request
+        # forget and unlock response
+        if request.args(0) == 'auth':
+            current.session.forget(current.response)
+            username = request.vars[self.user_param]
+            password = request.vars[self.pass_param]
+            valid_user = self.auth.login_bare(username, password)
+            if valid_user:
+                payload = self.serialize_auth_session(current.session.auth)
+                self.alter_payload(payload)
+                return self.generate_token(payload)
+            else:
+                raise HTTP(
+                    401, u'Not Authorized',
+                    **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
+        elif request.args(0) == 'refresh':
+            if not self.allow_refresh:
+                raise HTTP(403, u'Refreshing token is not allowed')
+            token = request.vars.token
+            tokend = self.load_token(token)
+            # verification can fail here
+            refreshed = self.refresh_token(tokend)
+            return self.generate_token(refreshed)
+
+    def inject_token(self, tokend):
+        """
+        The real deal, not touching the db but still logging-in the user
+        """
+        self.auth.user = Storage(tokend['user'])
+        self.auth.user_groups = tokend['user_groups']
+        self.auth.hmac_key = tokend['hmac_key']
+
+    def requires_jwt(self, otherwise=None):
+        """
+        The validator that checks for the header or the
+        _token var
+        """
+        request = current.request
+        token_in_header = request.env.http_authorization
+        if token_in_header:
+            parts = token_in_header.split()
+            if parts[0].lower() != self.header_prefix.lower():
+                raise HTTP(400, u'Invalid JWT header')
+            elif len(parts) == 1:
+                raise HTTP(400, u'Invalid JWT header, missing token')
+            elif len(parts) > 2:
+                raise HTTP(400, 'Invalid JWT header, token contains spaces')
+            token = parts[1]
+        else:
+            token = request.vars._token
+        if token and len(token) < self.max_header_length:
+            tokend = self.load_token(token)
+            self.inject_token(tokend)
+        return self.auth.requires(True, otherwise=otherwise)

gluon/globals.py

@@ -208,7 +208,7 @@ class Request(Storage):
     def parse_get_vars(self):
         """Takes the QUERY_STRING and unpacks it to get_vars
         """
-        query_string = self.env.get('QUERY_STRING', '')
+        query_string = self.env.get('query_string', '')
         dget = urlparse.parse_qs(query_string, keep_blank_values=1)  # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
         get_vars = self._get_vars = Storage(dget)
         for (key, value) in get_vars.iteritems():
@@ -1023,10 +1023,16 @@ class Session(Storage):
     def _fixup_before_save(self):
         response = current.response
         rcookies = response.cookies
-        if self._forget and response.session_id_name in rcookies:
+        scookies = rcookies.get(response.session_id_name)
+        if not scookies:
+            return
+        if self._forget:
             del rcookies[response.session_id_name]
-        elif self._secure and response.session_id_name in rcookies:
+            return
-            rcookies[response.session_id_name]['secure'] = True
+        if self.get('httponly_cookies',True):
+            scookies['HttpOnly'] = True
+        if self._secure:
+            scookies['secure'] = True
 
     def clear_session_cookies(self):
         request = current.request
@@ -1074,6 +1080,7 @@ class Session(Storage):
         if response.session_storage_type == 'file':
             target = recfile.generate(response.session_filename)
             try:
+                self._close(response)
                 os.unlink(target)
             except:
                 pass

gluon/html.py

@@ -668,7 +668,7 @@ class XML(XmlComponent):
 
 
 def XML_unpickle(data):
-    return marshal.loads(data)
+    return XML(marshal.loads(data))
 
 
 def XML_pickle(data):
@@ -784,6 +784,9 @@ class DIV(XmlComponent):
         else:
             return self.components[i]
 
+    def get(self, i):
+        return self.attributes.get(i)
+
     def __setitem__(self, i, value):
         """
         Sets attribute with name 'i' or component #i.
@@ -1135,7 +1138,7 @@ class DIV(XmlComponent):
         for (key, value) in kargs.iteritems():
             if key not in ['first_only', 'replace', 'find_text']:
                 if isinstance(value, (str, int)):
-                    if self[key] != str(value):
+                    if str(self[key]) != str(value):
                         check = False
                 elif key in self.attributes:
                     if not value.search(str(self[key])):

gluon/main.py

@@ -370,8 +370,8 @@ def wsgibase(environ, responder):
                     cid = env.http_web2py_component_element,
                     is_local = (env.remote_addr in local_hosts and
                                 client == env.remote_addr),
-                    is_shell = cmd_opts and cmd_opts.shell,
+                    is_shell = False,
-                    is_sheduler = cmd_opts and cmd_opts.scheduler,
+                    is_scheduler = False,
                     is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
                         request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
                         or env.https == 'on'
@@ -423,10 +423,13 @@ def wsgibase(environ, responder):
                 # ##################################################
 
                 if env.http_cookie:
+                    for single_cookie in env.http_cookie.split(';'):
+                        single_cookie = single_cookie.strip()
+                        if single_cookie:
                             try:
-                        request.cookies.load(env.http_cookie)
+                                request.cookies.load(single_cookie)
-                    except Cookie.CookieError, e:
+                            except Cookie.CookieError:
-                        pass  # invalid cookies
+                                pass  # single invalid cookie ignore
 
                 # ##################################################
                 # try load session or create new session file

gluon/shell.py

@@ -129,6 +129,8 @@ def env(
     if global_settings.cmd_options:
         ip = global_settings.cmd_options.ip
         port = global_settings.cmd_options.port
+        request.is_shell = global_settings.cmd_options.shell is not None
+        request.is_scheduler = global_settings.cmd_options.scheduler is not None
     else:
         ip, port = '127.0.0.1', '8000'
     request.env.http_host = '%s:%s' % (ip, port)

gluon/sqlhtml.py

@@ -646,13 +646,12 @@ class AutocompleteWidget(object):
     def __init__(self, request, field, id_field=None, db=None,
                  orderby=None, limitby=(0, 10), distinct=False,
                  keyword='_autocomplete_%(tablename)s_%(fieldname)s',
-                 min_length=2, help_fields=None, help_string=None):
+                 min_length=2, help_fields=None, help_string=None, at_beginning = True):
 
         self.help_fields = help_fields or []
         self.help_string = help_string
         if self.help_fields and not self.help_string:
-            self.help_string = ' '.join('%%(%s)s' % f.name
+            self.help_string = ' '.join('%%(%s)s' % f.name for f in self.help_fields)
-                                        for f in self.help_fields)
 
         self.request = request
         self.keyword = keyword % dict(tablename=field.tablename,
@@ -662,6 +661,7 @@ class AutocompleteWidget(object):
         self.limitby = limitby
         self.distinct = distinct
         self.min_length = min_length
+        self.at_beginning = at_beginning
         self.fields = [field]
         if id_field:
             self.is_reference = True
@@ -679,8 +679,10 @@ class AutocompleteWidget(object):
             field = self.fields[0]
             if settings and settings.global_settings.web2py_runtime_gae:
                 rows = self.db(field.__ge__(self.request.vars[self.keyword]) & field.__lt__(self.request.vars[self.keyword] + u'\ufffd')).select(orderby=self.orderby, limitby=self.limitby, *(self.fields+self.help_fields))
-            else:
+            elif self.at_beginning:
                 rows = self.db(field.like(self.request.vars[self.keyword] + '%', case_sensitive=False)).select(orderby=self.orderby, limitby=self.limitby, distinct=self.distinct, *(self.fields+self.help_fields))
+            else:
+                rows = self.db(field.contains(self.request.vars[self.keyword], case_sensitive=False)).select(orderby=self.orderby, limitby=self.limitby, distinct=self.distinct, *(self.fields+self.help_fields))
             if rows:
                 if self.is_reference:
                     id_field = self.fields[1]
@@ -838,7 +840,7 @@ def formstyle_bootstrap(form, fields):
             controls.add_class('span4')
 
         if isinstance(label, LABEL):
-            label['_class'] = 'control-label'
+            label['_class'] = add_class(label.get('_class'),'control-label')
 
         if _submit:
             # submit button has unwrapped label and controls, different class
@@ -888,7 +890,7 @@ def formstyle_bootstrap3_stacked(form, fields):
                 e.add_class('form-control')
 
         if isinstance(label, LABEL):
-            label['_class'] = 'control-label'
+            label['_class'] = add_class(label.get('_class'),'control-label')
 
         parent.append(DIV(label, _controls, _class='form-group', _id=id))
     return parent
@@ -936,8 +938,10 @@ def formstyle_bootstrap3_inline_factory(col_label_size=3):
             elif isinstance(controls, UL):
                 for e in controls.elements("input"):
                     e.add_class('form-control')
+            elif controls is None or isinstance(controls, basestring):
+                _controls = P(controls, _class="form-control-static %s" % col_class)
             if isinstance(label, LABEL):
-                label['_class'] = 'control-label %s' % label_col_class
+                label['_class'] = add_class(label.get('_class'),'control-label %s' % label_col_class)
 
             parent.append(DIV(label, _controls, _class='form-group', _id=id))
         return parent
@@ -1120,10 +1124,12 @@ class SQLFORM(FORM):
                 raise HTTP(404, "Object not found")
         self.record = record
 
-        self.record_id = record_id
         if keyed:
             self.record_id = dict([(k, record and str(record[k]) or None)
                                    for k in table._primarykey])
+        else:
+            self.record_id = record_id
+
         self.field_parent = {}
         xfields = []
         self.fields = fields
@@ -1181,6 +1187,14 @@ class SQLFORM(FORM):
             label = LABEL(label, label and sep, _for=field_id,
                           _id=field_id + SQLFORM.ID_LABEL_SUFFIX)
 
+            cond = readonly or \
+                (not ignore_rw and not field.writable and field.readable)
+
+            if cond:
+                label['_class'] = 'readonly'
+            else:
+                label['_class'] = ''
+
             row_id = field_id + SQLFORM.ID_ROW_SUFFIX
             if field.type == 'id':
                 self.custom.dspval.id = nbsp
@@ -1209,8 +1223,6 @@ class SQLFORM(FORM):
                 default = field.default
                 if isinstance(default, CALLABLETYPES):
                     default = default()
-            cond = readonly or \
-                (not ignore_rw and not field.writable and field.readable)
 
             if default is not None and not cond:
                 default = field.formatter(default)
@@ -1498,7 +1510,7 @@ class SQLFORM(FORM):
 
         self.custom.end = CAT(self.hidden_fields(), self.custom.end)
 
-        auch = record_id and self.errors and self.deleted
+        auch = self.record_id and self.errors and self.deleted
 
         if self.record_changed and self.detect_record_change:
             message_onchange = \
@@ -1541,7 +1553,8 @@ class SQLFORM(FORM):
             self.accepted = ret
             return ret
 
-        if record_id and str(record_id) != str(self.record_id):
+        if self.record_id:
+            if str(record_id) != str(self.record_id):
                 raise SyntaxError('user is tampering with form\'s record_id: '
                                   '%s != %s' % (record_id, self.record_id))
 
@@ -1666,6 +1679,9 @@ class SQLFORM(FORM):
             elif field.type == 'double':
                 if value is not None:
                     fields[fieldname] = safe_float(value)
+            elif field.type in ('string', 'text'):
+                if fieldname in self.request_vars:
+                    fields[fieldname] = self.request_vars[fieldname]
 
         for fieldname in self.vars:
             if fieldname != 'id' and fieldname in self.table.fields\
@@ -2069,7 +2085,7 @@ class SQLFORM(FORM):
             ## if it's not an integer
             if cache_count is None or isinstance(cache_count, tuple):
                 if groupby:
-                    c = 'count(*)'
+                    c = 'count(*) AS count_all'
                     nrows = db.executesql(
                         'select count(*) from (%s) _tmp;' %
                         dbset._select(c, left=left, cacheable=True,

gluon/template.py

@@ -898,6 +898,9 @@ def render(content="hello world",
         if not 'NOESCAPE' in context:
             context['NOESCAPE'] = NOESCAPE
 
+    if isinstance(content, unicode):
+        content = content.encode('utf8')
+
     # save current response class
     if context and 'response' in context:
         old_response_body = context['response'].body

gluon/tests/test_dal.py

@@ -4,15 +4,16 @@
     Unit tests for gluon.dal
 """
 
+import sys
 import os
 import unittest
 from fix_path import fix_sys_path
 
 fix_sys_path(__file__)
 
-
 from gluon.dal import DAL, Field
 
+
 def tearDownModule():
     try:
         os.unlink('dummy.db')
@@ -50,6 +51,73 @@ class TestDefaultValidators(unittest.TestCase):
         pass
 """
 
+
+def _prepare_exec_for_file(filename):
+    module = []
+    if filename.endswith('.py'):
+        filename = filename[:-3]
+    elif os.path.split(filename)[1] == '__init__.py':
+        filename = os.path.dirname(filename)
+    else:
+        raise 'The file provided (%s) does is not a valid Python file.'
+    filename = os.path.realpath(filename)
+    dirpath = filename
+    while 1:
+        dirpath, extra = os.path.split(dirpath)
+        module.append(extra)
+        if not os.path.isfile(os.path.join(dirpath, '__init__.py')):
+            break
+    sys.path.insert(0, dirpath)
+    return '.'.join(module[::-1])
+
+
+def load_pydal_tests_module():
+    path = os.path.dirname(os.path.abspath(__file__))
+    if not os.path.isfile(os.path.join(path, 'web2py.py')):
+        i = 0
+        while i < 10:
+            i += 1
+            if os.path.exists(os.path.join(path, 'web2py.py')):
+                break
+            path = os.path.abspath(os.path.join(path, '..'))
+    pydal_test_path = os.path.join(
+        path, "gluon", "packages", "dal", "tests", "__init__.py")
+    mname = _prepare_exec_for_file(pydal_test_path)
+    mod = __import__(mname)
+    return mod
+
+
+def pydal_suite():
+    mod = load_pydal_tests_module()
+    suite = unittest.TestSuite()
+    tlist = [
+        getattr(mod, el) for el in mod.__dict__.keys() if el.startswith("Test")
+    ]
+    for t in tlist:
+        suite.addTest(unittest.makeSuite(t))
+    return suite
+
+
+class TestDALAdapters(unittest.TestCase):
+    def _run_tests(self):
+        suite = pydal_suite()
+        return unittest.TextTestRunner(verbosity=2).run(suite)
+
+    def test_mysql(self):
+        if os.environ.get('APPVEYOR'):
+            return
+        os.environ["DB"] = "mysql://root:@localhost/pydal"
+        result = self._run_tests()
+        self.assertTrue(result)
+
+    def test_pg8000(self):
+        if os.environ.get('APPVEYOR'):
+            return
+        os.environ["DB"] = "postgres:pg8000://postgres:@localhost/pydal"
+        result = self._run_tests()
+        self.assertTrue(result)
+
+
 if __name__ == '__main__':
     unittest.main()
     tearDownModule()

gluon/tests/test_globals.py

@@ -6,17 +6,43 @@
 """
 
 
+import re
 import unittest
 from fix_path import fix_sys_path
 
 fix_sys_path(__file__)
 
-from gluon.globals import Response
+from gluon.globals import Request, Response, Session
 from gluon import URL
 
+def setup_clean_session():
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        return current
 
 class testResponse(unittest.TestCase):
 
+    #port from python 2.7, needed for 2.5 and 2.6 tests
+    def assertRegexpMatches(self, text, expected_regexp, msg=None):
+        """Fail the test unless the text matches the regular expression."""
+        if isinstance(expected_regexp, basestring):
+            expected_regexp = re.compile(expected_regexp)
+        if not expected_regexp.search(text):
+            msg = msg or "Regexp didn't match"
+            msg = '%s: %r not found in %r' % (
+                msg, expected_regexp.pattern, text)
+            raise self.failureException(msg)
+
     def test_include_files(self):
 
         def return_includes(response, extensions=None):
@@ -120,5 +146,43 @@ class testResponse(unittest.TestCase):
         content = return_includes(response)
         self.assertEqual(content, '')
 
+    def test_cookies(self):
+        current = setup_clean_session()
+        cookie = str(current.response.cookies)
+        session_key='%s=%s'%(current.response.session_id_name,current.response.session_id)
+        self.assertRegexpMatches(cookie, r'^Set-Cookie: ')
+        self.assertTrue(session_key in cookie)
+        self.assertTrue('Path=/' in cookie)
+
+    def test_cookies_secure(self):
+        current = setup_clean_session()
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('secure' not in cookie)
+
+        current = setup_clean_session()
+        current.session.secure()
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('secure' in cookie)
+
+    def test_cookies_httponly(self):
+        current = setup_clean_session()
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('httponly' in cookie)
+
+        current = setup_clean_session()
+        current.session.httponly_cookies = True
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('httponly' in cookie)
+
+        current = setup_clean_session()
+        current.session.httponly_cookies = False
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('httponly' not in cookie)
+
 if __name__ == '__main__':
     unittest.main()

gluon/tools.py

@@ -478,16 +478,16 @@ class Mail(object):
             text = message
             html = None
 
-        if (not text is None or not html is None) and (not raw):
+        if (text is not None or html is not None) and (not raw):
 
-            if not text is None:
+            if text is not None:
                 if not isinstance(text, basestring):
                     text = text.read()
                 if isinstance(text, unicode):
                     text = text.encode('utf-8')
                 elif not encoding == 'utf-8':
                     text = text.decode(encoding).encode('utf-8')
-            if not html is None:
+            if html is not None:
                 if not isinstance(html, basestring):
                     html = html.read()
                 if isinstance(html, unicode):
@@ -523,14 +523,15 @@ class Mail(object):
                 payload_in.attach(attachment)
         else:
             payload_in.attach(attachments)
+            attachments = [attachments]
 
         #######################################################
         #                      CIPHER                         #
         #######################################################
         cipher_type = cipher_type or self.settings.cipher_type
-        sign = sign if sign != None else self.settings.sign
+        sign = sign if sign is not None else self.settings.sign
         sign_passphrase = sign_passphrase or self.settings.sign_passphrase
-        encrypt = encrypt if encrypt != None else self.settings.encrypt
+        encrypt = encrypt if encrypt is not None else self.settings.encrypt
         #######################################################
         #                       GPGME                         #
         #######################################################
@@ -571,11 +572,11 @@ class Mail(object):
                     c.op_sign(plain, sig, mode.DETACH)
                     sig.seek(0, 0)
                     # make it part of the email
-                    payload = MIMEMultipart.MIMEMultipart('signed',
+                    payload = \
+                        MIMEMultipart.MIMEMultipart('signed',
                                                     boundary=None,
                                                     _subparts=None,
-                                                          **dict(
+                                                    **dict(micalg="pgp-sha1",
-                                                          micalg="pgp-sha1",
                                                            protocol="application/pgp-signature"))
                     # insert the origin payload
                     payload.attach(payload_in)
@@ -637,20 +638,15 @@ class Mail(object):
                 self.error = "No sign and no encrypt is set but cipher type to x509"
                 return False
             import os
-            x509_sign_keyfile = x509_sign_keyfile or\
+            x509_sign_keyfile = x509_sign_keyfile or self.settings.x509_sign_keyfile
-                                      self.settings.x509_sign_keyfile
 
-            x509_sign_chainfile = x509_sign_chainfile or\
+            x509_sign_chainfile = x509_sign_chainfile or self.settings.x509_sign_chainfile
-                                      self.settings.x509_sign_chainfile
 
-            x509_sign_certfile = x509_sign_certfile or\
+            x509_sign_certfile = x509_sign_certfile or self.settings.x509_sign_certfile or \
-                                      self.settings.x509_sign_certfile or\
+                                 x509_sign_keyfile or self.settings.x509_sign_certfile
-                                      x509_sign_keyfile or\
-                                      self.settings.x509_sign_certfile
 
             # crypt certfiles could be a string or a list
-            x509_crypt_certfiles = x509_crypt_certfiles or\
+            x509_crypt_certfiles = x509_crypt_certfiles or self.settings.x509_crypt_certfiles
-                                      self.settings.x509_crypt_certfiles
 
             x509_nocerts = x509_nocerts or\
                                       self.settings.x509_nocerts
@@ -784,7 +780,7 @@ class Mail(object):
                 attachments = attachments and [mail.Attachment(
                         a.my_filename,
                         a.my_payload,
-                        contebt_id='<attachment-%s>' % k
+                        content_id='<attachment-%s>' % k
                         ) for k, a in enumerate(attachments) if not raw]
                 if attachments:
                     result = mail.send_mail(
@@ -1158,6 +1154,8 @@ class Auth(object):
         manager_actions={},
         auth_manager_role=None,
         two_factor_authentication_group = None,
+        auth_two_factor_enabled = False,
+        auth_two_factor_tries_left = 3,
         login_captcha=None,
         register_captcha=None,
         pre_registration_div=None,
@@ -1239,6 +1237,7 @@ class Auth(object):
         invalid_login='Invalid login',
         invalid_user='Invalid user',
         invalid_password='Invalid password',
+        invalid_two_factor_code = 'Incorrect code. {0} more attempt(s) remaining.',
         is_empty="Cannot be empty",
         mismatched_password="Password fields don't match",
         verify_email='Welcome %(username)s! Click on the link %(link)s to verify your email',
@@ -1253,6 +1252,8 @@ class Auth(object):
         reset_password='Click on the link %(link)s to reset your password',
         reset_password_subject='Password reset',
         bulk_invite_subject='Invitation to join %(site)s',
+        retrieve_two_factor_code='Your temporary login code is {0}',
+        retrieve_two_factor_code_subject='Two-step Login Authentication Code',
         bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete the process',
         invalid_reset_password='Invalid reset password',
         profile_updated='Profile updated',
@@ -1297,6 +1298,8 @@ class Auth(object):
         label_client_ip='Client IP',
         label_origin='Origin',
         label_remember_me="Remember me (for 30 days)",
+        label_two_factor='Authentication code',
+        two_factor_comment = 'This code was emailed to you and is required for login.',
         verify_password_comment='please input your password again',
     )
 
@@ -1513,7 +1516,9 @@ class Auth(object):
             reset_password_onaccept=[],
             hmac_key=hmac_key,
             formstyle=current.response.formstyle,
-            label_separator = current.response.form_label_separator
+            label_separator=current.response.form_label_separator,
+            two_factor_methods = [],
+            two_factor_onvalidation = [],
         )
         settings.lock_keys = True
         # ## these are messages that can be customized
@@ -1669,10 +1674,10 @@ class Auth(object):
                                                           urllib.quote(
                                                           logout_next)),
                           'icon': 'icon-off'})
-            if not 'profile' in self.settings.actions_disabled:
+            if 'profile' not in self.settings.actions_disabled:
                 items.append({'name': T('Profile'), 'href': href('profile'),
                               'icon': 'icon-user'})
-            if not 'change_password' in self.settings.actions_disabled:
+            if 'change_password' not in self.settings.actions_disabled:
                 items.append({'name': T('Password'),
                               'href': href('change_password'),
                               'icon': 'icon-lock'})
@@ -1690,10 +1695,10 @@ class Auth(object):
         else:  # User is not logged in
             items.append({'name': T('Log In'), 'href': href('login'),
                           'icon': 'icon-off'})
-            if not 'register' in self.settings.actions_disabled:
+            if 'register' not in self.settings.actions_disabled:
                 items.append({'name': T('Sign Up'), 'href': href('register'),
                               'icon': 'icon-user'})
-            if not 'request_reset_password' in self.settings.actions_disabled:
+            if 'request_reset_password' not in self.settings.actions_disabled:
                 items.append({'name': T('Lost password?'),
                               'href': href('request_reset_password'),
                               'icon': 'icon-lock'})
@@ -1787,10 +1792,7 @@ class Auth(object):
                       custom.navbar(auth.navbar(mode='bare')) or ''}}</ul>
 
             """
-            bare = {}
+            bare = {'prefix': prefix, 'user': user_identifier if self.user_id else None}
-
-            bare['prefix'] = prefix
-            bare['user'] = user_identifier if self.user_id else None
 
             for i in items:
                 if i['name'] == T('Log In'):
@@ -1964,7 +1966,7 @@ class Auth(object):
         settings.enable_tokens = enable_tokens
         if not self.signature:
             self.define_signature()
-        if signature == True:
+        if signature:
             signature_list = [self.signature]
         elif not signature:
             signature_list = []
@@ -1981,7 +1983,7 @@ class Auth(object):
                          error_message=self.messages.email_taken)]
         if not settings.email_case_sensitive:
             is_unique_email.insert(1, IS_LOWER())
-        if not settings.table_user_name in db.tables:
+        if settings.table_user_name not in db.tables:
             passfield = settings.password_field
             extra_fields = settings.extra_fields.get(
                 settings.table_user_name, []) + signature_list
@@ -2056,7 +2058,7 @@ class Auth(object):
                         fake_migrate=fake_migrate,
                         format='%(first_name)s %(last_name)s (%(id)s)'))
         reference_table_user = 'reference %s' % settings.table_user_name
-        if not settings.table_group_name in db.tables:
+        if settings.table_group_name not in db.tables:
             extra_fields = settings.extra_fields.get(
                 settings.table_group_name, []) + signature_list
             db.define_table(
@@ -2073,7 +2075,7 @@ class Auth(object):
                     fake_migrate=fake_migrate,
                     format='%(role)s (%(id)s)'))
         reference_table_group = 'reference %s' % settings.table_group_name
-        if not settings.table_membership_name in db.tables:
+        if settings.table_membership_name not in db.tables:
             extra_fields = settings.extra_fields.get(
                 settings.table_membership_name, []) + signature_list
             db.define_table(
@@ -2087,7 +2089,7 @@ class Auth(object):
                     migrate=self.__get_migrate(
                         settings.table_membership_name, migrate),
                     fake_migrate=fake_migrate))
-        if not settings.table_permission_name in db.tables:
+        if settings.table_permission_name not in db.tables:
             extra_fields = settings.extra_fields.get(
                 settings.table_permission_name, []) + signature_list
             db.define_table(
@@ -2107,7 +2109,7 @@ class Auth(object):
                     migrate=self.__get_migrate(
                         settings.table_permission_name, migrate),
                     fake_migrate=fake_migrate))
-        if not settings.table_event_name in db.tables:
+        if settings.table_event_name not in db.tables:
             db.define_table(
                 settings.table_event_name,
                 Field('time_stamp', 'datetime',
@@ -2131,7 +2133,7 @@ class Auth(object):
                     fake_migrate=fake_migrate))
         now = current.request.now
         if settings.cas_domains:
-            if not settings.table_cas_name in db.tables:
+            if settings.table_cas_name not in db.tables:
                 db.define_table(
                     settings.table_cas_name,
                     Field('user_id', reference_table_user, default=None,
@@ -2148,13 +2150,13 @@ class Auth(object):
         if settings.enable_tokens:
             extra_fields = settings.extra_fields.get(
                 settings.table_token_name, []) + signature_list
-            if not settings.table_token_name in db.tables:
+            if settings.table_token_name not in db.tables:
                 db.define_table(
                     settings.table_token_name,
                     Field('user_id', reference_table_user, default=None,
                           label=self.messages.label_user_id),
                     Field('expires_on', 'datetime', default=datetime.datetime(2999,12,31)),
-                    Field('token',writable=False,default=web2py_uuid(),unique=True),
+                    Field('token',writable=False,default=web2py_uuid,unique=True),
                     *extra_fields,
                     **dict(
                         migrate=self.__get_migrate(
@@ -2235,7 +2237,7 @@ class Auth(object):
                     break
         if not checks:
             return None
-        if not 'registration_id' in keys:
+        if 'registration_id' not in keys:
             keys['registration_id'] = keys[checks[0]]
         # if we think we found the user but registration_id does not match,
         # make new user
@@ -2257,7 +2259,8 @@ class Auth(object):
             if not 'first_name' in keys and 'first_name' in table_user.fields:
                 guess = keys.get('email', 'anonymous').split('@')[0]
                 keys['first_name'] = keys.get('username', guess)
-            user_id = table_user.insert(**table_user._filter_fields(keys))
+            form = table_user._filter_fields(keys)
+            user_id = table_user.insert(**form)
             user = table_user[user_id]
             if self.settings.create_user_groups:
                 group_id = self.add_group(
@@ -2267,6 +2270,8 @@ class Auth(object):
                 self.add_membership(self.settings.everybody_group_id, user_id)
             if login:
                 self.user = user
+            if self.settings.register_onaccept:
+                callback(self.settings.register_onaccept, form)
         return user
 
     def basic(self, basic_auth_realm=False):
@@ -2336,17 +2341,16 @@ class Auth(object):
         userfield = self.settings.login_userfield or 'username' \
             if 'username' in table_user.fields else 'email'
         passfield = self.settings.password_field
-        return Storage({"table_user": table_user,
+        return Storage({'table_user': table_user,
-                        "userfield": userfield,
+                        'userfield': userfield,
-                        "passfield": passfield})
+                        'passfield': passfield})
 
     def login_bare(self, username, password):
         """
         Logins user as specified by username (or email) and password
         """
         settings = self._get_login_settings()
-        user = settings.table_user(**{settings.userfield: \
+        user = settings.table_user(**{settings.userfield: username})
-                       username})
         if user and user.get(settings.passfield, False):
             password = settings.table_user[
                 settings.passfield].validate(password)[0]
@@ -2375,8 +2379,8 @@ class Auth(object):
             fields[settings.passfield] = \
                 settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
         if not fields.get(settings.userfield):
-            raise ValueError("register_bare: " +
+            raise ValueError('register_bare: ' +
-                             "userfield not provided or invalid")
+                             'userfield not provided or invalid')
         user = self.get_or_create_user(fields, login=False, get=False,
                                        update_fields=self.settings.update_fields)
         if not user:
@@ -2396,7 +2400,7 @@ class Auth(object):
         session = current.session
         db, table = self.db, self.table_cas()
         session._cas_service = request.vars.service or session._cas_service
-        if not request.env.http_host in self.settings.cas_domains or \
+        if request.env.http_host not in self.settings.cas_domains or \
                 not session._cas_service:
             raise HTTP(403, 'not authorized')
 
@@ -2424,7 +2428,7 @@ class Auth(object):
         if self.is_logged_in() and not 'renew' in request.vars:
             return allow_access()
         elif not self.is_logged_in() and 'gateway' in request.vars:
-            redirect(service)
+            redirect(session._cas_service)
 
         def cas_onaccept(form, onaccept=onaccept):
             if not onaccept is DEFAULT:
@@ -2478,15 +2482,27 @@ class Auth(object):
         raise HTTP(200, message)
 
     def _reset_two_factor_auth(self, session):
-        """When two-step authentication is enabled, this function is used to
+        """
+        When two-step authentication is enabled, this function is used to
         clear the session after successfully completing second challenge
         or when the maximum number of tries allowed has expired.
         """
         session.auth_two_factor_user = None
         session.auth_two_factor = None
         session.auth_two_factor_enabled = False
-        # Allow up to 4 attempts (the 1st one plus 3 more)
+        # Set the number of attempts. It should be more than 1.
-        session.auth_two_factor_tries_left = 3
+        session.auth_two_factor_tries_left = self.settings.auth_two_factor_tries_left
+
+    def when_is_logged_in_bypass_next_in_url(self, next, session):
+        """
+        This function should be use when someone want to avoid asking for user
+        credentials when loaded page contains "user/login?_next=NEXT_COMPONENT"
+        in the URL is refresh but user is already authenticated.
+        """
+        if self.is_logged_in():
+            if next == session._auth_next:
+                del session._auth_next
+            redirect(next, client_side=self.settings.client_side)
 
     def login(self,
               next=DEFAULT,
@@ -2497,29 +2513,11 @@ class Auth(object):
         """
         Returns a login form
         """
-
-        table_user = self.table_user()
         settings = self.settings
-        if 'username' in table_user.fields or \
-                not settings.login_email_validate:
-            tmpvalidator = IS_NOT_EMPTY(error_message=self.messages.is_empty)
-            if not settings.username_case_sensitive:
-                tmpvalidator = [IS_LOWER(), tmpvalidator]
-        else:
-            tmpvalidator = IS_EMAIL(error_message=self.messages.invalid_email)
-            if not settings.email_case_sensitive:
-                tmpvalidator = [IS_LOWER(), tmpvalidator]
-
         request = current.request
         response = current.response
         session = current.session
 
-        passfield = settings.password_field
-        try:
-            table_user[passfield].requires[-1].min_length = 0
-        except:
-            pass
-
         ### use session for federated login
         snext = self.get_vars_next()
 
@@ -2543,6 +2541,27 @@ class Auth(object):
                         next = user_next
                 else:
                     next = user_next
+                    # Avoid asking unnecessary user credentials when user is logged in
+                    self.when_is_logged_in_bypass_next_in_url(next=next, session=session)
+
+        # Moved here to avoid unnecessary execution in case of redirection to next in case of logged in user
+        table_user = self.table_user()
+        if 'username' in table_user.fields or \
+                not settings.login_email_validate:
+            tmpvalidator = IS_NOT_EMPTY(error_message=self.messages.is_empty)
+            if not settings.username_case_sensitive:
+                tmpvalidator = [IS_LOWER(), tmpvalidator]
+        else:
+            tmpvalidator = IS_EMAIL(error_message=self.messages.invalid_email)
+            if not settings.email_case_sensitive:
+                tmpvalidator = [IS_LOWER(), tmpvalidator]
+
+        passfield = settings.password_field
+        try:
+            table_user[passfield].requires[-1].min_length = 0
+        except:
+            pass
+
         if onvalidation is DEFAULT:
             onvalidation = settings.login_onvalidation
         if onaccept is DEFAULT:
@@ -2554,7 +2573,6 @@ class Auth(object):
 
         user = None  # default
 
-
         # Setup the default field used for the form
         multi_login = False
         if self.settings.login_userfield:
@@ -2601,8 +2619,7 @@ class Auth(object):
 
             # do we use our own login form, or from a central source?
             if settings.login_form == self:
-                form = SQLFORM(
+                form = SQLFORM(table_user,
-                    table_user,
                                fields=[username, passfield],
                                hidden=dict(_next=next),
                                showid=settings.showid,
@@ -2613,7 +2630,6 @@ class Auth(object):
                                extra_fields=extra_fields,
                                )
 
-
                 captcha = settings.login_captcha or \
                     (settings.login_captcha != False and settings.captcha)
                 if captcha:
@@ -2637,13 +2653,13 @@ class Auth(object):
                     if user:
                         # user in db, check if registration pending or disabled
                         temp_user = user                        
-                        if temp_user.registration_key == 'pending':
+                        if (temp_user.registration_key or '').startswith('pending'):
                             response.flash = self.messages.registration_pending
                             return form
                         elif temp_user.registration_key in ('disabled', 'blocked'):
                             response.flash = self.messages.login_disabled
                             return form
-                        elif (not temp_user.registration_key is None
+                        elif (temp_user.registration_key is not None
                               and temp_user.registration_key.strip()):
                             response.flash = \
                                 self.messages.registration_verifying
@@ -2655,7 +2671,7 @@ class Auth(object):
                             if login_method != self and \
                                     login_method(request.vars[username],
                                                  request.vars[passfield]):
-                                if not self in settings.login_methods:
+                                if self not in settings.login_methods:
                                     # do not store password in db
                                     form.vars[passfield] = None
                                 user = self.get_or_create_user(
@@ -2676,7 +2692,7 @@ class Auth(object):
                                 if login_method != self and \
                                         login_method(request.vars[username],
                                                      request.vars[passfield]):
-                                    if not self in settings.login_methods:
+                                    if self not in settings.login_methods:
                                         # do not store password in db
                                         form.vars[passfield] = None
                                     user = self.get_or_create_user(
@@ -2715,17 +2731,21 @@ class Auth(object):
         # authentication step was successful (i.e. user provided correct
         # username and password at the first challenge).
         # Check if this user is signed up for two-factor authentication
-        # Default rule is that the user must be part of a group that is called
+        # If auth.settings.auth_two_factor_enabled it will enable two factor
-        # auth.settings.two_factor_authentication_group
+        # for all the app. Another way to anble two factor is that the user
-        if user and self.settings.two_factor_authentication_group:
+        # must be part of a group that is called auth.settings.two_factor_authentication_group
+        if user and self.settings.auth_two_factor_enabled == True:
+            session.auth_two_factor_enabled = True
+        elif user and self.settings.two_factor_authentication_group:
             role = self.settings.two_factor_authentication_group
             session.auth_two_factor_enabled = self.has_membership(user_id=user.id, role=role)
         # challenge
         if session.auth_two_factor_enabled:
             form = SQLFORM.factory(
                 Field('authentication_code',
+                      label=self.messages.label_two_factor,
                       required=True,
-                      comment='This code was emailed to you and is required for login.'),
+                      comment=self.messages.two_factor_comment),
                 hidden=dict(_next=next),
                 formstyle=settings.formstyle,
                 separator=settings.label_separator
@@ -2740,17 +2760,81 @@ class Auth(object):
             if session.auth_two_factor_user is None and user is not None:
                 session.auth_two_factor_user = user  # store the validated user and associate with this session
                 session.auth_two_factor = random.randint(100000, 999999)
-                session.auth_two_factor_tries_left = 3 # Allow user to try up to 4 times
+                session.auth_two_factor_tries_left = self.settings.auth_two_factor_tries_left
+                # Set the way we generate the code or we send the code. For example using SMS...
+                two_factor_methods = self.settings.two_factor_methods
+
+                if two_factor_methods == []:
                     # TODO: Add some error checking to handle cases where email cannot be sent
                     self.settings.mailer.send(
                         to=user.email,
-                    subject="Two-step Login Authentication Code",
+                        subject=self.messages.retrieve_two_factor_code_subject,
-                    message="Your temporary login code is {0}".format(session.auth_two_factor))
+                        message=self.messages.retrieve_two_factor_code.format(session.auth_two_factor))
+                else:
+                    #Check for all method. It is possible to have multiples
+                    for two_factor_method in two_factor_methods:
+                        try:
+                            # By default we use session.auth_two_factor generated before.
+                            session.auth_two_factor = two_factor_method(user, session.auth_two_factor)
+                        except:
+                            pass
+                        else:
+                            break
+
             if form.accepts(request, session if self.csrf_prevention else None,
                             formname='login', dbio=False,
                             onvalidation=onvalidation,
                             hideerror=settings.hideerror):
                 accepted_form = True
+
+                accepted_form = True
+
+                '''
+                The lists is executed after form validation for each of the corresponding action.
+                For example, in your model:
+
+                In your models copy and paste:
+
+                #Before define tables, we add some extra field to auth_user
+                auth.settings.extra_fields['auth_user'] = [
+                    Field('motp_secret', 'password', length=512, default='', label='MOTP Secret'),
+                    Field('motp_pin', 'string', length=128, default='', label='MOTP PIN')]
+
+                OFFSET = 60 #Be sure is the same in your OTP Client
+
+                #Set session.auth_two_factor to None. Because the code is generated by external app.
+                # This will avoid to use the default setting and send a code by email.
+                def _set_two_factor(user, auth_two_factor):
+                    return None
+
+                def verify_otp(user, otp):
+                import time
+                from hashlib import md5
+                epoch_time = int(time.time())
+                time_start = int(str(epoch_time - OFFSET)[:-1])
+                time_end = int(str(epoch_time + OFFSET)[:-1])
+                for t in range(time_start - 1, time_end + 1):
+                    to_hash = str(t) + user.motp_secret + user.motp_pin
+                    hash = md5(to_hash).hexdigest()[:6]
+                    if otp == hash:
+                    return hash
+
+                auth.settings.auth_two_factor_enabled = True
+                auth.messages.two_factor_comment = "Verify your OTP Client for the code."
+                auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)]
+                auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)]
+
+                '''
+                if self.settings.two_factor_onvalidation != []:
+
+                    for two_factor_onvalidation in self.settings.two_factor_onvalidation:
+                        try:
+                            session.auth_two_factor = two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
+                        except:
+                            pass
+                        else:
+                            break
+
                 if form.vars['authentication_code'] == str(session.auth_two_factor):
                     # Handle the case when the two-factor form has been successfully validated
                     # and the user was previously stored (the current user should be None because
@@ -2771,9 +2855,20 @@ class Auth(object):
                     # session usernamem will still exist
                     self._reset_two_factor_auth(session)
                 else:
-                    # TODO: Limit the number of retries allowed.
-                    response.flash = 'Incorrect code. {0} more attempt(s) remaining.'.format(session.auth_two_factor_tries_left)
                     session.auth_two_factor_tries_left -= 1
+                    # If the number of retries are higher than auth_two_factor_tries_left
+                    # Require user to enter username and password again.
+                    if session.auth_two_factor_enabled and session.auth_two_factor_tries_left < 1:
+                        # Exceeded maximum allowed tries for this code. Require user to enter
+                        # username and password again.
+                        user = None
+                        accepted_form = False
+                        self._reset_two_factor_auth(session)
+                        # Redirect to the default 'next' page without logging
+                        # in. If that page requires login, user will be redirected
+                        # back to the main login form
+                        redirect(next, client_side=settings.client_side)
+                    response.flash = self.messages.invalid_two_factor_code.format(session.auth_two_factor_tries_left)
                     return form
             else:
                 return form
@@ -2840,7 +2935,7 @@ class Auth(object):
         if self.settings.renew_session_onlogout:
             current.session.renew(clear_session=not self.settings.keep_session_onlogout)
         current.session.flash = self.messages.logged_out
-        if not next is None:
+        if next is not None:
             redirect(next)
 
     def register(self,
@@ -2894,14 +2989,19 @@ class Auth(object):
 
         passfield = self.settings.password_field
         formstyle = self.settings.formstyle
+        try: # Make sure we have our original minimum length as other auth forms change it
+            table_user[passfield].requires[-1].min_length = self.settings.password_min_length
+        except:
+            pass
+
         if self.settings.register_verify_password:
-            if self.settings.register_fields == None:
+            if self.settings.register_fields is None:
                 self.settings.register_fields = [f.name for f in table_user if f.writable]
                 k = self.settings.register_fields.index("password")
                 self.settings.register_fields.insert(k+1, "password_two")
             extra_fields = [
-                Field("password_two", "password", requires=IS_EQUAL_TO(
+                Field("password_two", "password",
-                        request.post_vars.get(passfield, None),
+                      requires=IS_EQUAL_TO(request.post_vars.get(passfield, None),
                                            error_message=self.messages.mismatched_password),
                       label=current.T("Confirm Password"))]
         else:
@@ -2928,7 +3028,11 @@ class Auth(object):
                    DIV(_id="pre-reg", *self.settings.pre_registration_div),
                    '', formstyle, '')
 
-        table_user.registration_key.default = key = web2py_uuid()
+        key = web2py_uuid()
+        if self.settings.registration_requires_approval:
+            key = 'pending-'+key
+
+        table_user.registration_key.default = key
         if form.accepts(request, session if self.csrf_prevention else None,
                         formname='register',
                         onvalidation=onvalidation,
@@ -3031,7 +3135,7 @@ class Auth(object):
         """
 
         table_user = self.table_user()
-        if not 'username' in table_user.fields:
+        if 'username' not in table_user.fields:
             raise HTTP(404)
         request = current.request
         response = current.response
@@ -3142,11 +3246,12 @@ class Auth(object):
                         formname='retrieve_password', dbio=False,
                         onvalidation=onvalidation, hideerror=self.settings.hideerror):
             user = table_user(email=form.vars.email)
+            key = user.registration_key
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
                 redirect(self.url(args=request.args))
-            elif user.registration_key in ('pending', 'disabled', 'blocked'):
+            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 current.session.flash = \
                     self.messages.registration_pending
                 redirect(self.url(args=request.args))
@@ -3174,8 +3279,7 @@ class Auth(object):
         table_user.email.requires = old_requires
         return form
 
-    def confirm_registration(
+    def confirm_registration(self,
-        self,
                              next=DEFAULT,
                              onvalidation=DEFAULT,
                              onaccept=DEFAULT,
@@ -3228,8 +3332,7 @@ class Auth(object):
                   requires=self.table_user()[passfield].requires),
             Field('new_password2', 'password',
                   label=self.messages.verify_password,
-                  requires=[IS_EXPR(
+                  requires=[IS_EXPR('value==%s' % repr(request.vars.new_password),
-                        'value==%s' % repr(request.vars.new_password),
                                     self.messages.mismatched_password)]),
             submit_button='Confirm Registration',
             hidden=dict(_next=next),
@@ -3273,7 +3376,7 @@ class Auth(object):
         """
         if not self.user:
             redirect(self.settings.login_url)
-        if not self.setting.bulk_register_enabled:
+        if not self.settings.bulk_register_enabled:
             return HTTP(404)
 
         form = SQLFORM.factory(
@@ -3353,6 +3456,11 @@ class Auth(object):
             session.flash = self.messages.invalid_reset_password
             redirect(next, client_side=self.settings.client_side)
         
+        key = user.registration_key
+        if key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
+            session.flash = self.messages.registration_pending
+            redirect(next, client_side=self.settings.client_side)
+
         if onvalidation is DEFAULT:
             onvalidation = self.settings.reset_password_onvalidation
         if onaccept is DEFAULT:
@@ -3446,11 +3554,12 @@ class Auth(object):
                         onvalidation=onvalidation,
                         hideerror=self.settings.hideerror):
             user = table_user(**{userfield:form.vars.get(userfield)})
+            key = user.registration_key 
             if not user:
                 session.flash = self.messages['invalid_%s' % userfield]
                 redirect(self.url(args=request.args),
                          client_side=self.settings.client_side)
-            elif user.registration_key in ('pending', 'disabled', 'blocked'):
+            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 session.flash = self.messages.registration_pending
                 redirect(self.url(args=request.args),
                          client_side=self.settings.client_side)
@@ -3536,8 +3645,7 @@ class Auth(object):
                   requires=table_user[passfield].requires),
             Field('new_password2', 'password',
                   label=self.messages.verify_password,
-                requires=[IS_EXPR(
+                  requires=[IS_EXPR('value==%s' % repr(request.vars.new_password),
-                    'value==%s' % repr(request.vars.new_password),
                                     self.messages.mismatched_password)]),
             submit_button=self.messages.password_change_button,
             hidden=dict(_next=next),
@@ -3739,7 +3847,12 @@ class Auth(object):
 
                 basic_allowed, basic_accepted, user = self.basic()
                 user = user or self.user
-                if requires_login:
+
+                login_required = requires_login
+                if callable(login_required):
+                    login_required = login_required()
+
+                if login_required:
                     if not user:
                         if current.request.ajax:
                             raise HTTP(401, self.messages.ajax_failed_authentication)
@@ -3915,8 +4028,13 @@ class Auth(object):
         if not user_id and self.user:
             user_id = self.user.id
         membership = self.table_membership()
-        record = membership(user_id=user_id, group_id=group_id)
+        db = membership._db
+        record = db((membership.user_id==user_id)&
+                    (membership.group_id==group_id),
+                    ignore_common_filters=True).select().first()
         if record:
+            if hasattr(record, 'is_active') and not record.is_active:
+                record.update_record(is_active=True)
             return record.id
         else:
             id = membership.insert(group_id=group_id, user_id=user_id)
@@ -3972,7 +4090,7 @@ class Auth(object):
             rows = self.db(membership.user_id
                            == user_id).select(membership.group_id)
             groups = set([row.group_id for row in rows])
-            if group_id and not group_id in groups:
+            if group_id and group_id not in groups:
                 return False
         else:
             groups = set([group_id])
@@ -4011,9 +4129,15 @@ class Auth(object):
         permission = self.table_permission()
         if group_id == 0:
             group_id = self.user_group()
-        record = self.db(permission.group_id == group_id)(permission.name == name)(permission.table_name == str(table_name))(
+        record = self.db((permission.group_id == group_id)&
-                permission.record_id == long(record_id)).select(limitby=(0, 1), orderby_on_limitby=False).first()
+                         (permission.name == name)&
+                         (permission.table_name == str(table_name))&
+                         (permission.record_id == long(record_id)),
+                          ignore_common_filters=True).select(
+            limitby=(0, 1), orderby_on_limitby=False).first()
         if record:
+            if hasattr(record, 'is_active') and not record.is_ctive:
+                record.update_record(is_active=True)
             id = record.id
         else:
             id = permission.insert(group_id=group_id, name=name,
@@ -4206,6 +4330,7 @@ class Auth(object):
                               function=function,
                               groups=groups)
         else:
+            self._wiki.settings.extra = extra or {}
             self._wiki.env.update(env or {})
 
         # if resolve is set to True, process request as wiki call
@@ -4390,7 +4515,7 @@ class Crud(object):
             deletable = self.settings.update_deletable
         if message is DEFAULT:
             message = self.messages.record_updated
-        if not 'hidden' in attributes:
+        if 'hidden' not in attributes:
             attributes['hidden'] = {}
         attributes['hidden']['_next'] = next
         form = SQLFORM(
@@ -4415,7 +4540,7 @@ class Crud(object):
         if not record and captcha:
             addrow(form, captcha.label, captcha, captcha.comment,
                          self.settings.formstyle, 'captcha__row')
-        if not request.extension in ('html', 'load'):
+        if request.extension not in ('html', 'load'):
             (_session, _formname) = (None, None)
         else:
             (_session, _formname) = (
@@ -4441,7 +4566,7 @@ class Crud(object):
                 callback(ondelete, form, table._tablename)
             response.flash = message
             callback(onaccept, form, table._tablename)
-            if not request.extension in ('html', 'load'):
+            if request.extension not in ('html', 'load'):
                 raise HTTP(200, 'RECORD CREATED/UPDATED')
             if isinstance(next, (list, tuple)):  # fix issue with 2.6
                 next = next[0]
@@ -4449,7 +4574,7 @@ class Crud(object):
                 next = replace_id(next, form)
                 session.flash = response.flash
                 redirect(next)
-        elif not request.extension in ('html', 'load'):
+        elif request.extension not in ('html', 'load'):
             raise HTTP(401, serializers.json(dict(errors=form.errors)))
         return form
 
@@ -4474,8 +4599,7 @@ class Crud(object):
             log = self.messages['create_log']
         if message is DEFAULT:
             message = self.messages.record_created
-        return self.update(
+        return self.update(table,
-            table,
                            None,
                            next=next,
                            onvalidation=onvalidation,
@@ -4505,7 +4629,7 @@ class Crud(object):
             formstyle=self.settings.formstyle,
             separator=self.settings.label_separator
             )
-        if not current.request.extension in ('html', 'load'):
+        if current.request.extension not in ('html', 'load'):
             return table._filter_fields(form.record, id=True)
         return form
 
@@ -4537,8 +4661,7 @@ class Crud(object):
             session.flash = message
         redirect(next)
 
-    def rows(
+    def rows(self,
-        self,
              table,
              query=None,
              fields=None,
@@ -4576,9 +4699,9 @@ class Crud(object):
         rows = self.rows(table, query, fields, orderby, limitby)
         if not rows:
             return None  # Nicer than an empty table.
-        if not 'upload' in attr:
+        if 'upload' not in attr:
             attr['upload'] = self.url('download')
-        if not current.request.extension in ('html', 'load'):
+        if current.request.extension not in ('html', 'load'):
             return rows.as_list()
         if not headers:
             if isinstance(table, str):
@@ -4770,8 +4893,7 @@ regex_geocode = \
 def geocode(address):
     try:
         a = urllib.quote(address)
-        txt = fetch('http://maps.googleapis.com/maps/api/geocode/xml?sensor=false&address=%s'
+        txt = fetch('http://maps.googleapis.com/maps/api/geocode/xml?sensor=false&address=%s' % a)
-                     % a)
         item = regex_geocode.search(txt)
         (la, lo) = (float(item.group('la')), float(item.group('lo')))
         return (la, lo)
@@ -5172,7 +5294,8 @@ class Service(object):
 
     # jsonrpc 2.0 error types.  records the following structure {code: (message,meaning)}
     jsonrpc_errors = {
-        -32700: ("Parse error. Invalid JSON was received by the server.",  "An error occurred on the server while parsing the JSON text."),
+        -32700: ("Parse error. Invalid JSON was received by the server.",
+                 "An error occurred on the server while parsing the JSON text."),
         -32600: ("Invalid Request", "The JSON sent is not a valid Request object."),
         -32601: ("Method not found", "The method does not exist / is not available."),
         -32602: ("Invalid params", "Invalid method parameter(s)."),
@@ -5181,8 +5304,7 @@ class Service(object):
 
     def serve_jsonrpc(self):
         def return_response(id, result):
-            return serializers.json({'version': '1.1',
+            return serializers.json({'version': '1.1', 'id': id, 'result': result, 'error': None})
-                'id': id, 'result': result, 'error': None})
 
         def return_error(id, code, message, data=None):
             error = {'name': 'JSONRPCError',
@@ -5367,8 +5489,7 @@ class Service(object):
         response = current.response
         procedures = self.soap_procedures
 
-        location = "%s://%s%s" % (
+        location = "%s://%s%s" % (request.env.wsgi_url_scheme,
-                        request.env.wsgi_url_scheme,
                                   request.env.http_host,
                                   URL(r=request, f="call/soap", vars={}))
         namespace = 'namespace' in response and response.namespace or location
@@ -5669,10 +5790,10 @@ class PluginManager(object):
         settings = self.__getattr__(plugin)
         settings.installed = True
         settings.update(
-            (k, v) for k, v in defaults.items() if not k in settings)
+            (k, v) for k, v in defaults.items() if k not in settings)
 
     def __getattr__(self, key):
-        if not key in self.__dict__:
+        if key not in self.__dict__:
             self.__dict__[key] = Storage()
         return self.__dict__[key]
 
@@ -5747,10 +5868,9 @@ class Expose(object):
 
     def table_folders(self):
         if self.folders:
-            return SPAN(H3('Folders'), TABLE(
+            return SPAN(H3('Folders'),
-                    *[TR(TD(A(folder, _href=URL(args=self.args + [folder]))))
+                        TABLE(*[TR(TD(A(folder, _href=URL(args=self.args + [folder]))))
-                      for folder in self.folders],
+                                for folder in self.folders], **dict(_class="table")))
-                     **dict(_class="table")))
         return ''
 
     @staticmethod
@@ -5769,8 +5889,7 @@ class Expose(object):
                                    TD(IMG(_src=URL(args=self.args + [f]),
                                           _style='max-width:%spx' % width)
                                       if width and self.isimage(f) else ''))
-                                for f in self.filenames],
+                                for f in self.filenames], **dict(_class="table")))
-                               **dict(_class="table")))
         return ''
 
     def xml(self):
@@ -5941,7 +6060,7 @@ class Wiki(object):
         # define only non-existent tables
         for key, value in table_definitions:
             args = []
-            if not key in db.tables():
+            if key not in db.tables():
                 # look for wiki_ extra fields in auth.settings
                 extra_fields = auth.settings.extra_fields
                 if extra_fields:
@@ -5975,7 +6094,7 @@ class Wiki(object):
 
         if (auth.user and
             check_credentials(current.request, gae_login=False) and
-            not 'wiki_editor' in auth.user_groups.values() and
+            'wiki_editor' not in auth.user_groups.values() and
                 self.settings.groups == auth.user_groups.values()):
             group = db.auth_group(role='wiki_editor')
             gid = group.id if group else db.auth_group.insert(
@@ -5990,8 +6109,7 @@ class Wiki(object):
         raise HTTP(401)
 
     def can_read(self, page):
-        if 'everybody' in page.can_read or not \
+        if 'everybody' in page.can_read or not self.settings.manage_permissions:
-            self.settings.manage_permissions:
             return True
         elif self.auth.user:
             groups = self.settings.groups
@@ -6034,9 +6152,7 @@ class Wiki(object):
 
     def automenu(self):
         """adds the menu if not present"""
-        if (not self.wiki_menu_items and
+        if (not self.wiki_menu_items and self.settings.controller and self.settings.function):
-            self.settings.controller and
-            self.settings.function):
             self.wiki_menu_items = self.menu(self.settings.controller,
                                              self.settings.function)
             current.response.menu += self.wiki_menu_items
@@ -6080,8 +6196,7 @@ class Wiki(object):
     def first_paragraph(self, page):
         if not self.can_read(page):
             mm = (page.body or '').replace('\r', '')
-            ps = [p for p in mm.split('\n\n')
+            ps = [p for p in mm.split('\n\n') if not p.startswith('#') and p.strip()]
-                      if not p.startswith('#') and p.strip()]
             if ps:
                 return ps[0]
         return ''
@@ -6227,10 +6342,7 @@ class Wiki(object):
             fragment = self.auth.db.wiki_media.id.represent
             csv = False
             create = False
-            links= [
+            links = [lambda row: A('copy into source', _href='#', _onclick=script % (fragment(row.id, row)))]
-                lambda row:
-                    A('copy into source', _href='#', _onclick=script % (fragment(row.id, row)))
-                    ]
         content = SQLFORM.grid(
             self.auth.db.wiki_media.wiki_page == page.id,
             orderby=self.auth.db.wiki_media.title,
@@ -6399,7 +6511,7 @@ class Wiki(object):
             count = db.wiki_tag.wiki_page.count()
             fields = [db.wiki_page.id, db.wiki_page.slug,
                       db.wiki_page.title, db.wiki_page.tags,
-                      db.wiki_page.can_read, db.wiki+page.can_edit]
+                      db.wiki_page.can_read, db.wiki_page.can_edit]
             if preview:
                 fields.append(db.wiki_page.body)
             if query is None:
@@ -6467,7 +6579,7 @@ class Wiki(object):
         request = current.request
         # FIXME: This is an ugly hack to ensure a default render
         # engine if not specified (with multiple render engines)
-        if not "render" in request.post_vars:
+        if 'render' not in request.post_vars:
             request.post_vars.render = None
         return render(request.post_vars)
 

gluon/validators.py

@@ -694,7 +694,7 @@ class IS_NOT_IN_DB(Validator):
                 return (value, translate(self.error_message))
         else:
             row = subset.select(table._id, field, limitby=(0, 1), orderby_on_limitby=False).first()
-            if row and str(row.id) != str(id):
+            if row and str(row[table._id]) != str(id):
                 return (value, translate(self.error_message))
         return (value, None)
 
@@ -2165,29 +2165,22 @@ class IS_DATE(Validator):
             INPUT(_type='text', _name='name', requires=IS_DATE())
 
     date has to be in the ISO8960 format YYYY-MM-DD
-    timezome must be None or a pytz.timezone("America/Chicago") object
     """
 
     def __init__(self, format='%Y-%m-%d',
-                 error_message='Enter date as %(format)s',
+                 error_message='Enter date as %(format)s'):
-                 timezone=None):
         self.format = translate(format)
         self.error_message = str(error_message)
-        self.timezone = timezone
         self.extremes = {}
 
     def __call__(self, value):
         ovalue = value
         if isinstance(value, datetime.date):
-            if self.timezone is not None:
-                value = value - datetime.timedelta(seconds=self.timezone*3600)
             return (value, None)
         try:
             (y, m, d, hh, mm, ss, t0, t1, t2) = \
                 time.strptime(value, str(self.format))
             value = datetime.date(y, m, d)
-            if self.timezone is not None:
-                value = self.timezone.localize(value).astimezone(utc)
             return (value, None)
         except:
             self.extremes.update(IS_DATETIME.nice(self.format))
@@ -2203,10 +2196,6 @@ class IS_DATE(Validator):
         format = format.replace('%Y', y)
         if year < 1900:
             year = 2000
-        if self.timezone is not None:
-            d = datetime.datetime(year, value.month, value.day)
-            d = d.replace(tzinfo=utc).astimezone(self.timezone)
-        else:
         d = datetime.date(year, value.month, value.day)
         return d.strftime(format)
 
@@ -2258,7 +2247,8 @@ class IS_DATETIME(Validator):
                 time.strptime(value, str(self.format))
             value = datetime.datetime(y, m, d, hh, mm, ss)
             if self.timezone is not None:
-                value = self.timezone.localize(value).astimezone(utc)
+                # TODO: https://github.com/web2py/web2py/issues/1094 (temporary solution)
+                value = self.timezone.localize(value).astimezone(utc).replace(tzinfo=None)
             return (value, None)
         except:
             self.extremes.update(IS_DATETIME.nice(self.format))
@@ -2307,8 +2297,7 @@ class IS_DATE_IN_RANGE(IS_DATE):
                  minimum=None,
                  maximum=None,
                  format='%Y-%m-%d',
-                 error_message=None,
+                 error_message=None):
-                 timezone=None):
         self.minimum = minimum
         self.maximum = maximum
         if error_message is None:
@@ -2320,8 +2309,7 @@ class IS_DATE_IN_RANGE(IS_DATE):
                 error_message = "Enter date in range %(min)s %(max)s"
         IS_DATE.__init__(self,
                          format=format,
-                         error_message=error_message,
+                         error_message=error_message)
-                         timezone=timezone)
         self.extremes = dict(min=self.formatter(minimum),
                              max=self.formatter(maximum))
 
@@ -2847,9 +2835,11 @@ class CRYPT(object):
         self.salt = salt
 
     def __call__(self, value):
-        value = value and value[:self.max_length]
+        v = value and str(value)[:self.max_length]
-        if len(value) < self.min_length:
+        if not v or len(v) < self.min_length:
             return ('', translate(self.error_message))
+        if isinstance(value, LazyCrypt):
+            return (value, None)
         return (LazyCrypt(self, value), None)
 
 #  entropy calculator for IS_STRONG
@@ -3377,7 +3367,8 @@ class IS_IPV4(Validator):
                     (number == self.localhost)):
                     ok = False
             if not (self.is_private is None or self.is_private ==
-                    (sum([number[0] <= number <= number[1] for number in self.private]) > 0)):
+                    (sum([private_number[0] <= number <= private_number[1]
+                          for private_number in self.private]) > 0)):
                     ok = False
             if not (self.is_automatic is None or self.is_automatic ==
                     (self.automatic[0] <= number <= self.automatic[1])):

scripts/setup-web2py-fedora.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 echo "This script will:
 1) Install modules needed to run web2py on Fedora and CentOS/RHEL
 2) Install Python 2.6 to /opt and recompile wsgi if not provided
@@ -27,7 +28,7 @@ Press ENTER to continue...[ctrl+C to abort]"
 
 read CONFIRM
 
-#!/bin/bash
+
 
 ###
 ###  Phase 0 - This may get messy.  Lets work from a temporary directory

scripts/setup-web2py-ubuntu.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 echo "This script will:
 1) install all modules need to run web2py on Ubuntu 14.04
 2) install web2py in /home/www-data/
@@ -12,7 +13,7 @@ Press a key to continue...[ctrl+C to abort]"
 
 read CONFIRM
 
-#!/bin/bash
+
 # optional
 # dpkg-reconfigure console-setup
 # dpkg-reconfigure timezoneconf
@@ -88,9 +89,27 @@ WSGIDaemonProcess web2py user=www-data group=www-data
 
 <VirtualHost *:80>
 
-  RewriteEngine On
+ WSGIProcessGroup web2py
-  RewriteCond %{HTTPS} !=on
+  WSGIScriptAlias / /home/www-data/web2py/wsgihandler.py
-  RewriteRule ^/?(.*) https://%{SERVER_NAME}/$1 [R,L]
+  WSGIPassAuthorization On
+
+  <Directory /home/www-data/web2py>
+    AllowOverride None
+    Require all denied
+    <Files wsgihandler.py>
+      Require all granted
+    </Files>
+  </Directory>
+
+  AliasMatch ^/([^/]+)/static/(?:_[\d]+.[\d]+.[\d]+/)?(.*) \
+        /home/www-data/web2py/applications/$1/static/$2
+
+  <Directory /home/www-data/web2py/applications/*/static/>
+    Options -Indexes
+    ExpiresActive On
+    ExpiresDefault "access plus 1 hour"
+    Require all granted
+  </Directory>
 
   CustomLog /var/log/apache2/access.log common
   ErrorLog /var/log/apache2/error.log