R-2.3.1

explorigin/Rocket merged and IPv6 support added

CHANGELOG

@@ -1,3 +1,30 @@
+## 2.3.1
+
+- new virtual fields syntax: 
+  ``db.define_table('person',Field('name'),Field.Virtual('namey',lambda row: row.person.name+'y'))``
+- db.thing(name='Cohen',_orderby=db.thing.name), thanks Yair
+- made many modules Python 3.3 friendly (compile but not tested)
+- better welcome css, thanks Paolo
+- jQuery 1.8.3
+- Bootstrap 2.2.2
+- Modernizr 2.6.2 (custom full options)
+- integration with analyitics.js (0.2.0)
+- better scheduler, thanks Niphlod
+- page and media preview in wiki, thanks Niphlod
+- create new auth.wiki page from slug model, thanks Nico
+- conditional menus with auth.wiki(menugroups=['wiki_editor'])
+- better security in grid/smartgrid
+- allow LOADing multiple grids, thanks Niphlod
+- auth.settings.login_onfail, thanks Yair
+- better handling of session files for speed
+- added heroku support (experimental)
+- added rocket support for IPV6, thanks Chirs Winebrinner
+- more customizable menus with MENU(li_first, li_last..)
+- added support for paymentech (gluon/contrib/paymentech.py)
+- fixed broken cron
+- fixed possible xss with share.js
+- many bug fixes. Closed more than 50 tickets since 2.2.1
+
 ## 2.2.1
 
 - session.connect(cookie_key='secret', compression_level=9) stores sessions in cookies

Makefile

@@ -29,7 +29,7 @@ update:
 	wget -O gluon/contrib/simplejsonrpc.py http://rad2py.googlecode.com/hg/ide2py/simplejsonrpc.py
 	echo "remember that pymysql was tweaked"
 src:
-	echo 'Version 2.2.1 ('`date +%Y-%m-%d\ %H:%M:%S`') stable' > VERSION
+	echo 'Version 2.3.1 ('`date +%Y-%m-%d\ %H:%M:%S`') stable' > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
@@ -42,7 +42,9 @@ src:
 	rm -f applications/examples/databases/*             
 	rm -f applications/admin/uploads/*                 
 	rm -f applications/welcome/uploads/*               
-	rm -f applications/examples/uploads/*             
+	rm -f applications/examples/uploads/* 
+	### make epydoc
+	make epydoc
 	### make welcome layout and appadmin the default
 	cp applications/welcome/views/appadmin.html applications/admin/views
 	cp applications/welcome/views/appadmin.html applications/examples/views
@@ -54,7 +56,6 @@ src:
 	cd ..; zip -r web2py/web2py_src.zip web2py/gluon/*.py web2py/gluon/contrib/* web2py/splashlogo.gif web2py/*.py web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/Makefile web2py/epydoc.css web2py/epydoc.conf web2py/app.example.yaml web2py/logging.example.conf web2py_exe.conf web2py/queue.example.yaml MANIFEST.in w2p_apps w2p_clone w2p_run startweb2py web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
 
 mdp:
-	make epydoc
 	make src
 	make app
 	make win

NEWINSTALL

@@ -0,0 +1 @@
+

VERSION

@@ -1 +1 @@
-Version 2.2.1 (2012-10-21 10:54:10) stable
+Version 2.3.1 (2012-12-14 09:21:31) stable

applications/admin/controllers/appadmin.py

@@ -396,12 +396,13 @@ def ccache():
             if value[0] < ram['oldest']:
                 ram['oldest'] = value[0]
             ram['keys'].append((key, GetInHMS(time.time() - value[0])))
-
-    locker = open(os.path.join(request.folder,
-                               'cache/cache.lock'), 'a')
+    folder = os.path.join(request.folder,'cache')
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    locker = open(os.path.join(folder, 'cache.lock'), 'a')
     portalocker.lock(locker, portalocker.LOCK_EX)
     disk_storage = shelve.open(
-        os.path.join(request.folder, 'cache/cache.shelve'))
+        os.path.join(folder, 'cache.shelve'))
     try:
         for key, value in disk_storage.items():
             if isinstance(value, dict):

applications/admin/controllers/debug.py

@@ -176,6 +176,8 @@ def toggle_breakpoint():
     try:
         filename = os.path.join(request.env['applications_parent'],
                                 'applications', request.vars.filename)
+        # normalize path name: replace slashes, references, etc...
+        filename = os.path.normpath(os.path.normcase(filename))
         if not request.vars.data:
             # ace send us the line number!
             lineno = int(request.vars.sel_start) + 1
@@ -192,6 +194,8 @@ def toggle_breakpoint():
         if lineno is not None:
             for bp in qdb_debugger.do_list_breakpoint():
                 no, bp_filename, bp_lineno, temporary, enabled, hits, cond = bp
+                # normalize path name: replace slashes, references, etc...
+                bp_filename = os.path.normpath(os.path.normcase(bp_filename))
                 if filename == bp_filename and lineno == bp_lineno:
                     err = qdb_debugger.do_clear_breakpoint(filename, lineno)
                     response.flash = T("Removed Breakpoint on %s at line %s", (

applications/admin/controllers/default.py

@@ -998,11 +998,11 @@ def design():
     statics.sort()
 
     # Get all languages
+    langpath = os.path.join(apath(app, r=request),'languages')
     languages = dict([(lang, info) for lang, info
-                      in read_possible_languages(
-                      apath(app, r=request)).iteritems()
+                      in read_possible_languages(langpath).iteritems()
                       if info[2] != 0])  # info[2] is langfile_mtime:
-                                   # get only existed files
+                                         # get only existed files
 
     #Get crontab
     cronfolder = apath('%s/cron' % app, r=request)

applications/admin/controllers/shell.py

@@ -35,7 +35,7 @@ def callback():
         except ValueError:
             return ''
     session['commands:' + app].append(command)
-    environ = env(app, True)
+    environ = env(app, True, extra_request=dict(is_https=request.is_https))
     output = gluon.contrib.shell.run(history, command, environ)
     k = len(session['commands:' + app]) - 1
     #output = PRE(output)

applications/admin/cron/expire_sessions.py

@@ -1,10 +1,10 @@
 EXPIRATION_MINUTES=60
 DIGITS=('0','1','2','3','4','5','6','7','8','9')
 import os, time, stat, cPickle, logging
-path=os.path.join(request.folder,'sessions')
+path = os.path.join(request.folder,'sessions')
 if not os.path.exists(path):
     os.mkdir(path)
-now=time.time()
+now = time.time()
 for filename in os.listdir(path):
     fullpath=os.path.join(path,filename)
     if os.path.isfile(fullpath) and filename.startswith(DIGITS):
@@ -18,4 +18,4 @@ for filename in os.listdir(path):
             if (now - filetime) > expiration:
                 os.unlink(fullpath)
         except:
-            logging.exception('failure to check %s'%fullpath)
+            logging.exception('failure to check %s' % fullpath)

applications/admin/static/css/web2py.css

@@ -61,7 +61,7 @@ input[type=text],input[type=password],select{width:300px; margin-right:5px}
     border-top:1px #DEDEDE solid;
 }
 .header {
-	// background:<fill here for header image>;
+	/* background:<fill here for header image>; */
 }
 
 
@@ -115,6 +115,10 @@ div.flash {
     z-index:2000;
 }
 
+div.flash #closeflash{color:inherit; float:right; margin-left:15px;}
+.ie-lte7  div.flash #closeflash
+{color:expression(this.parentNode.currentStyle['color']);float:none;position:absolute;right:4px;}
+
 div.flash:hover { opacity:0.25; }
 
 div.error_wrapper {display:block}
@@ -182,7 +186,7 @@ div.error {
 * will look better with the declarations below
 * if needed to remove base.css consider keeping these following lines in some css file.
 */
-// .web2py_table {border:1px solid #ccc}
+/* .web2py_table {border:1px solid #ccc} */
 .web2py_paginator {}
 .web2py_grid {width:100%}
 .web2py_grid table {width:100%}

applications/admin/static/js/web2py.js

@@ -55,7 +55,7 @@ function web2py_event_handlers() {
 jQuery(function() {
    var flash = jQuery('.flash');
    flash.hide();
-   if(flash.html()) flash.append('<span style="float:right;">&times;</span>').slideDown();
+   if(flash.html()) flash.append('<span id="closeflash">&times;</span>').slideDown();
    web2py_ajax_init(document);
    web2py_event_handlers();
 });
@@ -102,15 +102,20 @@ function web2py_ajax_page(method, action, data, target) {
       web2py_ajax_init('#'+target);
       if(command)
           eval(decodeURIComponent(command));
-      if(flash)
-          jQuery('.flash').html(decodeURIComponent(flash)).slideDown();
+      if(flash) {
+          jQuery('.flash')
+                .html(decodeURIComponent(flash))
+                .append('<span id="closeflash">&times;</span>')
+                .slideDown();
+        }
     }
   });
 }
 
 function web2py_component(action, target, timeout, times){
   jQuery(function(){
-    var element = jQuery("#" + target).get(0);
+    var jelement = jQuery("#" + target);
+    var element = jelement.get(0);
     var statement = "jQuery('#" + target + "').get(0).reload();";
     element.reload = function (){
         // Continue if times is Infinity or
@@ -119,6 +124,7 @@ function web2py_component(action, target, timeout, times){
             web2py_ajax_page('get', action, null, target);} }; // reload
     // Method to check timing limit
     element.reload_check = function (){
+        if (jelement.hasClass('w2p_component_stop')) {return false;}
         if (this.reload_counter == Infinity){return true;}
         else {
             if (!isNaN(this.reload_counter)){

applications/admin/views/debug/interact.html

@@ -0,0 +1,246 @@
+{{extend 'layout.html'}}
+
+{{block sectionclass}}debug{{end}}
+<style type="text/css">
+
+.prompt, #output {
+  width: 45em;
+  height: 1em;
+  border: 1px solid #CCCCCC;
+  font-size: 10pt;
+  margin: 0.5em;
+  padding: 0.5em;
+  padding-right: 0em;
+  overflow: auto;
+  wrap: hard;
+}
+
+#output {
+  height:150px;overflow:auto;
+}
+
+#toolbar {
+  margin-left: 0.5em;
+  padding-left: 0.5em;
+}
+
+#caret {
+  width: 2.5em;
+  margin-right: 0px;
+  padding-right: 0px;
+  border-right: 0px;
+  float: left;
+}
+
+#statement {
+  width: 43em;
+  margin-left: 1em;
+  padding-left: 0px;
+  border-left: 0px;
+  background-position: top right;
+  background-repeat: no-repeat;
+}
+
+.processing {
+  background-image: url("{{=URL('static','images/spinner.gif')}}");
+}
+
+#ajax-status {
+  font-weight: bold;
+}
+
+.message {
+  color: #8AD;
+  font-weight: bold;
+  font-style: italic;
+}
+
+.error {
+  color: #F44;
+}
+
+.username {
+  font-weight: bold;
+}
+
+</style>
+<script src="{{=URL('static', 'js/autoscroll.js')}}"></script>
+
+<div class="applist f60">
+  <div class="applist_inner">
+    <h2>{{=T("web2py online debugger")}}</h2>
+
+    {{if filename:}}
+        <h3>{{=T("Interaction at %s line %s") % (filename, lineno)}}</h3>
+
+        {{if exception:}}
+              <h3 class="exception">{{=T("Exception %s", exception['title'])}}</h3>
+        {{pass}}
+        
+        <h5>{{=T("Code listing")}}</h5>
+        {{if lines:}}
+        <pre>{{=CODE('\n'.join([x[1] for x in sorted(lines.items(),key=lambda x: x[0])]),
+                language='python', link=None, counter=min(lines.keys()), 
+                highlight_line=lineno, context_lines=10)}}</pre>
+        {{pass}}
+
+        <div class="help">
+          <ul>
+            <li>{{=T("Your application will be blocked until you click an action button (next, step, continue, etc.)")}}</li>
+            <li>{{=T("Your can inspect variables using the console bellow")}}</li>
+          </ul>
+        </div>
+
+        
+        <h3>{{=T("Interactive console")}}</h3>
+          <textarea id="output" readonly="readonly">{{=data}}</textarea>
+
+          <form id="form" action="{{=URL(r=request,f='execute',args=app)}}" method="get">
+            <div id="shellwrapper">
+              <div id="caret">&gt;&gt;&gt;</div>
+              <div id="autoscroll" style="cursor:pointer;float:right;">autoscroll</div>
+              <div class="tooltip">
+                <textarea class="prompt" name="statement" id="statement"></textarea>
+                <span>{{=T("Type python statement in here and hit Return (Enter) to execute it.")}}</span>
+              </div>
+            </div>
+          </form>
+    {{elif request.env.get('wsgi_multiprocess') or not request.env.get('wsgi_multithread'):}}
+          <h3 class="not_paused">{{=T("Unsupported webserver working mode: %s", request.env.get('server_software', ''))}}</h3>
+            <div class="help">
+                <li><b>{{=T("WARNING:")}} </b>{{=T("This debugger may not work properly if you don't have a threaded webserver or you're using multiple daemon processes.")}}</li>
+                <li>{{=T("In development, use the default Rocket webserver that is currently supported by this debugger.")}}</li>
+                <li>{{=T("On production, you'll have to configure your webserver to use one process and multiple threads to use this debugger.")}}</li>
+              </ul>
+            </div>
+            {{#=BEAUTIFY(request.env)}}
+    {{else:}}
+          <h3 class="not_paused">{{=T("No Interaction yet")}}</h3>
+            <div class="help">
+              <ul>
+                <li>{{=T("You need to set up and reach a")}} {{=A(T("breakpoint"), _href=URL('breakpoints'))}} {{=T('to use the debugger!')}}</li>
+                <li>{{=T('To emulate a breakpoint programatically, write:')}}
+                {{=CODE("from gluon.debug import dbg\n"
+                        "dbg.set_trace() # stop here!\n",
+                        counter=None)}}</li>
+                <li>{{=T('Please')}} {{=A(T("refresh"), _href=URL('interact'))}} {{=T('this page to see if a breakpoint was hit and debug interaction is required.')}}</li>
+              </ul>
+            </div>
+    {{pass}}
+  </div>
+</div>
+
+
+<div class="sidebar fl60">
+  <div class="sidebar_inner controls">
+    <span class="pwdchange">
+        {{if filename:}}
+            {{=sp_button(URL('step'), T("step"))}}
+            {{=sp_button(URL('next'), T("next"))}}
+            {{=sp_button(URL('ret'), T("return"))}}
+            {{=sp_button(URL('cont'), T("continue"))}}
+            {{=sp_button(URL('stop'), T("stop"))}}
+        {{pass}}
+            {{=button(URL('breakpoints'), T("breakpoints"))}}
+    </span>
+
+    {{if exception:}}
+        <div class="box">
+          <h3>{{=T('Exception %(extype)s: %(exvalue)s', dict(extype=exception['extype'], exvalue=exception['exvalue']))}}</h3>
+          <div class="formfield">
+               {{=CODE((exception['request']), counter=None)}}
+          </div>
+        </div>
+    {{pass}}
+
+    <div class="box">
+      <h3>{{=T('Locals##debug')}}</h3>
+      <div class="formfield">
+	{{=BEAUTIFY(f_locals)}}
+      </div>
+    </div>
+    <div class="box">
+      <h3>{{=T('Globals##debug')}}</h3>
+      <div class="formfield">
+	{{=BEAUTIFY(f_globals)}}
+      </div>
+    </div>
+  </div>
+</div>
+
+
+<script type="text/javascript">
+var bShellScrolling=0
+jQuery(document).ready(function(){
+  jQuery('#statement').focus();
+
+  jQuery('#statement').keyup(function(event){
+    var t=jQuery(this),
+    s=t.val(),
+    o=jQuery('#output'),
+    RETURN = 38;
+    if(s=='\n') t.val('');
+    if(s.length>1 && s.substr(s.length-1,1)=='\n' && s.substr(s.length-2,1)!=':' &&
+    (s.indexOf(':\n ')<0 || s.substr(s.length-2,1)=='\n')) {
+      t.val('');
+      jQuery.post("{{=URL(r=request,f='execute',args=app)}}",
+      {statement:s},function(data){o.html(o.html()+data).attr('scrollTop',o.attr('scrollHeight'));});
+    } else { };
+    if(event.keyCode==RETURN){
+      var i=s.length
+      if(i==0){
+        var s=o.find('table:last pre:first').text();
+        bShellScrolling=o.find('table').length;
+      }else if(bShellScrolling){
+        var i=bShellScrolling
+        if(i<1){
+          return
+        }else{
+          i--
+          var s=o.find('table:nth-child('+(i)+') pre:first').text();
+          bShellScrolling=i
+        }
+      }else if(s.indexOf('\n')<0){
+        var oo=o.find('tr:first-child pre:contains("'+s+'")')
+        if(oo.length==0){
+          return
+        }else if(oo.length==1){
+          s=oo.text();
+        }else{
+          sVar=oo.text();
+          o.html(o.html()+'<dd>'+s+' ?</dd><dt>'+sVar+'</dt>').attr('scrollTop',o.attr('scrollHeight'))
+          return
+        }
+      }else{
+        //multistring expr
+        return;
+      }
+      //  if(s.slice(s.length-1)=='\n'){
+      s=s.slice(0,s.length-1)
+      //	}
+      t.val(s);
+    }
+    if(bShellScrolling && event.keyCode==40){
+      var i=bShellScrolling
+      i++
+      var s=o.find('table:nth-child('+i+') tr:first-child pre').text();
+      if(s){
+        s=s.slice(0,s.length-1)
+        t.val(s);
+        bShellScrolling=i
+      }else{
+        bShellScrolling=0
+        t.val('')
+      }
+    };
+    if(bShellScrolling && (event.keyCode==37 || event.keyCode==39)){
+      bShellScrolling=0;
+    };
+    if(event.keyCode==27){
+      bShellScrolling=0;
+      t.val('');
+    };
+  });
+});
+</script>
+

applications/admin/views/shell/index.html

@@ -66,7 +66,7 @@
 <div id="wrapper">
   <textarea id="output" readonly="readonly">web2py Shell {{=request.env.web2py_version}}</textarea>
 
-  <form id="form" action="{{=URL(r=request,f='callback',args=app)}}" method="get">
+  <form id="form" action="{{=URL('callback',args=app)}}" method="get">
     <div id="shellwrapper">
       <div id="caret">&gt;&gt;&gt;</div>
       <div class="tooltip">
@@ -103,7 +103,7 @@ jQuery(document).ready(function(){
     if(s.length>1 && s.substr(s.length-1,1)=='\n' && s.substr(s.length-2,1)!=':' &&
     (s.indexOf(':\n ')<0 || s.substr(s.length-2,1)=='\n')) {
       t.val('');
-      jQuery.post("{{=URL(r=request,f='callback',args=app)}}",
+      jQuery.post("{{=URL('callback',args=app)}}",
       {statement:s},function(data){o.html(o.html()+data).attr('scrollTop',o.attr('scrollHeight'));});
     } else { };
     if(event.keyCode==RETURN){

applications/examples/controllers/appadmin.py

@@ -396,12 +396,13 @@ def ccache():
             if value[0] < ram['oldest']:
                 ram['oldest'] = value[0]
             ram['keys'].append((key, GetInHMS(time.time() - value[0])))
-
-    locker = open(os.path.join(request.folder,
-                               'cache/cache.lock'), 'a')
+    folder = os.path.join(request.folder,'cache')
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    locker = open(os.path.join(folder, 'cache.lock'), 'a')
     portalocker.lock(locker, portalocker.LOCK_EX)
     disk_storage = shelve.open(
-        os.path.join(request.folder, 'cache/cache.shelve'))
+        os.path.join(folder, 'cache.shelve'))
     try:
         for key, value in disk_storage.items():
             if isinstance(value, dict):

applications/examples/private/content/en/default/documentation/more.markmin

@@ -7,6 +7,7 @@
 - [[User Voice http://web2py.uservoice.com/ popup]]
 
 #### Learning and Demos
+- [[Intro video http://www.youtube.com/watch?v=BXzqmHx6edY]] and [[code examples https://github.com/mjhea0/web2py]]
 - [[Killer Web Development Tutorial http://killer-web-development.com/]]
 - [[Admin Demo http://www.web2py.com/demo_admin popup]] (web-based IDE)
 - [[Welcome App Demo http://www.web2py.com/welcome]] (scaffolding application)

applications/examples/static/css/web2py.css

@@ -61,7 +61,7 @@ input[type=text],input[type=password],select{width:300px; margin-right:5px}
     border-top:1px #DEDEDE solid;
 }
 .header {
-	// background:<fill here for header image>;
+	/* background:<fill here for header image>; */
 }
 
 
@@ -115,6 +115,10 @@ div.flash {
     z-index:2000;
 }
 
+div.flash #closeflash{color:inherit; float:right; margin-left:15px;}
+.ie-lte7  div.flash #closeflash
+{color:expression(this.parentNode.currentStyle['color']);float:none;position:absolute;right:4px;}
+
 div.flash:hover { opacity:0.25; }
 
 div.error_wrapper {display:block}
@@ -182,7 +186,7 @@ div.error {
 * will look better with the declarations below
 * if needed to remove base.css consider keeping these following lines in some css file.
 */
-// .web2py_table {border:1px solid #ccc}
+/* .web2py_table {border:1px solid #ccc} */
 .web2py_paginator {}
 .web2py_grid {width:100%}
 .web2py_grid table {width:100%}

applications/examples/static/js/web2py.js

@@ -55,7 +55,7 @@ function web2py_event_handlers() {
 jQuery(function() {
    var flash = jQuery('.flash');
    flash.hide();
-   if(flash.html()) flash.append('<span style="float:right;">&times;</span>').slideDown();
+   if(flash.html()) flash.append('<span id="closeflash">&times;</span>').slideDown();
    web2py_ajax_init(document);
    web2py_event_handlers();
 });
@@ -102,15 +102,20 @@ function web2py_ajax_page(method, action, data, target) {
       web2py_ajax_init('#'+target);
       if(command)
           eval(decodeURIComponent(command));
-      if(flash)
-          jQuery('.flash').html(decodeURIComponent(flash)).slideDown();
+      if(flash) {
+          jQuery('.flash')
+                .html(decodeURIComponent(flash))
+                .append('<span id="closeflash">&times;</span>')
+                .slideDown();
+        }
     }
   });
 }
 
 function web2py_component(action, target, timeout, times){
   jQuery(function(){
-    var element = jQuery("#" + target).get(0);
+    var jelement = jQuery("#" + target);
+    var element = jelement.get(0);
     var statement = "jQuery('#" + target + "').get(0).reload();";
     element.reload = function (){
         // Continue if times is Infinity or
@@ -119,6 +124,7 @@ function web2py_component(action, target, timeout, times){
             web2py_ajax_page('get', action, null, target);} }; // reload
     // Method to check timing limit
     element.reload_check = function (){
+        if (jelement.hasClass('w2p_component_stop')) {return false;}
         if (this.reload_counter == Infinity){return true;}
         else {
             if (!isNaN(this.reload_counter)){

applications/examples/views/form_examples/form.html

@@ -0,0 +1,9 @@
+{{extend 'layout.html'}}
+
+<h2>form.vars</h2>
+<pre>{{=vars}}</pre>
+
+<h2>form</h2>
+
+{{=form}}
+

applications/examples/views/generic.xml

@@ -1,15 +1 @@
-{{
-###
-# response._vars contains the dictionary returned by thecontroller action
-###
-try:
-   from gluon.serializers import xml
-   response.write(xml(response._vars), escape=False)
-   response.headers['Content-Type'] = 'text/xml'
-except (TypeError, ValueError):
-	raise HTTP(405, 'XML serialization error')
-except ImportError:
-	raise HTTP(405, 'XML not available')
-except:
-	raise HTTP(405, 'XML error')
-}}
+{{from gluon.serializers import xml}}{{=XML(xml(response._vars,quote=False))}}

applications/examples/views/simple_examples/status.html

@@ -0,0 +1,3 @@
+{{extend 'layout.html'}}
+
+{{=toolbar}}

applications/welcome/controllers/appadmin.py

@@ -396,12 +396,13 @@ def ccache():
             if value[0] < ram['oldest']:
                 ram['oldest'] = value[0]
             ram['keys'].append((key, GetInHMS(time.time() - value[0])))
-
-    locker = open(os.path.join(request.folder,
-                               'cache/cache.lock'), 'a')
+    folder = os.path.join(request.folder,'cache')
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    locker = open(os.path.join(folder, 'cache.lock'), 'a')
     portalocker.lock(locker, portalocker.LOCK_EX)
     disk_storage = shelve.open(
-        os.path.join(request.folder, 'cache/cache.shelve'))
+        os.path.join(folder, 'cache.shelve'))
     try:
         for key, value in disk_storage.items():
             if isinstance(value, dict):

applications/welcome/models/db.py

@@ -11,7 +11,7 @@
 
 if not request.env.web2py_runtime_gae:
     ## if NOT running on Google App Engine use SQLite or other DB
-    db = DAL('sqlite://storage.sqlite')
+    db = DAL('sqlite://storage.sqlite',pool_size=1,check_reserved=['all'])
 else:
     ## connect to Google BigTable (optional 'google:datastore://namespace')
     db = DAL('google:datastore')

applications/welcome/models/menu.py

@@ -5,6 +5,8 @@
 ## Customize your APP title, subtitle and menus here
 #########################################################################
 
+response.logo = A(B('web',SPAN(2),'py'),XML('™ '),
+                  _class="brand",_href="http://www.web2py.com/")
 response.title = ' '.join(
     word.capitalize() for word in request.application.split('_'))
 response.subtitle = T('customize me!')
@@ -26,11 +28,12 @@ response.menu = [
     (T('Home'), False, URL('default', 'index'), [])
 ]
 
+DEVELOPMENT_MENU = True
+
 #########################################################################
 ## provide shortcuts for development. remove in production
 #########################################################################
 
-
 def _():
     # shortcuts
     app = request.application
@@ -133,4 +136,4 @@ def _():
                         ])
                 ]
          )]
-_()
+if DEVELOPMENT_MENU: _()

applications/welcome/static/css/web2py.css

@@ -115,6 +115,10 @@ div.flash {
     z-index:2000;
 }
 
+div.flash #closeflash{color:inherit; float:right; margin-left:15px;}
+.ie-lte7  div.flash #closeflash
+{color:expression(this.parentNode.currentStyle['color']);float:none;position:absolute;right:4px;}
+
 div.flash:hover { opacity:0.25; }
 
 div.error_wrapper {display:block}

applications/welcome/static/css/web2py_bootstrap.css

@@ -1,5 +1,5 @@
 /*=============================================================
-                       CUSTOM RULES 
+                       CUSTOM RULES
 ==============================================================*/
 
 body{height:auto;} /* to avoid vertical scroll bar */
@@ -39,7 +39,7 @@ div.flash.flash-top,div.flash.flash-top:hover{
 /* auth navbar - primitive style */
 .auth_navbar,.auth_navbar a{color:inherit;}
 .ie-lte7 .auth_navbar,.auth_navbar a{color:expression(this.parentNode.currentStyle['color']); /* ie7 doesn't support inherit */}
-.auth_navbar a{white-space:nowrap;} /* to avoid the nav split on more lines */ 
+.auth_navbar a{white-space:nowrap;} /* to avoid the nav split on more lines */
 .auth_navbar a:hover{color:white;text-decoration:none;}
 ul#navbar>.auth_navbar{
   display:inline-block;
@@ -51,7 +51,7 @@ div.error_wrapper .error{
   border-radius: 4px;
   -o-border-radius: 4px;
   -moz-border-radius: 4px;
-  -webkit-border-radius: 4px;  
+  -webkit-border-radius: 4px;
 }
 /* below rules are only for formstyle = bootstrap
 trying to make errors look like bootstrap ones */
@@ -67,10 +67,13 @@ div.controls .error{
   border:none;
   padding:0;
   margin:0;
-  //display:inline; /* uncommenting this, the animation effect is lost */
+  /*display:inline;*/ /* uncommenting this, the animation effect is lost */
 }
-div.controls .inline-help{color:#3A87AD;}
-div.controls .error_wrapper+.inline-help{margin-left:-99999px;}
+div.controls .help-inline{color:#3A87AD;}
+div.controls .error_wrapper +.help-inline {margin-left:-99999px;}
+div.controls select +.error_wrapper {margin-left:5px;}
+.ie-lte7 div.error{color:#fff;}
+
 /* beautify brand */
 .navbar-inverse .brand{color:#c6cecc;}
 .navbar-inverse .brand b{display:inline-block;margin-top:-1px;}
@@ -89,13 +92,13 @@ a{white-space:normal;}
 li{margin-bottom:0;}
 textarea,button{display:block;}
 /*reset ul padding */
-ul#navbar{padding:0;} 
+ul#navbar{padding:0;}
 /* label aligned to related input */
 td.w2p_fl,td.w2p_fc {padding:0;}
 #web2py_user_form td{vertical-align:middle;}
 
 /*=============================================================
-                 OVERRIDING BOOTSTRAP.CSS RULES 
+                 OVERRIDING BOOTSTRAP.CSS RULES
 ==============================================================*/
 
 /* because web2py handles this via js */
@@ -175,9 +178,6 @@ background-repeat:repeat;
                        OTHER RULES
 ==============================================================*/
 
-.navbar-inner{
-  position:relative; /*unnecessary ??*/
-}
 /* Massimo Di Pierro fixed alignment in forms with list:string */
 form table tr{margin-bottom:9px;}
 td.w2p_fw ul{margin-left:0px;}
@@ -189,8 +189,8 @@ td.w2p_fw ul{margin-left:0px;}
   margin-bottom: 0;
   vertical-align: middle;
 }
-.web2py_console input[type="submit"], 
-.web2py_console input[type="button"], 
+.web2py_console input[type="submit"],
+.web2py_console input[type="button"],
 .web2py_console button{
   padding-top:4px;
   padding-bottom:4px;
@@ -212,7 +212,7 @@ td.w2p_fw ul{margin-left:0px;}
 
 @media only screen and (max-width:979px){
   body{padding-top:0px;}
-  #navbar{bottom:-10px;left:4px;}
+  #navbar{top:5px;}
   div.flash{right:5px;}
   .dropdown-menu ul{visibility:visible;}
 }
@@ -224,8 +224,8 @@ td.w2p_fw ul{margin-left:0px;}
   .navbar-fixed-top,.navbar-fixed-bottom {
     margin-left:-10px;
     margin-right:-10px;
-  }  
+  }
   input[type="text"],input[type="password"],select{
     width:95%;
   }
-}
+}

applications/welcome/static/js/share.js

@@ -16,7 +16,7 @@ jQuery(function(){
             return match && decodeURIComponent(match[1].replace(/\+/g, ' '))||default_value;
         }
 	var path = params('static','social');
-	var url = window.location.href;
+	var url = encodeURIComponent(window.location.href);
 	var host =  window.location.hostname;
 	var title = escape(jQuery('title').text());
 	var twit = 'http://twitter.com/home?status='+title+'%20'+url;

applications/welcome/static/js/web2py.js

@@ -55,7 +55,7 @@ function web2py_event_handlers() {
 jQuery(function() {
    var flash = jQuery('.flash');
    flash.hide();
-   if(flash.html()) flash.append('<span style="float:right;">&times;</span>').slideDown();
+   if(flash.html()) flash.append('<span id="closeflash">&times;</span>').slideDown();
    web2py_ajax_init(document);
    web2py_event_handlers();
 });
@@ -102,15 +102,20 @@ function web2py_ajax_page(method, action, data, target) {
       web2py_ajax_init('#'+target);
       if(command)
           eval(decodeURIComponent(command));
-      if(flash)
-          jQuery('.flash').html(decodeURIComponent(flash)).slideDown();
+      if(flash) {
+          jQuery('.flash')
+                .html(decodeURIComponent(flash))
+                .append('<span id="closeflash">&times;</span>')
+                .slideDown();
+        }
     }
   });
 }
 
 function web2py_component(action, target, timeout, times){
   jQuery(function(){
-    var element = jQuery("#" + target).get(0);
+    var jelement = jQuery("#" + target);
+    var element = jelement.get(0);
     var statement = "jQuery('#" + target + "').get(0).reload();";
     element.reload = function (){
         // Continue if times is Infinity or
@@ -119,6 +124,7 @@ function web2py_component(action, target, timeout, times){
             web2py_ajax_page('get', action, null, target);} }; // reload
     // Method to check timing limit
     element.reload_check = function (){
+        if (jelement.hasClass('w2p_component_stop')) {return false;}
         if (this.reload_counter == Infinity){return true;}
         else {
             if (!isNaN(this.reload_counter)){

applications/welcome/views/generic.xml

@@ -1 +1 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>{{from gluon.serializers import xml}}{{=XML(xml(response._vars,quote=False))}}
+{{from gluon.serializers import xml}}{{=XML(xml(response._vars,quote=False))}}

applications/welcome/views/layout.html

@@ -29,7 +29,7 @@
         device-width: Occupy full width of the screen in its current orientation
         initial-scale = 1.0 retains dimensions instead of zooming out if page height > device height
         user-scalable = yes allows the user to zoom in -->
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
   <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
   <link rel="apple-touch-icon" href="{{=URL('static','images/favicon.png')}}">
@@ -76,7 +76,7 @@
           <span class="icon-bar"></span>
           <span class="icon-bar"></span>
         </button>
-        <a class="brand" href="http://www.web2py.com/"><b>web<span>2</span>py</b>&trade;&nbsp;</a>
+	{{=response.logo or ''}}
         <ul id="navbar" class="nav pull-right">{{='auth' in globals() and auth.navbar(mode="dropdown") or ''}}</ul>
         <div class="nav-collapse">
           {{is_mobile=request.user_agent().is_mobile}}
@@ -152,7 +152,14 @@
       <script src="{{=URL('static','js/dd_belatedpng.js')}}"></script>
       <script> DD_belatedPNG.fix('img, .png_bg'); //fix any <img> or .png_bg background-images </script>
       <![endif]-->
-  {{if response.google_analytics_id:}}<script type="text/javascript"> var _gaq = _gaq || []; _gaq.push(['_setAccount', '{{=response.google_analytics_id}}']); _gaq.push(['_trackPageview']); (function() { var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); })(); </script> {{pass}}
+  
+  {{if response.google_analytics_id:}}
+  <script src="{{=URL('static','js/analytics.js')}}"></script>
+  <script type="text/javascript">
+  analytics.initialize({
+    'Google Analytics':{trackingId:'{{=response.google_analytics_id}}'} 
+  });</script>
+  {{pass}}
   <script src="{{=URL('static','js/share.js',vars=dict(static=URL('static','images')))}}"></script>
 </body>
 </html>

gluon/admin.py

@@ -189,9 +189,10 @@ def app_create(app, request, force=False, key=None, info=False):
             return False
     try:
         w2p_unpack('welcome.w2p', path)
-        for subfolder in ['models', 'views', 'controllers', 'databases',
-                          'modules', 'cron', 'errors', 'sessions',
-                          'languages', 'static', 'private', 'uploads']:
+        for subfolder in [
+            'models', 'views', 'controllers', 'databases',
+            'modules', 'cron', 'errors', 'sessions', 'cache',
+            'languages', 'static', 'private', 'uploads']:
             subpath = os.path.join(path, subfolder)
             if not os.path.exists(subpath):
                 os.mkdir(subpath)

gluon/cache.py

@@ -147,6 +147,7 @@ class CacheInRam(CacheAbstract):
     def __init__(self, request=None):
         self.initialized = False
         self.request = request
+        self.storage = {}
 
     def initialize(self):
         if self.initialized:
@@ -303,6 +304,7 @@ class CacheOnDisk(CacheAbstract):
         self.initialized = False
         self.request = request
         self.folder = folder
+        self.storage = {}
 
     def initialize(self):
         if self.initialized:

gluon/compileapp.py

@@ -396,7 +396,8 @@ def build_environment(request, response, session, store_current=True):
     response.models_to_run = [r'^\w+\.py$', r'^%s/\w+\.py$' % request.controller,
                               r'^%s/%s/\w+\.py$' % (request.controller, request.function)]
 
-    t = environment['T'] = translator(request)
+    t = environment['T'] = translator(os.path.join(request.folder,'languages'),
+                                      request.env.http_accept_language)
     c = environment['cache'] = Cache(request)
 
     if store_current:
@@ -452,7 +453,7 @@ def compile_views(folder):
     """
 
     path = pjoin(folder, 'views')
-    for file in listdir(path, '^[\w/\-]+(\.\w+)+$'):
+    for file in listdir(path, '^[\w/\-]+(\.\w+)*$'):
         try:
             data = parse_template(file, path)
         except Exception, e:
@@ -514,11 +515,11 @@ def run_models_in(environment):
         for model in listdir(cpath, '^models_\w+\.pyc$', 0):
             restricted(read_pyc(model), environment, layer=model)
         path = pjoin(cpath, 'models')
-        models = listdir(path, '^\w+\.pyc$', 0, sort=False)
+        models = listdir(path, '^\w+\.pyc$', 0)
         compiled = True
     else:
         path = pjoin(folder, 'models')
-        models = listdir(path, '^\w+\.py$', 0, sort=False)
+        models = listdir(path, '^\w+\.py$', 0)
         compiled = False
     n = len(path) + 1
     for model in models:

gluon/contenttype.py

@@ -644,6 +644,7 @@ CONTENT_TYPE = {
     '.wmls': 'text/vnd.wap.wmlscript',
     '.wmv': 'video/x-ms-wmv',
     '.wmx': 'audio/x-ms-asx',
+    '.woff': 'application/font-woff',
     '.wp': 'application/vnd.wordperfect',
     '.wp4': 'application/vnd.wordperfect',
     '.wp5': 'application/vnd.wordperfect',

gluon/contrib/aes.py

@@ -64,7 +64,7 @@ def new(key, mode=MODE_CBC, IV=None):
         return ECBMode(AES(key))
     elif mode == MODE_CBC:
         if IV is None:
-            raise ValueError, "CBC mode needs an IV value!"
+            raise ValueError("CBC mode needs an IV value!")
 
         return CBCMode(AES(key), IV)
     else:
@@ -91,7 +91,7 @@ class AES(object):
         elif self.key_size == 32:
             self.rounds = 14
         else:
-            raise ValueError, "Key length must be 16, 24 or 32 bytes"
+            raise ValueError("Key length must be 16, 24 or 32 bytes")
 
         self.expand_key()
 
@@ -313,7 +313,7 @@ class ECBMode(object):
         """Perform ECB mode with the given function"""
 
         if len(data) % self.block_size != 0:
-            raise ValueError, "Plaintext length must be multiple of 16"
+            raise ValueError("Plaintext length must be multiple of 16")
 
         block_size = self.block_size
         data = array('B', data)
@@ -357,7 +357,7 @@ class CBCMode(object):
 
         block_size = self.block_size
         if len(data) % block_size != 0:
-            raise ValueError, "Plaintext length must be multiple of 16"
+            raise ValueError("Plaintext length must be multiple of 16")
 
         data = array('B', data)
         IV = self.IV
@@ -381,7 +381,7 @@ class CBCMode(object):
 
         block_size = self.block_size
         if len(data) % block_size != 0:
-            raise ValueError, "Ciphertext length must be multiple of 16"
+            raise ValueError("Ciphertext length must be multiple of 16")
 
         data = array('B', data)
         IV = self.IV

gluon/contrib/autolinks.py

@@ -146,7 +146,6 @@ def oembed(url):
             oembed = v + '?format=json&url=' + cgi.escape(url)
             try:
                 data = urllib.urlopen(oembed).read()
-                print data
                 return loads(data)  # json!
             except:
                 pass

gluon/contrib/fpdf/html.py

@@ -26,8 +26,9 @@ def hex2dec(color = "#000000"):
 class HTML2FPDF(HTMLParser):
     "Render basic HTML to FPDF"
 
-    def __init__(self, pdf):
+    def __init__(self, pdf, image_map = None):
         HTMLParser.__init__(self)
+        self.image_map = image_map or (lambda src: src)
         self.style = {}
         self.pre = False
         self.href = ''
@@ -265,7 +266,8 @@ class HTML2FPDF(HTMLParser):
                 h = px2mm(attrs.get('height',0))
                 if self.align and self.align[0].upper() == 'C':
                     x = (self.pdf.w-x)/2.0 - w/2.0
-                self.pdf.image(attrs['src'], x, y, w, h, link=self.href)
+                self.pdf.image(self.image_map(attrs['src']),
+                               x, y, w, h, link=self.href)
                 self.pdf.set_x(x+w)
                 self.pdf.set_y(y+h)
         if tag=='b' or tag=='i' or tag=='u':
@@ -389,9 +391,9 @@ class HTML2FPDF(HTMLParser):
         self.pdf.ln(3)
 
 class HTMLMixin(object):
-    def write_html(self, text):
+    def write_html(self, text, image_map=None):
         "Parse HTML and convert it to PDF"
-        h2p = HTML2FPDF(self)
+        h2p = HTML2FPDF(self, image_map)
         h2p.feed(text)
 
 

gluon/contrib/google_wallet.py

@@ -1,16 +1,15 @@
 from gluon import XML
 
-
 def button(merchant_id="123456789012345",
            products=[dict(name="shoes",
                           quantity=1,
                           price=23.5,
                           currency='USD',
                           description="running shoes black")]):
-    t = '<input name="item_%(key)s_%(k)s" type="hidden" value="%(value)s"/>'
+    t = '<input name="item_%(key)s_%(k)s" type="hidden" value="%(value)s"/>\n'
     list_products = ''
     for k, product in enumerate(products):
-        for key, value in product.items():
-            list_products += t % dict(k=k + 1, key=key, value=value)
-    button = '<form action="https://checkout.google.com/api/checkout/v2/checkoutForm/Merchant/%s" id="BB_BuyButtonForm" method="post" name="BB_BuyButtonForm" target="_top">%s<input name="_charset_" type="hidden" value="utf-8"/><input alt="" src="https://checkout.google.com/buttons/buy.gif?merchant_id=%s&amp;w=117&amp;h=48&amp;style=white&amp;variant=text&amp;loc=en_US" type="image"/></form>' % (merchant_id, list_products, merchant_id)
+        for key in ('name','description','quantity','price','currency'):
+            list_products += t % dict(k=k + 1, key=key, value=product[key])
+    button = """<form action="https://checkout.google.com/api/checkout/v2/checkoutForm/Merchant/%(merchant_id)s" id="BB_BuyButtonForm" method="post" name="BB_BuyButtonForm" target="_top">\n%(list_products)s<input name="_charset_" type="hidden" value="utf-8"/>\n<input alt="" src="https://checkout.google.com/buttons/buy.gif?merchant_id=%(merchant_id)s&amp;w=117&amp;h=48&amp;style=white&amp;variant=text&amp;loc=en_US" type="image"/>\n</form>""" % dict(merchant_id=merchant_id, list_products=list_products)
     return XML(button)

gluon/contrib/heroku.py

@@ -0,0 +1,29 @@
+"""
+Usage: in web2py models/db.py
+
+from gluon.contrib.heroku import get_db
+db = get_db()
+
+"""
+import os
+from gluon import *
+from gluon.dal import ADAPTERS, UseDatabaseStoredFile,PostgreSQLAdapter
+
+class HerokuPostgresAdapter(UseDatabaseStoredFile,PostgreSQLAdapter):
+    drivers = ('psycopg2',)
+    uploads_in_blob = True
+
+ADAPTERS['postgres'] = HerokuPostgresAdapter
+
+def get_db(name = None, pool_size=10):
+    if not name:
+        names = [n for n in os.environ.keys() 
+                 if n[:18]+n[-4:]=='HEROKU_POSTGRESQL__URL']
+        if names:
+            name = names[0]
+    if name:
+        db = DAL(os.environ[name], pool_size=pool_size)
+        current.session.connect(current.request, current.response, db=db)
+    else:
+        db = DAL('sqlite://heroku.test.sqlite')
+    return db

gluon/contrib/login_methods/dropbox_account.py

@@ -99,13 +99,21 @@ class DropboxAccount(object):
         redirect('https://www.dropbox.com/logout')
         return next
 
+    def get_client(self):
+        access_token = current.session.dropbox_access_token
+        self.sess.set_token(access_token[0], access_token[1])
+        self.client = client.DropboxClient(self.sess)
+
     def put(self, filename, file):
+        if not hasattr(self,'client'): self.get_client()
         return json.loads(self.client.put_file(filename, file))['bytes']
 
     def get(self, filename, file):
+        if not hasattr(self,'client'): self.get_client()
         return self.client.get_file(filename)
 
     def dir(self, path):
+        if not hasattr(self,'client'): self.get_client()
         return json.loads(self.client.metadata(path))
 
 

gluon/contrib/login_methods/ldap_auth.py

@@ -222,7 +222,8 @@ def ldap_auth(server='ldap', port=None,
                 con.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
                 # In cases where ForestDnsZones and DomainDnsZones are found,
                 # result will look like the following:
-                # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones,DC=domain,DC=com']
+                # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones,
+                #    DC=domain,DC=com']
                 if ldap_binddn:
                     # need to search directory with an admin account 1st
                     con.simple_bind_s(ldap_binddn, ldap_bindpw)
@@ -238,8 +239,9 @@ def ldap_auth(server='ldap', port=None,
                                            user_mail_attrib])
                 result = con.search_ext_s(
                     ldap_basedn, ldap.SCOPE_SUBTREE,
-                    "(&(sAMAccountName=%s)(%s))" % (ldap.filter.escape_filter_chars(username_bare),
-                                                    filterstr),
+                    "(&(sAMAccountName=%s)(%s))" % (
+                                ldap.filter.escape_filter_chars(username_bare),
+                                filterstr),
                     requested_attrs)[0][1]
                 if not isinstance(result, dict):
                     # result should be a dict in the form
@@ -292,8 +294,9 @@ def ldap_auth(server='ldap', port=None,
                 # bind anonymously
                 con.simple_bind_s(dn, pw)
                 # search by e-mail address
-                filter = '(&(mail=%s)(%s))' % (ldap.filter.escape_filter_chars(username),
-                                               filterstr)
+                filter = '(&(mail=%s)(%s))' % (
+                                ldap.filter.escape_filter_chars(username),
+                                filterstr)
                 # find the uid
                 attrs = ['uid']
                 if manage_user:
@@ -330,8 +333,10 @@ def ldap_auth(server='ldap', port=None,
                             break
                     except ldap.LDAPError, detail:
                         (exc_type, exc_value) = sys.exc_info()[:2]
-                        logger.warning("ldap_auth: searching %s for %s resulted in %s: %s\n" %
-                                       (basedn, filter, exc_type, exc_value))
+                        logger.warning(
+                        "ldap_auth: searching %s for %s resulted in %s: %s\n" %
+                                       (basedn, filter, exc_type, exc_value)
+                                       )
                 if not found:
                     logger.warning('User [%s] not found!' % username)
                     return False
@@ -365,8 +370,10 @@ def ldap_auth(server='ldap', port=None,
                             break
                     except ldap.LDAPError, detail:
                         (exc_type, exc_value) = sys.exc_info()[:2]
-                        logger.warning("ldap_auth: searching %s for %s resulted in %s: %s\n" %
-                                       (basedn, filter, exc_type, exc_value))
+                        logger.warning(
+                        "ldap_auth: searching %s for %s resulted in %s: %s\n" %
+                                       (basedn, filter, exc_type, exc_value)
+                                       )
                 if not found:
                     logger.warning('User [%s] not found!' % username)
                     return False
@@ -425,6 +432,8 @@ def ldap_auth(server='ldap', port=None,
                 if not do_manage_groups(username, password):
                     return False
             return True
+        except ldap.INVALID_CREDENTIALS, e:
+            return False
         except ldap.LDAPError, e:
             import traceback
             logger.warning('[%s] Error in ldap processing' % str(username))
@@ -502,8 +511,8 @@ def ldap_auth(server='ldap', port=None,
                     'There is no username or email for %s!' % username)
                 raise
             db_group_search = db((db.auth_membership.user_id == db_user_id) &
-                                (db.auth_user.id == db.auth_membership.user_id) &
-                                 (db.auth_group.id == db.auth_membership.group_id))
+                            (db.auth_user.id == db.auth_membership.user_id) &
+                            (db.auth_group.id == db.auth_membership.group_id))
             db_groups_of_the_user = list()
             db_group_id = dict()
 
@@ -522,7 +531,8 @@ def ldap_auth(server='ldap', port=None,
             for group_to_del in db_groups_of_the_user:
                 if ldap_groups_of_the_user.count(group_to_del) == 0:
                     db((db.auth_membership.user_id == db_user_id) &
-                       (db.auth_membership.group_id == db_group_id[group_to_del])).delete()
+                       (db.auth_membership.group_id == \
+                         db_group_id[group_to_del])).delete()
 
             #
             # Create user membership in groups where user is not in already
@@ -531,7 +541,7 @@ def ldap_auth(server='ldap', port=None,
                 if db_groups_of_the_user.count(group_to_add) == 0:
                     if db(db.auth_group.role == group_to_add).count() == 0:
                         gid = db.auth_group.insert(role=group_to_add,
-                                                   description='Generated from LDAP')
+                                            description='Generated from LDAP')
                     else:
                         gid = db(db.auth_group.role == group_to_add).select(
                             db.auth_group.id).first().id
@@ -608,7 +618,8 @@ def ldap_auth(server='ldap', port=None,
             con.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
             # In cases where ForestDnsZones and DomainDnsZones are found,
             # result will look like the following:
-            # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones,DC=domain,DC=com']
+            # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones,
+            #     DC=domain,DC=com']
             if ldap_binddn:
                 # need to search directory with an admin account 1st
                 con.simple_bind_s(ldap_binddn, ldap_bindpw)
@@ -620,7 +631,8 @@ def ldap_auth(server='ldap', port=None,
             # We have to use the full string
             username = con.search_ext_s(base_dn, ldap.SCOPE_SUBTREE,
                                         "(&(sAMAccountName=%s)(%s))" %
-                                        (ldap.filter.escape_filter_chars(username_bare), filterstr), ["cn"])[0][0]
+                            (ldap.filter.escape_filter_chars(username_bare),
+                            filterstr), ["cn"])[0][0]
         else:
             if ldap_binddn:
                 # need to search directory with an bind_dn account 1st
@@ -630,7 +642,9 @@ def ldap_auth(server='ldap', port=None,
                 con.simple_bind_s('', '')
 
         # search for groups where user is in
-        filter = '(&(%s=%s)(%s))' % (ldap.filter.escape_filter_chars(group_member_attrib),
+        filter = '(&(%s=%s)(%s))' % (ldap.filter.escape_filter_chars(
+                                                            group_member_attrib
+                                                            ),
                                      ldap.filter.escape_filter_chars(username),
                                      group_filterstr)
         group_search_result = con.search_s(group_dn,
@@ -648,3 +662,4 @@ def ldap_auth(server='ldap', port=None,
     if filterstr[0] == '(' and filterstr[-1] == ')':  # rfc4515 syntax
         filterstr = filterstr[1:-1]  # parens added again where used
     return ldap_auth_aux
+

gluon/contrib/markmin/markmin2html.py

@@ -1262,8 +1262,9 @@ def render(text,
         t = t or ''
         a = escape(a) if a else ''
         if k:
-            if k.startswith('#'):
-                k = '#'+id_prefix+k[1:]
+            if '#' in k and not ':' in k.split('#')[0]: 
+                # wikipage, not external url
+                k=k.replace('#','#'+id_prefix)
             k = escape(k)
             title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
             target = ' target="_blank"' if p == 'popup' else ''

gluon/contrib/paymentech.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This module provides a simple API for Paymentech(c) payments
+# The original code was taken from this web2py issue post
+# http://code.google.com/p/web2py/issues/detail?id=1170 by Adnan Smajlovic
+#
+#    Copyright (C) <2012>  Alan Etkin <spametki@gmail.com>
+#    License: BSD
+#
+
+import sys, httplib, urllib, urllib2
+from xml.dom.minidom import parseString
+
+# TODO: input validation, test, debugging output
+
+class PaymenTech(object):
+    """
+    The base class for connecting to the Paymentech service
+
+    Format notes
+    ============
+
+    - Credit card expiration date (exp argument) must be of mmyyyy form
+    - The amount is an all integers string with two decimal places:
+      For example, $2.15 must be formatted as "215"
+
+    Point of sale and service options (to be passed on initialization)
+    ==================================================================
+
+    user
+    password
+    industry
+    message
+    bin_code
+    merchant
+    terminal
+
+    (WARNING!: this is False by default)
+    development <bool>
+
+    (the following arguments have default values)
+    target
+    host
+    api_url
+
+
+    Testing
+    =======
+
+    As this module consumes webservice methods, it should be tested
+    with particular user data with the paymentech development environment
+
+    The simplest test would be running something like the following:
+
+    from paymentech import PaymenTech
+    # Read the basic point of sale argument list required above
+    # Remember to use development = True!
+    pos_data = {'user': <username>, ...}
+
+    # The data arguments are documented in the .charge() method help
+    charge_test = {'account': <account>, ...}
+    mypayment = PaymentTech(**pos_data)
+    result = mypayment.charge(**charge_test)
+
+    print "##################################"
+    print "#       Charge test result       #"
+    print "##################################"
+    print result
+
+    #################################################################
+    #               Notes for web2py implementations                #
+    #################################################################    
+
+    # A recommended model for handling payments
+
+    # Store this constants in a private model file (i.e. 0_private.py)
+
+    PAYMENTECH_USER = <str>
+    PAYMENTECH_PASSWORD = <str>
+    PAYMENTECH_INDUSTRY = <str>
+    PAYMENTECH_MESSAGE = <str>
+    PAYMENTECH_BIN_CODE= <str>
+    PAYMENTECH_MERCHANT = <str>
+    PAYMENTECH_terminal = <str>
+    DEVELOPMENT = True
+    PAYMENTECH_TARGET = <str>
+    PAYMENTECH_HOST = <str>
+    PAYMENTECH_API_URL = <str>
+    
+    # The following table would allow passing data with web2py and to
+    # update records with the webservice authorization output by using
+    # the DAL
+    #
+    #   For example:
+    #
+    #   # Create a PaymenTech instance
+    #   mypaymentech = paymentech.PaymenTech(user=PAYMENTECH_USER, ...)
+    #
+    #   # Fetch a payment inserted within the app
+    #   myrow = db.paymentech[<id>]
+    #
+    #   # Send the authorization request to the webservice
+    #   result = mypaymentech.charge(myrow.as_dict())
+    #   
+    #   # Update the db record with the webservice response
+    #   myrow.update_record(**result)
+    
+    db.define_table("paymentech",
+        Field("account"),
+        Field("exp", comment="Must be of the mmyyyy form"),
+        Field("currency_code"),
+        Field("currency_exponent"),
+        Field("card_sec_val_ind"),
+        Field("card_sec_val"),
+        Field("avs_zip"),
+        Field("avs_address_1"),
+        Field("avs_address_2"),
+        Field("avs_city"),
+        Field("avs_state"),
+        Field("avs_phone"),
+        Field("avs_country"),
+        Field("profile_from_order_ind"),
+        Field("profile_order_override_ind"),
+        Field("order_id"),
+        Field("amount",
+              comment="all integers with two decimal digits, \
+                                 without dot separation"),
+        Field("header"),
+        Field("status_code"),
+        Field("status_message"),
+        Field("resp_code"),
+        Field("tx_ref_num"),
+        format="%(order_id)s")
+
+    TODO: add model form validators (for exp date and amount)
+    """
+
+    charge_xml = """
+        <?xml version="1.0" encoding="UTF-8"?>
+        <Request>
+            <NewOrder>
+                <OrbitalConnectionUsername>%(user)s</OrbitalConnectionUsername>
+                <OrbitalConnectionPassword>%(password)s</OrbitalConnectionPassword>
+                <IndustryType>%(industry)s</IndustryType>
+                <MessageType>%(message)s</MessageType>
+                <BIN>%(bin)s</BIN>
+                <MerchantID>%(merchant)s</MerchantID>
+                <TerminalID>%(terminal)s</TerminalID>
+                <AccountNum>%(account)s</AccountNum>
+                <Exp>%(exp)s</Exp>
+                <CurrencyCode>%(currency_code)s</CurrencyCode>
+                <CurrencyExponent>%(currency_exponent)s</CurrencyExponent>
+                <CardSecValInd>%(card_sec_val_ind)s</CardSecValInd>
+                <CardSecVal>%(card_sec_val)s</CardSecVal>
+                <AVSzip>%(avs_zip)s</AVSzip>
+                <AVSaddress1>%(avs_address_1)s</AVSaddress1>
+                <AVSaddress2>%(avs_address_2)s</AVSaddress2>
+                <AVScity>%(avs_city)s</AVScity>
+                <AVSstate>%(avs_state)s</AVSstate>
+                <AVSphoneNum>%(avs_phone)s</AVSphoneNum>
+                <AVScountryCode>%(avs_country)s</AVScountryCode>
+                <CustomerProfileFromOrderInd>%(profile_from_order_ind)s</CustomerProfileFromOrderInd>
+                <CustomerProfileOrderOverrideInd>%(profile_order_override_ind)s</CustomerProfileOrderOverrideInd>
+                <OrderID>%(order_id)s</OrderID>
+                <Amount>%(amount)s</Amount>
+            </NewOrder>
+        </Request>
+    """
+
+    def __init__(self, development=False, user=None, password=None,
+               industry=None, message=None, api_url=None,
+               bin_code=None, merchant=None, host=None,
+               terminal=None, target=None):
+
+        # PaymenTech point of sales data
+        self.user = user
+        self.password = password
+        self.industry = industry
+        self.message = message
+        self.bin_code = bin_code
+        self.merchant = merchant
+        self.terminal = terminal
+
+        # Service options
+        self.development = development
+        self.target = target
+        self.host = host
+        self.api_url = api_url
+
+        # dev: https://orbitalvar1.paymentech.net/authorize:443
+        # prod: https://orbital1.paymentech.net/authorize
+
+        if self.development is False:
+            if not self.target:
+                # production
+                self.target = "https://orbital1.paymentech.net/authorize"
+
+            self.host, self.api_url = \
+                urllib2.splithost(urllib2.splittype(self.target)[1])
+
+        else:
+            if not self.target:
+                # development
+                self.target = "https://orbitalvar1.paymentech.net/authorize"
+            if not self.host:
+                self.host = "orbitalvar1.paymentech.net/authorize:443"
+            if not self.api_url:
+                self.api_url = "/"
+
+    def charge(self, raw=None, **kwargs):
+        """
+        Post an XML request to Paymentech
+        This is an example of a call with raw xml data:
+
+             from paymentech import PaymenTech
+
+             # Note: user/password/etc data is not mandatory as it
+             # is retrieved from instance attributes (set on init)
+
+             pt = PaymenTech(user="<myuser>",
+                             password="<mypassword>",
+                             ...) # see basic user in the class help
+             result = pt.charge(raw=xml_string)
+
+        A better way to make a charge request is to unpack a dict object
+        with the operation data:
+
+             ...
+             # The complete input values are listed below in
+             # "Transacion data..."
+
+             charge_data = dict(account=<str>, exp=<str mmyyyy>, ...)
+             result = pt.charge(**charge_data)
+
+
+        Variable xml_string contains all details about the order,
+        plus we are sending username/password in there too...
+
+
+        Transaction data (to be passed to the charge() method)
+        ======================================================
+
+        (Note that it is possible to override the class user,
+        pass, etc. passing those arguments to the .charge() method,
+        which are documented in the class help)
+
+        account
+        exp <str mmyyyy>
+        currency_code
+        currency_exponent
+        card_sec_val_ind
+        card_sec_val
+        avs_zip
+        avs_address_1
+        avs_address_2
+        avs_city
+        avs_state
+        avs_phone
+        avs_country
+        profile_from_order_ind
+        profile_order_override_ind
+        order_id
+        amount <str> (all integers with two decimal digits, without dot
+                     separation)
+
+        Request header example
+        ======================
+
+        Request: sent as POST to https://orbitalvar1.paymentech.net/authorize:443
+        from 127.0.0.1
+        request headers:
+        Content-Type: application/PTI45
+        Content-Type: application/PTI46
+        Content-transfer-encoding: text
+        Request-number: 1
+        Document-type: Request
+        Trace-number: 1234556446
+        <?xml version="1.0" encoding="UTF-8"?>
+        """
+
+        # default charge data
+        data = dict(user=self.user, password=self.password,
+                    industry=self.industry, message=self.message,
+                    bin_code=self.bin_code, merchant=self.merchant,
+                    terminal=self.terminal, account="", exp="",
+                    currency_code="", currency_exponent="",
+                    card_sec_val_ind="", card_sec_val="", avs_zip="",
+                    avs_address_1="", avs_address_2="", avs_city="",
+                    avs_state="", avs_phone="", avs_country="",
+                    profile_from_order_ind="",
+                    profile_order_override_ind="", order_id="",
+                    amount="")
+
+        result = dict()
+
+        # Complete the charge request with the method kwargs
+        for k, v in kwargs.iteritems():
+            data[k] = v
+
+        status_code = status_message = header = resp_code = \
+        tx_ref_num = order_id = None
+        conn = httplib.HTTPS(self.host)
+        conn.putrequest('POST', self.api_url)
+       
+        if self.development:
+            content_type = "PTI56"
+        else:
+            content_type = "PTI46"
+
+        if raw is None:
+            xml_string = self.charge_xml % data
+        else:
+            xml_string = raw
+
+        conn.putheader("Content-Type",
+                       "application/%s") % content_type
+        conn.putheader("Content-transfer-encoding", "text")
+        conn.putheader("Request-number", "1")
+        conn.putheader("Content-length", str(len(xml_string)))
+        conn.putheader("Document-type", "Request")
+        conn.putheader("Trace-number", str(data["order_id"]))
+        conn.putheader("MIME-Version", "1.0")
+        conn.endheaders()
+        conn.send(xml_string)
+
+        result["status_code"], result["status_message"], \
+        result["header"] = conn.getreply()
+
+        fp = conn.getfile()
+        output = fp.read()
+        fp.close()
+
+        dom = parseString(output)
+        result["resp_code"] = \
+            dom.getElementsByTagName('RespCode')[0].firstChild.data
+        result["tx_ref_num"] = \
+            dom.getElementsByTagName('TxRefNum')[0].firstChild.data
+        result["order_id"] = \
+            dom.getElementsByTagName('CustomerRefNum')[0].firstChild.data
+
+        return result
+
+

gluon/custom_import.py

@@ -16,7 +16,7 @@ INVALID_MODULES = set(('', 'gluon', 'applications', 'custom_import'))
 
 
 def custom_import_install():
-    if __builtin__.__import__ != custom_importer:
+    if __builtin__.__import__ == NATIVE_IMPORTER:
         INVALID_MODULES.update(sys.modules.keys())
         __builtin__.__import__ = custom_importer
 

gluon/fileutils.py

@@ -7,6 +7,7 @@ Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
 """
 
+import sys
 import storage
 import os
 import re
@@ -241,6 +242,17 @@ def w2p_pack(filename, path, compiled=False):
 
 
 def w2p_unpack(filename, path, delete_tar=True):
+
+    if filename=='welcome.w2p' and (
+        not os.path.exists('welcome.w2p') or \
+            os.path.exists('NEWINSTALL')):
+        try:
+            w2p_pack('welcome.w2p', 'applications/welcome')
+            os.unlink('NEWINSTALL')
+        except:
+            msg = "New installation: unable to create welcome.w2p file"
+            sys.stderr.write(msg)
+
     filename = abspath(filename)
     path = abspath(path)
     if filename[-4:] == '.w2p' or filename[-3:] == '.gz':

gluon/globals.py

@@ -48,7 +48,6 @@ except ImportError:
     have_minify = False
 
 regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$')
-regex_nopasswd = re.compile('(?<=\:)([^:@/]+)(?=@.+)')
 
 __all__ = ['Request', 'Response', 'Session']
 
@@ -127,11 +126,17 @@ class Request(Storage):
         If request comes in over HTTP, redirect it to HTTPS
         and secure the session.
         """
-        if not global_settings.cronjob and not self.is_https:
+        cmd_opts = global_settings.cmd_options
+        #checking if this is called within the scheduler or within the shell
+        #in addition to checking if it's not a cronjob
+        if ((cmd_opts and (cmd_opts.shell or cmd_opts.scheduler))
+            or global_settings.cronjob or self.is_https):
+            current.session.secure()
+        else:
             current.session.forget()
             redirect(URL(scheme='https', args=self.args, vars=self.vars))
 
-        current.session.secure()
+
 
     def restful(self):
         def wrapper(action, self=self):
@@ -388,7 +393,10 @@ class Response(Storage):
         if not items:
             raise HTTP(404)
         (t, f) = (items.group('table'), items.group('field'))
-        field = db[t][f]
+        try:
+            field = db[t][f]
+        except AttributeError:
+            raise HTTP(404)
         try:
             (filename, stream) = field.retrieve(name)
         except IOError:
@@ -397,7 +405,7 @@ class Response(Storage):
         headers['Content-Type'] = contenttype(name)
         if attachment:
             headers['Content-Disposition'] = \
-                'attachment; filename=%s' % filename
+                'attachment; filename="%s"' % filename.replace('"','\"')
         return self.stream(stream, chunk_size=chunk_size, request=request)
 
     def json(self, data, default=None):
@@ -431,22 +439,16 @@ class Response(Storage):
         BUTTON = TAG.button
         admin = URL("admin", "default", "design",
                     args=current.request.application)
-        from gluon.dal import THREAD_LOCAL
-        if hasattr(THREAD_LOCAL, 'instances'):
-            dbstats = [TABLE(*[TR(PRE(row[0]), '%.2fms' % (row[1] * 1000))
-                               for row in i.db._timings])
-                       for i in THREAD_LOCAL.instances]
-            dbtables = dict([(regex_nopasswd.sub('******', i.uri),
-                              {'defined':
-                               sorted(list(set(i.db.tables) -
-                                           set(i.db._LAZY_TABLES.keys()))) or
-                               '[no defined tables]',
-                               'lazy': sorted(i.db._LAZY_TABLES.keys()) or
-                               '[no lazy tables]'})
-                             for i in THREAD_LOCAL.instances])
-        else:
-            dbstats = []  # if no db or on GAE
-            dbtables = {}
+        from gluon.dal import DAL
+        dbstats = []
+        dbtables = {}
+        infos = DAL.get_instances()
+        for k,v in infos.iteritems():
+            dbstats.append(TABLE(*[TR(PRE(row[0]),'%.2fms' %
+                                          (row[1]*1000))
+                                           for row in v['dbstats']]))
+            dbtables[k] = dict(defined=v['dbtables']['defined'] or '[no defined tables]',
+                               lazy=v['dbtables']['lazy'] or '[no lazy tables]')
         u = web2py_uuid()
         backtotop = A('Back to top', _href="#totop-%s" % u)
         return DIV(
@@ -504,7 +506,7 @@ class Session(Storage):
             request = current.request
         if response is None:
             response = current.response
-        if separate == True:
+        if separate is True:
             separate = lambda session_name: session_name[-2:]
         self._unlock(response)
         if not masterapp:
@@ -556,6 +558,7 @@ class Session(Storage):
                     response.session_id = None
             # do not try load the data from file is these was data in cookie
             if response.session_id and not session_cookie_data:
+                # os.path.exists(response.session_filename):
                 try:
                     response.session_file = \
                         open(response.session_filename, 'rb+')
@@ -569,12 +572,14 @@ class Session(Storage):
                             .split('-')[0]
                         if check_client and client != oc:
                             raise Exception("cookie attack")
+                    except:
+                        response.session_id = None
                     finally:
                         pass
                         #This causes admin login to break. Must find out why.
                         #self._close(response)
                 except:
-                    response.session_id = None
+                    response.session_file = None
             if not response.session_id:
                 uuid = web2py_uuid()
                 response.session_id = '%s-%s' % (client, uuid)
@@ -655,6 +660,12 @@ class Session(Storage):
         if self.flash:
             (response.flash, self.flash) = (self.flash, None)
 
+    def clear(self):
+        previous_session_hash = self.pop('_session_hash', None)
+        Storage.clear(self)
+        if previous_session_hash:
+            self._session_hash = previous_session_hash
+
     def is_new(self):
         if self._start_timestamp:
             return False
@@ -740,12 +751,10 @@ class Session(Storage):
     def _try_store_in_file(self, request, response):
         if response.session_storage_type != 'file':
             return False
-
         try:
             if not response.session_id or self._forget or self._unchanged():
                 return False
-
-            if response.session_new:
+            if response.session_new or not response.session_file:
                 # Tests if the session sub-folder exists, if not, create it
                 session_folder = os.path.dirname(response.session_filename)
                 if not os.path.exists(session_folder):

gluon/html.py

@@ -129,6 +129,11 @@ def xmlescape(data, quote=True):
     data = cgi.escape(data, quote).replace("'", "'")
     return data
 
+def call_as_list(f,*a,**b):
+    if not isinstance(f, (list,tuple)):
+        f = [f]
+    for item in f:
+        item(*a,**b)
 
 def truncate_string(text, length, dots='...'):
     text = text.decode('utf-8')
@@ -1247,19 +1252,20 @@ class HTML(DIV):
             lang = 'en'
         self.attributes['_lang'] = lang
         doctype = self['doctype']
-        if doctype:
-            if doctype == 'strict':
-                doctype = self.strict
-            elif doctype == 'transitional':
-                doctype = self.transitional
-            elif doctype == 'frameset':
-                doctype = self.frameset
-            elif doctype == 'html5':
-                doctype = self.html5
-            else:
-                doctype = '%s\n' % doctype
-        else:
+        if doctype is None:
             doctype = self.transitional
+        elif doctype == 'strict':
+            doctype = self.strict
+        elif doctype == 'transitional':
+            doctype = self.transitional
+        elif doctype == 'frameset':
+            doctype = self.frameset
+        elif doctype == 'html5':
+            doctype = self.html5
+        elif doctype == '':
+            doctype = ''
+        else:
+            doctype = '%s\n' % doctype
         (fa, co) = self._xml()
         return '%s<%s%s>%s</%s>' % (doctype, self.tag, fa, co, self.tag)
 
@@ -1482,8 +1488,9 @@ class A(DIV):
                     (self['callback'], self['target'] or '', d)
             self['_href'] = self['_href'] or '#null'
         elif self['cid']:
-            self['_onclick'] = 'web2py_component("%s","%s");%sreturn false;' % \
-                (self['_href'], self['cid'], d)
+	    pre = self['pre_call'] + ';' if self['pre_call'] else ''
+            self['_onclick'] = '%sweb2py_component("%s","%s");%sreturn false;' % \
+                (pre,self['_href'], self['cid'], d)
         return DIV.xml(self)
 
 
@@ -1756,7 +1763,7 @@ class INPUT(DIV):
             t = self['_type'] = 'text'
         t = t.lower()
         value = self['value']
-        if self['_value'] is None:
+        if self['_value'] is None or isinstance(self['_value'],cgi.FieldStorage):
             _value = None
         else:
             _value = str(self['_value'])
@@ -1890,7 +1897,8 @@ class SELECT(INPUT):
         if not value is None:
             if not self['_multiple']:
                 for c in options:  # my patch
-                    if value and str(c['_value']) == str(value):
+                    if ((value is not None) and
+                        (str(c['_value']) == str(value))):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -1900,7 +1908,8 @@ class SELECT(INPUT):
                 else:
                     values = [str(value)]
                 for c in options:  # my patch
-                    if value and str(c['_value']) in values:
+                    if ((value is not None) and
+                        (str(c['_value']) in values)):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -1960,7 +1969,7 @@ class FORM(DIV):
         **kwargs
     ):
         """
-        kwargs is not used but allows to specify the same interface for FROM and SQLFORM
+        kwargs is not used but allows to specify the same interface for FORM and SQLFORM
         """
         if request_vars.__class__.__name__ == 'Request':
             request_vars = request_vars.post_vars
@@ -1996,20 +2005,20 @@ class FORM(DIV):
                 onsuccess = onvalidation.get('onsuccess', None)
                 onfailure = onvalidation.get('onfailure', None)
                 onchange = onvalidation.get('onchange', None)
+                if [k for k in onvalidation if not k in (
+                        'onsuccess','onfailure','onchange')]:
+                    raise RuntimeError('Invalid key in onvalidate dict')
                 if onsuccess and status:
-                    onsuccess(self)
+                    call_as_list(onsuccess,self)
                 if onfailure and request_vars and not status:
-                    onfailure(self)
+                    call_as_list(onfailure,self)
                     status = len(self.errors) == 0
                 if changed:
                     if onchange and self.record_changed and \
                             self.detect_record_change:
-                        onchange(self)
+                        call_as_list(onchange,self)
             elif status:
-                if isinstance(onvalidation, (list, tuple)):
-                    [f(self) for f in onvalidation]
-                else:
-                    onvalidation(self)
+                call_as_list(onvalidation, self)
         if self.errors:
             status = False
         if not session is None:
@@ -2285,6 +2294,8 @@ class MENU(DIV):
       _class: defaults to 'web2py-menu web2py-menu-vertical'
       ul_class: defaults to 'web2py-menu-vertical'
       li_class: defaults to 'web2py-menu-expand'
+      li_first: defaults to 'web2py-menu-first'
+      li_last: defaults to 'web2py-menu-last'
 
     Example:
         menu = MENU([['name', False, URL(...), [submenu]], ...])
@@ -2303,6 +2314,10 @@ class MENU(DIV):
             self['ul_class'] = 'web2py-menu-vertical'
         if not 'li_class' in self.attributes:
             self['li_class'] = 'web2py-menu-expand'
+        if not 'li_first' in self.attributes:
+            self['li_first'] = 'web2py-menu-first'
+        if not 'li_last' in self.attributes:
+            self['li_last'] = 'web2py-menu-last'
         if not 'li_active' in self.attributes:
             self['li_active'] = 'web2py-menu-active'
         if not 'mobile' in self.attributes:
@@ -2319,6 +2334,8 @@ class MENU(DIV):
                 li = LI(link)
             elif 'no_link_url' in self.attributes and self['no_link_url'] == link:
                 li = LI(DIV(name))
+            elif isinstance(link,dict):
+                li = LI(A(name, **link))
             elif link:
                 li = LI(A(name, _href=link))
             elif not link and isinstance(name, A):
@@ -2326,6 +2343,10 @@ class MENU(DIV):
             else:
                 li = LI(A(name, _href='#',
                           _onclick='javascript:void(0);return false;'))
+            if level == 0 and item == data[0]:
+                li['_class'] = self['li_first']
+            elif level == 0 and item == data[-1]:
+                li['_class'] = self['li_last']
             if len(item) > 3 and item[3]:
                 li['_class'] = self['li_class']
                 li.append(self.serialize(item[3], level + 1))

gluon/http.py

@@ -140,7 +140,7 @@ class HTTP(BaseException):
         return self.message
 
 
-def redirect(location, how=303, client_side=False):
+def redirect(location='', how=303, client_side=False):
     if location:
         from gluon import current
         loc = location.replace('\r', '%0D').replace('\n', '%0A')
@@ -150,3 +150,8 @@ def redirect(location, how=303, client_side=False):
             raise HTTP(how,
                        'You are being redirected <a href="%s">here</a>' % loc,
                        Location=loc)
+    else:
+        from gluon import current
+        if client_side and current.request.ajax:
+            raise HTTP(200, **{'web2py-component-command': 'window.location.reload(true)'})
+    

gluon/languages.py

@@ -12,17 +12,24 @@ Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine)
 
 import os
 import re
+import sys
 import pkgutil
-from utf8 import Utf8
-from cgi import escape
-import portalocker
 import logging
 import marshal
-import copy_reg
+from cgi import escape
+from threading import RLock
+
+try:
+    import copyreg as copy_reg # python 3
+except ImportError:
+    import copy_reg # python 2
+
+from portalocker import read_locked, LockedFile
+from utf8 import Utf8
+
 from fileutils import listdir
 import settings
 from cfs import getcfs
-from thread import allocate_lock
 from html import XML, xmlescape
 from contrib.markmin.markmin2html import render, markmin_escape
 from string import maketrans
@@ -35,7 +42,7 @@ pjoin = os.path.join
 pexists = os.path.exists
 pdirname = os.path.dirname
 isdir = os.path.isdir
-is_gae = settings.global_settings.web2py_runtime_gae
+is_gae = False # settings.global_settings.web2py_runtime_gae
 
 DEFAULT_LANGUAGE = 'en'
 DEFAULT_LANGUAGE_NAME = 'English'
@@ -137,7 +144,7 @@ def get_from_cache(cache, val, fun):
 
 def clear_cache(filename):
     cache = global_language_cache.setdefault(
-        filename, ({}, allocate_lock()))
+        filename, ({}, RLock()))
     lang_dict, lock = cache
     lock.acquire()
     try:
@@ -147,11 +154,12 @@ def clear_cache(filename):
 
 
 def read_dict_aux(filename):
-    lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
+    lang_text = read_locked(filename).replace('\r\n', '\n')
     clear_cache(filename)
     try:
         return safe_eval(lang_text) or {}
-    except Exception, e:
+    except Exception:
+        e = sys.exc_info()[1]
         status = 'Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
         return {'__corrupted__': status}
@@ -187,7 +195,8 @@ def read_possible_plural_rules():
                     DEFAULT_CONSTRUCT_PLURAL_FORM)
                 plurals[lang] = (lang, nplurals, get_plural_id,
                                  construct_plural_form)
-    except ImportError, e:
+    except ImportError:
+        e = sys.exc_info()[1]
         logging.warn('Unable to import plural rules: %s' % e)
     return plurals
 
@@ -261,17 +270,17 @@ def read_possible_languages_aux(langdir):
     return langs
 
 
-def read_possible_languages(appdir):
-    langdir = pjoin(appdir, 'languages')
-    return getcfs('langs:' + langdir, langdir,
-                  lambda: read_possible_languages_aux(langdir))
+def read_possible_languages(langpath):
+    return getcfs('langs:' + langpath, langpath,
+                  lambda: read_possible_languages_aux(langpath))
 
 
 def read_plural_dict_aux(filename):
-    lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
+    lang_text = read_locked(filename).replace('\r\n', '\n')
     try:
         return eval(lang_text) or {}
-    except Exception, e:
+    except Exception:
+        e = sys.exc_info()[1]
         status = 'Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
         return {'__corrupted__': status}
@@ -286,7 +295,7 @@ def write_plural_dict(filename, contents):
     if '__corrupted__' in contents:
         return
     try:
-        fp = portalocker.LockedFile(filename, 'w')
+        fp = LockedFile(filename, 'w')
         fp.write('#!/usr/bin/env python\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n')
         # coding: utf8\n{\n')
         for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())):
@@ -306,7 +315,7 @@ def write_dict(filename, contents):
     if '__corrupted__' in contents:
         return
     try:
-        fp = portalocker.LockedFile(filename, 'w')
+        fp = LockedFile(filename, 'w')
     except (IOError, OSError):
         if not settings.global_settings.web2py_runtime_gae:
             logging.warning('Unable to write to file %s' % filename)
@@ -434,11 +443,9 @@ class translator(object):
         xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py
     """
 
-    def __init__(self, request):
-        self.request = request
-        self.folder = request.folder
-        self.langpath = pjoin(self.folder, 'languages')
-        self.http_accept_language = request.env.http_accept_language
+    def __init__(self, langpath, http_accept_language):
+        self.langpath = langpath
+        self.http_accept_language = http_accept_language
         self.is_writable = not is_gae
         # filled in self.force():
         #------------------------
@@ -493,7 +500,7 @@ class translator(object):
                   construct_plural_form) # construct_plural_form() for current language
             }
         """
-        info = read_possible_languages(self.folder)
+        info = read_possible_languages(self.langpath)
         if lang:
             info = info.get(lang)
         return info
@@ -501,7 +508,7 @@ class translator(object):
     def get_possible_languages(self):
         """ get list of all possible languages for current applications """
         return list(set(self.current_languages +
-                        [lang for lang in read_possible_languages(self.folder).iterkeys()
+                        [lang for lang in read_possible_languages(self.langpath).iterkeys()
                          if lang != 'default']))
 
     def set_current_languages(self, *languages):
@@ -581,7 +588,7 @@ class translator(object):
         default language will be selected if none
         of them matches possible_languages.
         """
-        pl_info = read_possible_languages(self.folder)
+        pl_info = read_possible_languages(self.langpath)
 
         def set_plural(language):
             """
@@ -642,14 +649,14 @@ class translator(object):
                     self.t = read_dict(self.language_file)
                     self.cache = global_language_cache.setdefault(
                         self.language_file,
-                        ({}, allocate_lock()))
+                        ({}, RLock()))
                     set_plural(language)
                     self.accepted_language = language
                     return languages
         self.accepted_language = language or self.current_languages[0]
         self.language_file = self.default_language_file
         self.cache = global_language_cache.setdefault(self.language_file,
-                                                      ({}, allocate_lock()))
+                                                      ({}, RLock()))
         self.t = self.default_t
         set_plural(self.accepted_language)
         return languages
@@ -670,7 +677,8 @@ class translator(object):
             try:
                 otherT = self.otherTs[language]
             except KeyError:
-                otherT = self.otherTs[language] = translator(self.request)
+                otherT = self.otherTs[language] = translator(
+                    self.langpath, self.http_accept_language)
                 otherT.force(language)
             return otherT(message, symbols, lazy=lazy)
 
@@ -892,7 +900,7 @@ def findT(path, language=DEFAULT_LANGUAGE):
     for filename in \
             listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\
             + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0):
-        data = portalocker.read_locked(filename)
+        data = read_locked(filename)
         items = regex_translate.findall(data)
         for item in items:
             try:

gluon/main.py

@@ -127,7 +127,7 @@ def get_client(env):
     guess the client address from the environment variables
 
     first tries 'http_x_forwarded_for', secondly 'remote_addr'
-    if all fails assume '127.0.0.1' (running locally)
+    if all fails, assume '127.0.0.1' or '::1' (running locally)
     """
     g = regex_client.search(env.get('http_x_forwarded_for', ''))
     if g:
@@ -136,8 +136,10 @@ def get_client(env):
         g = regex_client.search(env.get('remote_addr', ''))
         if g:
             client = g.group()
+        elif env.http_host.startswith('['):  # IPv6
+            client = '::1'
         else:
-            client = '127.0.0.1'
+            client = '127.0.0.1'  # IPv4
     if not is_valid_ip_address(client):
         raise HTTP(400, "Bad Request (request.client=%s)" % client)
     return client
@@ -432,35 +434,38 @@ def wsgibase(environ, responder):
                 app = request.application  # must go after url_in!
 
                 if not global_settings.local_hosts:
-                    local_hosts = ['127.0.0.1', '::ffff:127.0.0.1']
+                    local_hosts = set(['127.0.0.1', '::ffff:127.0.0.1', '::1'])
                     if not global_settings.web2py_runtime_gae:
                         try:
-                            local_hosts.append(socket.gethostname())
-                        except TypeError:
-                            pass
-                        try:
+                            fqdn = socket.getfqdn()
+                            local_hosts.add(socket.gethostname())
+                            local_hosts.add(fqdn)
+                            local_hosts.update([
+                                ip[4][0] for ip in socket.getaddrinfo(
+                                    fqdn, 0)])
                             if env.server_name:
-                                local_hosts += [
-                                    env.server_name,
-                                    socket.gethostbyname(env.server_name)]
+                                local_hosts.add(env.server_name)
+                                local_hosts.update([
+                                    ip[4][0] for ip in socket.getaddrinfo(
+                                        env.server_name, 0)])
                         except (socket.gaierror, TypeError):
                             pass
-                    global_settings.local_hosts = local_hosts
+                    global_settings.local_hosts = list(local_hosts)
                 else:
                     local_hosts = global_settings.local_hosts
                 client = get_client(env)
                 x_req_with = str(env.http_x_requested_with).lower()
 
                 request.update(
-                    client=client,
-                    folder=abspath('applications', app) + os.sep,
-                    ajax=x_req_with == 'xmlhttprequest',
-                    cid=env.http_web2py_component_element,
-                    is_local=env.remote_addr in local_hosts,
-                    is_https=env.wsgi_url_scheme in HTTPS_SCHEMES
-                    or request.env.http_x_forwarded_proto in HTTPS_SCHEMES
-                    or env.https == 'on')
-                request.uuid = request.compute_uuid()  # requires client
+                    client = client,
+                    folder = abspath('applications', app) + os.sep,
+                    ajax = x_req_with == 'xmlhttprequest',
+                    cid = env.http_web2py_component_element,
+                    is_local = env.remote_addr in local_hosts,
+                    is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
+                        request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
+                        or env.https == 'on')
+                request.compute_uuid()  # requires client
                 request.url = environ['PATH_INFO']
 
                 # ##################################################
@@ -572,9 +577,12 @@ def wsgibase(environ, responder):
 
                 if request.cid:
                     if response.flash:
-                        http_response.headers['web2py-component-flash'] = urllib2.quote(xmlescape(response.flash).replace('\n', ''))
+                        http_response.headers['web2py-component-flash'] = \
+                            urllib2.quote(xmlescape(response.flash)\
+                                              .replace('\n',''))
                     if response.js:
-                        http_response.headers['web2py-component-command'] = response.js.replace('\n', '')
+                        http_response.headers['web2py-component-command'] = \
+                            urllib2.quote(response.js.replace('\n',''))
 
                 # ##################################################
                 # store cookies in headers

gluon/myregex.py

@@ -18,7 +18,7 @@ regex_tables = re.compile(
 # pattern to find exposed functions in controller
 
 regex_expose = re.compile(
-    '^def\s+(?P<name>(?:[a-zA-Z0-9]\w*)|(?:_[a-zA-Z0-9]\w*))\(\)\s*:',
+    '^def\s+(?P<name>_?[a-zA-Z0-9]\w*)\( *\)\s*:',
     flags=re.M)
 
 regex_include = re.compile(

gluon/portalocker.py

@@ -167,5 +167,5 @@ if __name__ == '__main__':
     f.write('test ok')
     f.close()
     f = LockedFile('test.txt', mode='rb')
-    print f.read()
+    sys.stdout.write(f.read()+'\n')
     f.close()

gluon/rewrite.py

@@ -55,7 +55,8 @@ regex_space = re.compile('(\+|\s|%20)+')
 #   file and args may also contain '-', '=', '.' and '/'
 #   apps in routes_apps_raw must parse raw_args into args
 
-regex_url = re.compile('^/((?P<a>\w+)(/(?P<c>\w+)(/(?P<z>(?P<f>\w+)(\.(?P<e>[\w.]+))?(?P<s>[/\w@=-]*(\.[/\w@=-]+)*)))?)?)?$')
+regex_url = re.compile('^/((?P<a>\w+)(/(?P<c>\w+)(/(?P<z>(?P<f>\w+)(\.(?P<e>[\w.]+))?(?P<s>.*)))?)?)?$')
+regex_args = re.compile('^[/\w@=-]*(\.[/\w@=-]+)*$')
 
 
 def _router_default():
@@ -75,8 +76,13 @@ def _router_default():
         exclusive_domain=False,
         map_hyphen=False,
         acfe_match=r'\w+$',                   # legal app/ctlr/fcn/ext
-        file_match=r'([-+=@$%\w]+[./]?)+$',   # legal static subpath
-        args_match=r'([\w@ -]+[=.]?)*$',      # legal arg in args
+        #
+        #  Implementation note:
+        #  The file_match & args_match patterns use look-behind to avoid
+        #  pathological backtracking from nested patterns.
+        #
+        file_match = r'([-+=@$%\w]|(?<=[-+=@$%\w])[./])*$', # legal static subpath
+        args_match=r'([\w@ -]|(?<=[\w@ -])[.=])*$',         # legal arg in args
     )
     return router
 
@@ -538,7 +544,7 @@ def load_routers(all_apps):
 def regex_uri(e, regexes, tag, default=None):
     "filter incoming URI against a list of regexes"
     path = e['PATH_INFO']
-    host = e.get('http_host', e.get('SERVER_NAME', 'localhost')).lower()
+    host = e.get('HTTP_HOST', e.get('SERVER_NAME', 'localhost')).lower()
     i = host.find(':')
     if i > 0:
         host = host[:i]
@@ -600,6 +606,10 @@ def regex_filter_in(e):
 def sluggify(key):
     return key.lower().replace('.', '_')
 
+def invalid_url(routes):
+    raise HTTP(400,
+               routes.error_message % 'invalid request',
+               web2py_error='invalid path')
 
 def regex_url_in(request, environ):
     "rewrite and parse incoming URL"
@@ -627,18 +637,21 @@ def regex_url_in(request, environ):
         path = path[:-1]
     match = regex_url.match(path)
     if not match:
-        raise HTTP(400,
-                   routes.error_message % 'invalid request',
-                   web2py_error='invalid path')
-    elif match.group('c') == 'static':
+        invalid_url(routes)
+    request.raw_args = (match.group('s') or '')
+    if request.raw_args.startswith('/'):
+        request.raw_args = request.raw_args[1:]
+    if match.group('c') == 'static':
         application = match.group('a')
         version, filename = None, match.group('z')
         items = filename.split('/', 1)
         if regex_version.match(items[0]):
             version, filename = items
-        static_file = pjoin(request.env.applications_parent,
-                            'applications', application,
-                            'static', filename)
+        static_folder = pjoin(request.env.applications_parent,
+                              'applications', application,'static')
+        static_file = os.path.abspath(pjoin(static_folder,filename))
+        if not static_file.startswith(static_folder):
+            invalid_url(routes)
         return (static_file, version, environ)
     else:
         # ##################################################
@@ -649,12 +662,13 @@ def regex_url_in(request, environ):
         request.function = match.group('f') or routes.default_function
         request.raw_extension = match.group('e')
         request.extension = request.raw_extension or 'html'
-        request.raw_args = match.group('s')
         if request.application in routes.routes_apps_raw:
             # application is responsible for parsing args
             request.args = None
+        elif not regex_args.match(request.raw_args):
+            invalid_url(routes)
         elif request.raw_args:
-            request.args = List(request.raw_args.split('/')[1:])
+            request.args = List(request.raw_args.split('/'))
         else:
             request.args = List([])
     return (None, None, environ)

gluon/rocket.py

@@ -10,10 +10,9 @@ import errno
 import socket
 import logging
 import platform
-import traceback
 
 # Define Constants
-VERSION = '1.2.5'
+VERSION = '1.2.6'
 SERVER_NAME = socket.gethostname()
 SERVER_SOFTWARE = 'Rocket %s' % VERSION
 HTTP_SERVER_SOFTWARE = '%s Python/%s' % (
@@ -79,8 +78,8 @@ __all__ = ['VERSION', 'SERVER_SOFTWARE', 'HTTP_SERVER_SOFTWARE', 'BUF_SIZE',
            'IS_JYTHON', 'IGNORE_ERRORS_ON_CLOSE', 'DEFAULTS', 'PY3K', 'b', 'u',
            'Rocket', 'CherryPyWSGIServer', 'SERVER_NAME', 'NullHandler']
 
-# Monolithic build...end of module: rocket\__init__.py
-# Monolithic build...start of module: rocket\connection.py
+# Monolithic build...end of module: rocket/__init__.py
+# Monolithic build...start of module: rocket/connection.py
 
 # Import System Modules
 import sys
@@ -118,7 +117,7 @@ class Connection(object):
     ]
 
     def __init__(self, sock_tuple, port, secure=False):
-        self.client_addr, self.client_port = sock_tuple[1]
+        self.client_addr, self.client_port = sock_tuple[1][:2]
         self.server_port = port
         self.socket = sock_tuple[0]
         self.start_time = time.time()
@@ -133,7 +132,6 @@ class Connection(object):
 
         self.socket.settimeout(SOCKET_TIMEOUT)
 
-        self.sendall = self.socket.sendall
         self.shutdown = self.socket.shutdown
         self.fileno = self.socket.fileno
         self.setblocking = self.socket.setblocking
@@ -141,6 +139,26 @@ class Connection(object):
         self.send = self.socket.send
         self.makefile = self.socket.makefile
 
+        if sys.platform == 'darwin':
+            self.sendall = self._sendall_darwin
+        else:
+            self.sendall = self.socket.sendall
+
+    def _sendall_darwin(self, buf):
+        pending = len(buf)
+        offset = 0
+        while pending:
+            try:
+                sent = self.socket.send(buf[offset:])
+                pending -= sent
+                offset += sent
+            except socket.error:
+                import errno
+                info = sys.exc_info()
+                if info[1].args[0] != errno.EAGAIN:
+                    raise
+        return offset
+
 # FIXME - this is not ready for prime-time yet.
 #    def makefile(self, buf_size=BUF_SIZE):
 #        return FileLikeSocket(self, buf_size)
@@ -157,9 +175,8 @@ class Connection(object):
                     pass
         self.socket.close()
 
-
-# Monolithic build...end of module: rocket\connection.py
-# Monolithic build...start of module: rocket\filelike.py
+# Monolithic build...end of module: rocket/connection.py
+# Monolithic build...start of module: rocket/filelike.py
 
 # Import System Modules
 import socket
@@ -282,8 +299,8 @@ class FileLikeSocket(object):
         self.conn = None
         self.content_length = None
 
-# Monolithic build...end of module: rocket\filelike.py
-# Monolithic build...start of module: rocket\futures.py
+# Monolithic build...end of module: rocket/filelike.py
+# Monolithic build...start of module: rocket/futures.py
 
 # Import System Modules
 import time
@@ -396,8 +413,8 @@ class FuturesMiddleware(object):
         environ["wsgiorg.futures"] = self.executor.futures
         return self.app(environ, start_response)
 
-# Monolithic build...end of module: rocket\futures.py
-# Monolithic build...start of module: rocket\listener.py
+# Monolithic build...end of module: rocket/futures.py
+# Monolithic build...start of module: rocket/listener.py
 
 # Import System Modules
 import os
@@ -442,7 +459,10 @@ class Listener(Thread):
         self.err_log.addHandler(NullHandler())
 
         # Build the socket
-        listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if ':' in self.addr:
+            listener = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        else:
+            listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 
         if not listener:
             self.err_log.error("Failed to get socket.")
@@ -520,12 +540,10 @@ class Listener(Thread):
                                        certfile=self.interface[3],
                                        server_side=True,
                                        ssl_version=ssl.PROTOCOL_SSLv23)
-
         except SSLError:
             # Generally this happens when an HTTP request is received on a
             # secure socket. We don't do anything because it will be detected
             # by Worker and dealt with appropriately.
-            # self.err_log.error('SSL Error: %s' % traceback.format_exc())
             pass
 
         return sock
@@ -576,8 +594,9 @@ class Listener(Thread):
                                        self.secure))
 
             except socket.timeout:
-                # socket.timeout will be raised every THREAD_STOP_CHECK_INTERVAL
-                # seconds.  When that happens, we check if it's time to die.
+                # socket.timeout will be raised every
+                # THREAD_STOP_CHECK_INTERVAL seconds.  When that happens,
+                # we check if it's time to die.
 
                 if not self.ready:
                     if __debug__:
@@ -588,8 +607,8 @@ class Listener(Thread):
             except:
                 self.err_log.error(traceback.format_exc())
 
-# Monolithic build...end of module: rocket\listener.py
-# Monolithic build...start of module: rocket\main.py
+# Monolithic build...end of module: rocket/listener.py
+# Monolithic build...start of module: rocket/main.py
 
 # Import System Modules
 import sys
@@ -606,7 +625,6 @@ except ImportError:
 # Import Package Modules
 # package imports removed in monolithic build
 
-
 # Setup Logging
 log = logging.getLogger('Rocket')
 log.addHandler(NullHandler())
@@ -618,13 +636,13 @@ class Rocket(object):
 
     def __init__(self,
                  interfaces=('127.0.0.1', 8000),
-                 method = 'wsgi',
-                 app_info = None,
-                 min_threads = None,
-                 max_threads = None,
-                 queue_size = None,
-                 timeout = 600,
-                 handle_signals = True):
+                 method='wsgi',
+                 app_info=None,
+                 min_threads=None,
+                 max_threads=None,
+                 queue_size=None,
+                 timeout=600,
+                 handle_signals=True):
 
         self.handle_signals = handle_signals
         self.startstop_lock = Lock()
@@ -801,8 +819,8 @@ def CherryPyWSGIServer(bind_addr,
                   queue_size=request_queue_size,
                   timeout=timeout)
 
-# Monolithic build...end of module: rocket\main.py
-# Monolithic build...start of module: rocket\monitor.py
+# Monolithic build...end of module: rocket/main.py
+# Monolithic build...start of module: rocket/monitor.py
 
 # Import System Modules
 import time
@@ -983,8 +1001,8 @@ class Monitor(Thread):
         # Place a None sentry value to cause the monitor to die.
         self.monitor_queue.put(None)
 
-# Monolithic build...end of module: rocket\monitor.py
-# Monolithic build...start of module: rocket\threadpool.py
+# Monolithic build...end of module: rocket/monitor.py
+# Monolithic build...start of module: rocket/threadpool.py
 
 # Import System Modules
 import logging
@@ -1146,8 +1164,8 @@ class ThreadPool:
 
                 self.grow(queueSize)
 
-# Monolithic build...end of module: rocket\threadpool.py
-# Monolithic build...start of module: rocket\worker.py
+# Monolithic build...end of module: rocket/threadpool.py
+# Monolithic build...start of module: rocket/worker.py
 
 # Import System Modules
 import re
@@ -1192,14 +1210,14 @@ re_REQUEST_LINE = re.compile(r"""^
     (?P<host>[^/]+)                                          # Host
 )? #
 (?P<path>(\*|/[^ \?]*))                                      # Path
-(\? (?P<query_string>[^ ]+))?                                # Query String
+(\? (?P<query_string>[^ ]*))?                                # Query String
 \                                                            # (single space)
 (?P<protocol>HTTPS?/1\.[01])                                 # Protocol
 $
 """, re.X)
 LOG_LINE = '%(client_ip)s - "%(request_line)s" - %(status)s %(size)s'
 RESPONSE = '''\
-HTTP/1.1 %s
+%s %s
 Content-Length: %i
 Content-Type: %s
 
@@ -1207,7 +1225,7 @@ Content-Type: %s
 '''
 if IS_JYTHON:
     HTTP_METHODS = set(['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
-                       'DELETE', 'TRACE', 'CONNECT'])
+                        'DELETE', 'TRACE', 'CONNECT'])
 
 
 class Worker(Thread):
@@ -1232,6 +1250,7 @@ class Worker(Thread):
         self.status = "200 OK"
         self.closeConnection = True
         self.request_line = ""
+        self.protocol = 'HTTP/1.1'
 
         # Request Log
         self.req_log = logging.getLogger('Rocket.Requests')
@@ -1316,26 +1335,19 @@ class Worker(Thread):
                     self.err_log.debug('Serving a request')
                 try:
                     self.run_app(conn)
-                    log_info = dict(client_ip=conn.client_addr,
-                                    time=datetime.now().strftime('%c'),
-                                    status=self.status.split(' ')[0],
-                                    size=self.size,
-                                    request_line=self.request_line)
-                    self.req_log.info(LOG_LINE % log_info)
                 except:
                     exc = sys.exc_info()
                     handled = self._handleError(*exc)
                     if handled:
                         break
-                    else:
-                        if self.request_line:
-                            log_info = dict(client_ip=conn.client_addr,
-                                            time=datetime.now(
-                                            ).strftime('%c'),
-                                            status=self.status.split(' ')[0],
-                                            size=self.size,
-                                            request_line=self.request_line + ' - not stopping')
-                            self.req_log.info(LOG_LINE % log_info)
+                finally:
+                    if self.request_line:
+                        log_info = dict(client_ip=conn.client_addr,
+                                        time=datetime.now().strftime('%c'),
+                                        status=self.status.split(' ')[0],
+                                        size=self.size,
+                                        request_line=self.request_line)
+                        self.req_log.info(LOG_LINE % log_info)
 
                 if self.closeConnection:
                     try:
@@ -1353,7 +1365,8 @@ class Worker(Thread):
 
     def send_response(self, status):
         stat_msg = status.split(' ', 1)[1]
-        msg = RESPONSE % (status,
+        msg = RESPONSE % (self.protocol,
+                          status,
                           len(stat_msg),
                           'text/plain',
                           stat_msg)
@@ -1361,23 +1374,12 @@ class Worker(Thread):
             self.conn.sendall(b(msg))
         except socket.timeout:
             self.closeConnection = True
-            self.err_log.error(
-                'Tried to send "%s" to client but received timeout error'
-                % status)
+            msg = 'Tried to send "%s" to client but received timeout error'
+            self.err_log.error(msg % status)
         except socket.error:
             self.closeConnection = True
-            self.err_log.error(
-                'Tried to send "%s" to client but received socket error'
-                % status)
-
-    #def kill(self):
-    #    if self.isAlive() and hasattr(self, 'conn'):
-    #        try:
-    #            self.conn.shutdown(socket.SHUT_RDWR)
-    #        except socket.error:
-    #            info = sys.exc_info()
-    #            if info[1].args[0] != socket.EBADF:
-    #                self.err_log.debug('Error on shutdown: '+str(info))
+            msg = 'Tried to send "%s" to client but received socket error'
+            self.err_log.error(msg % status)
 
     def read_request_line(self, sock_file):
         self.request_line = ''
@@ -1396,10 +1398,11 @@ class Worker(Thread):
                 if PY3K:
                     d = d.decode('ISO-8859-1')
         except socket.timeout:
-            raise SocketTimeout("Socket timed out before request.")
+            raise SocketTimeout('Socket timed out before request.')
         except TypeError:
             raise SocketClosed(
-                "ssl bug caused closer of socket, upgrade to python 2.7")
+                'SSL bug caused closure of socket.  See '
+                '"https://groups.google.com/d/topic/web2py/P_Gw0JxWzCs".')
 
         d = d.strip()
 
@@ -1433,6 +1436,7 @@ class Worker(Thread):
                 req['path'] = r'%2F'.join(
                     [unquote(x) for x in re_SLASH.split(v)])
 
+        self.protocol = req['protocol']
         return req
 
     def _read_request_line_jython(self, d):
@@ -1440,7 +1444,7 @@ class Worker(Thread):
         try:
             method, uri, proto = d.split(' ')
             if not proto.startswith('HTTP') or \
-                proto[-3:] not in ('1.0', '1.1') or \
+                    proto[-3:] not in ('1.0', '1.1') or \
                     method not in HTTP_METHODS:
                 self.send_response('400 Bad Request')
                 raise BadRequest
@@ -1473,36 +1477,42 @@ class Worker(Thread):
                    host=host)
         return req
 
-    def read_headers(self, sock_file, environ):
+    def read_headers(self, sock_file):
         try:
+            headers = dict()
             lname = None
+            lval = None
             while True:
                 l = sock_file.readline()
+
                 if PY3K:
                     try:
                         l = str(l, 'ISO-8859-1')
                     except UnicodeDecodeError:
                         self.err_log.warning(
-                            'Invalid request header: ' + repr(l))
+                            'Client sent invalid header: ' + repr(l))
 
                 if l.strip().replace('\0', '') == '':
                     break
-                elif l[0] in ' \t' and lname:
+
+                if l[0] in ' \t' and lname:
                     # Some headers take more than one line
-                    environ[lname] += ' ' + l.strip()
+                    lval += ' ' + l.strip()
                 else:
                     # HTTP header values are latin-1 encoded
                     l = l.split(':', 1)
                     # HTTP header names are us-ascii encoded
 
-                    lname = str(
-                        'HTTP_' + l[0].strip().upper().replace('-', '_'))
-                    lval = str(l[-1].strip())
-                    environ[lname] = lval
+                    lname = l[0].strip().upper().replace('-', '_')
+                    lval = l[-1].strip()
+
+                headers[str(lname)] = str(lval)
 
         except socket.timeout:
             raise SocketTimeout("Socket timed out before request.")
 
+        return headers
+
 
 class SocketTimeout(Exception):
     "Exception for when a socket times out between requests."
@@ -1520,6 +1530,7 @@ class SocketClosed(Exception):
 
 
 class ChunkedReader(object):
+
     def __init__(self, sock_file):
         self.stream = sock_file
         self.chunk_size = 0
@@ -1571,8 +1582,11 @@ def get_method(method):
     methods = dict(wsgi=WSGIWorker)
     return methods[method.lower()]
 
-# Monolithic build...end of module: rocket\worker.py
-# Monolithic build...start of module: rocket\methods\wsgi.py
+# Monolithic build...end of module: rocket/worker.py
+# Monolithic build...start of module: rocket/methods/__init__.py
+
+# Monolithic build...end of module: rocket/methods/__init__.py
+# Monolithic build...start of module: rocket/methods/wsgi.py
 
 # Import System Modules
 import sys
@@ -1583,7 +1597,6 @@ from wsgiref.util import FileWrapper
 # Import Package Modules
 # package imports removed in monolithic build
 
-
 if PY3K:
     from email.utils import formatdate
 else:
@@ -1640,15 +1653,16 @@ class WSGIWorker(Worker):
         environ = self.base_environ.copy()
 
         # Grab the headers
-        self.read_headers(sock_file, environ)
+        for k, v in self.read_headers(sock_file).iteritems():
+            environ[str('HTTP_' + k)] = v
 
         # Add CGI Variables
-        environ['SERVER_PORT'] = str(conn.server_port)
-        environ['REMOTE_PORT'] = str(conn.client_port)
-        environ['REMOTE_ADDR'] = str(conn.client_addr)
         environ['REQUEST_METHOD'] = request['method']
         environ['PATH_INFO'] = request['path']
         environ['SERVER_PROTOCOL'] = request['protocol']
+        environ['SERVER_PORT'] = str(conn.server_port)
+        environ['REMOTE_PORT'] = str(conn.client_port)
+        environ['REMOTE_ADDR'] = str(conn.client_addr)
         environ['QUERY_STRING'] = request['query_string']
         if 'HTTP_CONTENT_LENGTH' in environ:
             environ['CONTENT_LENGTH'] = environ['HTTP_CONTENT_LENGTH']
@@ -1662,16 +1676,14 @@ class WSGIWorker(Worker):
         if conn.ssl:
             environ['wsgi.url_scheme'] = 'https'
             environ['HTTPS'] = 'on'
-        else:
-            environ['wsgi.url_scheme'] = 'http'
-
-        if conn.ssl:
             try:
                 peercert = conn.socket.getpeercert(binary_form=True)
                 environ['SSL_CLIENT_RAW_CERT'] = \
                     peercert and ssl.DER_cert_to_PEM_cert(peercert)
-            except Exception, e:
-                print e
+            except Exception:
+                print sys.exc_info()[1]
+        else:
+            environ['wsgi.url_scheme'] = 'http'
 
         if environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked':
             environ['wsgi.input'] = ChunkedReader(sock_file)
@@ -1684,36 +1696,36 @@ class WSGIWorker(Worker):
         h_set = self.header_set
 
         # Does the app want us to send output chunked?
-        self.chunked = h_set.get('transfer-encoding', '').lower() == 'chunked'
+        self.chunked = h_set.get('Transfer-Encoding', '').lower() == 'chunked'
 
         # Add a Date header if it's not there already
-        if not 'date' in h_set:
+        if not 'Date' in h_set:
             h_set['Date'] = formatdate(usegmt=True)
 
         # Add a Server header if it's not there already
-        if not 'server' in h_set:
+        if not 'Server' in h_set:
             h_set['Server'] = HTTP_SERVER_SOFTWARE
 
-        if 'content-length' in h_set:
-            self.size = int(h_set['content-length'])
+        if 'Content-Length' in h_set:
+            self.size = int(h_set['Content-Length'])
         else:
             s = int(self.status.split(' ')[0])
-            if s < 200 or s not in (204, 205, 304):
-                if not self.chunked:
-                    if sections == 1:
-                        # Add a Content-Length header if it's not there already
-                        h_set['Content-Length'] = str(len(data))
-                        self.size = len(data)
-                    else:
-                        # If they sent us more than one section, we blow chunks
-                        h_set['Transfer-Encoding'] = 'Chunked'
-                        self.chunked = True
-                        if __debug__:
-                            self.err_log.debug('Adding header...'
-                                               'Transfer-Encoding: Chunked')
+            if (s < 200 or s not in (204, 205, 304)) and not self.chunked:
+                if sections == 1 or self.protocol != 'HTTP/1.1':
+                    # Add a Content-Length header because it's not there
+                    self.size = len(data)
+                    h_set['Content-Length'] = str(self.size)
+                else:
+                    # If they sent us more than one section, we blow chunks
+                    h_set['Transfer-Encoding'] = 'Chunked'
+                    self.chunked = True
+                    if __debug__:
+                        self.err_log.debug('Adding header...'
+                                           'Transfer-Encoding: Chunked')
 
-        if 'connection' not in h_set:
-            # If the application did not provide a connection header, fill it in
+        if 'Connection' not in h_set:
+            # If the application did not provide a connection header,
+            # fill it in
             client_conn = self.environ.get('HTTP_CONNECTION', '').lower()
             if self.environ['SERVER_PROTOCOL'] == 'HTTP/1.1':
                 # HTTP = 1.1 defaults to keep-alive connections
@@ -1722,11 +1734,12 @@ class WSGIWorker(Worker):
                 else:
                     h_set['Connection'] = 'keep-alive'
             else:
-                # HTTP < 1.1 supports keep-alive but it's quirky so we don't support it
+                # HTTP < 1.1 supports keep-alive but it's quirky
+                # so we don't support it
                 h_set['Connection'] = 'close'
 
         # Close our connection if we need to.
-        self.closeConnection = h_set.get('connection', '').lower() == 'close'
+        self.closeConnection = h_set.get('Connection', '').lower() == 'close'
 
         # Build our output headers
         header_data = HEADER_RESPONSE % (self.status, str(h_set))
@@ -1758,6 +1771,8 @@ class WSGIWorker(Worker):
                     self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data)))
                 else:
                     self.conn.sendall(data)
+            except socket.timeout:
+                self.closeConnection = True
             except socket.error:
                 # But some clients will close the connection before that
                 # resulting in a socket error.
@@ -1853,55 +1868,4 @@ class WSGIWorker(Worker):
 
             sock_file.close()
 
-# Monolithic build...end of module: rocket\methods\wsgi.py
-
-#
-# the following code is not part of Rocket but was added in web2py for testing purposes
-#
-
-
-def demo_app(environ, start_response):
-    global static_folder
-    import os
-    types = {'htm': 'text/html', 'html': 'text/html', 'gif': 'image/gif',
-             'jpg': 'image/jpeg', 'png': 'image/png', 'pdf': 'applications/pdf'}
-    if static_folder:
-        if not static_folder.startswith('/'):
-            static_folder = os.path.join(os.getcwd(), static_folder)
-        path = os.path.join(
-            static_folder, environ['PATH_INFO'][1:] or 'index.html')
-        type = types.get(path.split('.')[-1], 'text')
-        if os.path.exists(path):
-            try:
-                data = open(path, 'rb').read()
-                start_response('200 OK', [('Content-Type', type)])
-            except IOError:
-                start_response('404 NOT FOUND', [])
-                data = '404 NOT FOUND'
-        else:
-            start_response('500 INTERNAL SERVER ERROR', [])
-            data = '500 INTERNAL SERVER ERROR'
-    else:
-        start_response('200 OK', [('Content-Type', 'text/html')])
-        data = '<html><body><h1>Hello from Rocket Web Server</h1></body></html>'
-    return [data]
-
-
-def demo():
-    from optparse import OptionParser
-    parser = OptionParser()
-    parser.add_option("-i", "--ip", dest="ip", default="127.0.0.1",
-                      help="ip address of the network interface")
-    parser.add_option("-p", "--port", dest="port", default="8000",
-                      help="post where to run web server")
-    parser.add_option("-s", "--static", dest="static", default=None,
-                      help="folder containing static files")
-    (options, args) = parser.parse_args()
-    global static_folder
-    static_folder = options.static
-    print 'Rocket running on %s:%s' % (options.ip, options.port)
-    r = Rocket((options.ip, int(options.port)), 'wsgi', {'wsgi_app': demo_app})
-    r.start()
-
-if __name__ == '__main__':
-    demo()
+# Monolithic build...end of module: rocket/methods/wsgi.py

gluon/rocket.py.footer

@@ -1,21 +1,22 @@
+# The following code is not part of Rocket but was added to
+# web2py for testing purposes.
 
-#
-# the following code is not part of Rocket but was added in web2py for testing purposes
-#
 
 def demo_app(environ, start_response):
     global static_folder
     import os
-    types = {'htm': 'text/html','html': 'text/html','gif': 'image/gif',
-             'jpg': 'image/jpeg','png': 'image/png','pdf': 'applications/pdf'}
+    types = {'htm': 'text/html', 'html': 'text/html',
+             'gif': 'image/gif', 'jpg': 'image/jpeg',
+             'png': 'image/png', 'pdf': 'applications/pdf'}
     if static_folder:
         if not static_folder.startswith('/'):
-            static_folder = os.path.join(os.getcwd(),static_folder)
-        path = os.path.join(static_folder, environ['PATH_INFO'][1:] or 'index.html')
-        type = types.get(path.split('.')[-1],'text')
+            static_folder = os.path.join(os.getcwd(), static_folder)
+        path = os.path.join(
+            static_folder, environ['PATH_INFO'][1:] or 'index.html')
+        type = types.get(path.split('.')[-1], 'text')
         if os.path.exists(path):
             try:
-                data = open(path,'rb').read()
+                data = open(path, 'rb').read()
                 start_response('200 OK', [('Content-Type', type)])
             except IOError:
                 start_response('404 NOT FOUND', [])
@@ -25,24 +26,26 @@ def demo_app(environ, start_response):
             data = '500 INTERNAL SERVER ERROR'
     else:
         start_response('200 OK', [('Content-Type', 'text/html')])
-        data = '<html><body><h1>Hello from Rocket Web Server</h1></body></html>'
+        data = '<html><body><h1>Hello from Rocket</h1></body></html>'
     return [data]
 
+
 def demo():
     from optparse import OptionParser
     parser = OptionParser()
-    parser.add_option("-i", "--ip", dest="ip",default="127.0.0.1",
+    parser.add_option("-i", "--ip", dest="ip", default="127.0.0.1",
                       help="ip address of the network interface")
-    parser.add_option("-p", "--port", dest="port",default="8000",
+    parser.add_option("-p", "--port", dest="port", default="8000",
                       help="post where to run web server")
-    parser.add_option("-s", "--static", dest="static",default=None,
+    parser.add_option("-s", "--static", dest="static", default=None,
                       help="folder containing static files")
     (options, args) = parser.parse_args()
     global static_folder
     static_folder = options.static
     print 'Rocket running on %s:%s' % (options.ip, options.port)
-    r=Rocket((options.ip,int(options.port)),'wsgi', {'wsgi_app':demo_app})
+    r = Rocket((options.ip, int(options.port)), 'wsgi', {'wsgi_app': demo_app})
     r.start()
 
-if __name__=='__main__':
+
+if __name__ == '__main__':
     demo()

gluon/sanitizer.py

@@ -220,7 +220,7 @@ def sanitize(text, permitted_tags=[
         'td': ['colspan'],
     },
         escape=True):
-    if not isinstance(text, str):
+    if not isinstance(text, basestring):
         return str(text)
     return XssCleaner(permitted_tags=permitted_tags,
                       allowed_attributes=allowed_attributes).strip(text, escape)

gluon/scheduler.py

@@ -40,8 +40,8 @@ http://127.0.0.1:8000/myapp/appadmin/select/db?query=db.scheduler_run.id>0
 ## view workers
 http://127.0.0.1:8000/myapp/appadmin/select/db?query=db.scheduler_worker.id>0
 
-## To install the scheduler as a permanent daemon on Linux (w/ Upstart), put the
-## following into /etc/init/web2py-scheduler.conf:
+## To install the scheduler as a permanent daemon on Linux (w/ Upstart), put
+## the following into /etc/init/web2py-scheduler.conf:
 ## (This assumes your web2py instance is installed in <user>'s home directory,
 ## running as <user>, with app <myapp>, on network interface eth0.)
 
@@ -63,7 +63,6 @@ import os
 import time
 import multiprocessing
 import sys
-import cStringIO
 import threading
 import traceback
 import signal
@@ -117,7 +116,7 @@ CALLABLETYPES = (types.LambdaType, types.FunctionType,
 
 class Task(object):
     def __init__(self, app, function, timeout, args='[]', vars='{}', **kwargs):
-        logger.debug(' new task allocated: %s.%s' % (app, function))
+        logger.debug(' new task allocated: %s.%s', app, function)
         self.app = app
         self.function = function
         self.timeout = timeout
@@ -131,11 +130,11 @@ class Task(object):
 
 class TaskReport(object):
     def __init__(self, status, result=None, output=None, tb=None):
-        logger.debug('    new task report: %s' % status)
+        logger.debug('    new task report: %s', status)
         if tb:
-            logger.debug('   traceback: %s' % tb)
+            logger.debug('   traceback: %s', tb)
         else:
-            logger.debug('   result: %s' % result)
+            logger.debug('   result: %s', result)
         self.status = status
         self.result = result
         self.output = output
@@ -206,7 +205,6 @@ def executor(queue, task, out):
         if task.app:
             os.chdir(os.environ['WEB2PY_PATH'])
             from gluon.shell import env, parse_path_info
-            from gluon.dal import BaseAdapter
             from gluon import current
             level = logging.getLogger().getEffectiveLevel()
             logging.getLogger().setLevel(logging.WARN)
@@ -215,7 +213,6 @@ def executor(queue, task, out):
             (a, c, f) = parse_path_info(task.app)
             _env = env(a=a, c=c, import_models=True)
             logging.getLogger().setLevel(level)
-            scheduler = current._scheduler
             f = task.function
             functions = current._scheduler.tasks
             if not functions:
@@ -436,14 +433,14 @@ class Scheduler(MetaScheduler):
         self.heartbeat = heartbeat
         self.worker_name = worker_name or socket.gethostname(
         ) + '#' + str(os.getpid())
-        self.worker_status = RUNNING, 1  # tuple containing status as recorded in
-                                        #the table, plus a boost parameter for
-                                        #hibernation (i.e. when someone stop the
-                                        #worker acting on the scheduler_worker table)
+        #list containing status as recorded in the table plus a boost parameter
+        #for hibernation (i.e. when someone stop the worker acting on the worker table)
+        self.worker_status = [RUNNING, 1]
         self.max_empty_runs = max_empty_runs
         self.discard_results = discard_results
         self.is_a_ticker = False
         self.do_assign_tasks = False
+        self.greedy = False
         self.utc_time = utc_time
 
         from gluon import current
@@ -463,7 +460,7 @@ class Scheduler(MetaScheduler):
 
     def define_tables(self, db, migrate):
         from gluon.dal import DEFAULT
-        logger.debug('defining tables (migrate=%s)' % migrate)
+        logger.debug('defining tables (migrate=%s)', migrate)
         now = self.now
         db.define_table(
             'scheduler_task',
@@ -533,14 +530,16 @@ class Scheduler(MetaScheduler):
             self.start_heartbeats()
             while True and self.have_heartbeat:
                 if self.worker_status[0] == DISABLED:
-                    logger.debug('Someone stopped me, sleeping until better times come (%s)' % self.worker_status[1])
+                    logger.debug('Someone stopped me, sleeping until better times come (%s)', self.worker_status[1])
                     self.sleep()
                     continue
                 logger.debug('looping...')
                 task = self.pop_task()
                 if task:
                     self.empty_runs = 0
+                    self.worker_status[0] = RUNNING
                     self.report_task(task, self.async(task))
+                    self.worker_status[0] = ACTIVE
                 else:
                     self.empty_runs += 1
                     logger.debug('sleeping...')
@@ -556,23 +555,29 @@ class Scheduler(MetaScheduler):
             logger.info('catched')
             self.die()
 
+    def wrapped_assign_tasks(self, db):
+        db.commit()  # ?don't know if it's useful, let's be completely sure
+        x = 0
+        while x < 10:
+            try:
+                self.assign_tasks(db)
+                db.commit()
+                break
+            except:
+                db.rollback()
+                logger.error('TICKER(%s): error assigning tasks', self.worker_name)
+                x += 1
+                time.sleep(0.5)
+
     def pop_task(self):
         now = self.now()
         db, st = self.db, self.db.scheduler_task
         if self.is_a_ticker and self.do_assign_tasks:
             #I'm a ticker, and 5 loops passed without reassigning tasks, let's do
             #that and loop again
-            db.commit()  # ?don't know if it's useful, let's be completely sure
-            while True:
-                try:
-                    self.assign_tasks()
-                    db.commit()
-                    break
-                except:
-                    db.rollback()
-                    logger.error('TICKER: error assigning tasks')
+            self.wrapped_assign_tasks(db)
             return None
-        db.commit()
+        #ready to process something
         grabbed = db(st.assigned_worker_name == self.worker_name)(
             st.status == ASSIGNED)
 
@@ -581,16 +586,23 @@ class Scheduler(MetaScheduler):
             task.update_record(status=RUNNING, last_run_time=now)
             #noone will touch my task!
             db.commit()
-            logger.debug('   work to do %s' % task.id)
+            logger.debug('   work to do %s', task.id)
         else:
-            logger.debug('nothing to do')
+            if self.greedy and self.is_a_ticker:
+                #there are other tasks ready to be assigned
+                logger.info('TICKER (%s): greedy loop', self.worker_name)
+                self.wrapped_assign_tasks(db)
+            else:
+                logger.info('nothing to do')
             return None
         next_run_time = task.last_run_time + datetime.timedelta(
             seconds=task.period)
         times_run = task.times_run + 1
         if times_run < task.repeats or task.repeats == 0:
+            #need to run (repeating task)
             run_again = True
         else:
+            #no need to run again
             run_again = False
         run_id = 0
         while True and not self.discard_results:
@@ -604,6 +616,7 @@ class Scheduler(MetaScheduler):
                 db.commit()
                 break
             except:
+                time.sleep(0.5)
                 db.rollback()
         logger.info('new task %(id)s "%(task_name)s" %(application_name)s.%(function_name)s' % task)
         return Task(
@@ -632,7 +645,7 @@ class Scheduler(MetaScheduler):
                         #result is 'null' as a string if task completed
                         #if it's stopped it's None as NoneType, so we record
                         #the STOPPED "run" anyway
-                        logger.debug(' recording task report in db (%s)' %
+                        logger.debug(' recording task report in db (%s)',
                                      task_report.status)
                         db(db.scheduler_run.id == task.run_id).update(
                             status=task_report.status,
@@ -643,6 +656,7 @@ class Scheduler(MetaScheduler):
                     else:
                         logger.debug(' deleting task report in db because of no result')
                         db(db.scheduler_run.id == task.run_id).delete()
+                #if there is a stop_time and the following run would exceed it
                 is_expired = (task.stop_time
                               and task.next_run_time > task.stop_time
                               and True or False)
@@ -665,21 +679,26 @@ class Scheduler(MetaScheduler):
                               and task.times_failed < task.retry_failed
                               and QUEUED or task.retry_failed == -1
                               and QUEUED or st_mapping)
-                    db(db.scheduler_task.id == task.task_id)(db.scheduler_task.status == RUNNING).update(
+                    db(
+                        (db.scheduler_task.id == task.task_id) &
+                        (db.scheduler_task.status == RUNNING)
+                    ).update(
                         times_failed=db.scheduler_task.times_failed + 1,
                         next_run_time=task.next_run_time,
-                        status=status)
+                        status=status
+                    )
                 db.commit()
-                logger.info('task completed (%s)' % task_report.status)
+                logger.info('task completed (%s)', task_report.status)
                 break
             except:
                 db.rollback()
+                time.sleep(0.5)
 
     def adj_hibernation(self):
         if self.worker_status[0] == DISABLED:
-            hibernation = self.worker_status[1] + 1 if self.worker_status[
-                1] < MAXHIBERNATION else MAXHIBERNATION
-            self.worker_status = DISABLED, hibernation
+            wk_st = self.worker_status[1]
+            hibernation = wk_st + 1 if wk_st < MAXHIBERNATION else MAXHIBERNATION
+            self.worker_status[1] = hibernation
 
     def send_heartbeat(self, counter):
         if not self.db_thread:
@@ -691,9 +710,6 @@ class Scheduler(MetaScheduler):
             db = self.db_thread
             sw, st = db.scheduler_worker, db.scheduler_task
             now = self.now()
-            expiration = now - datetime.timedelta(seconds=self.heartbeat * 3)
-            departure = now - datetime.timedelta(
-                seconds=self.heartbeat * 3 * MAXHIBERNATION)
             # record heartbeat
             mybackedstatus = db(
                 sw.worker_name == self.worker_name).select().first()
@@ -701,35 +717,39 @@ class Scheduler(MetaScheduler):
                 sw.insert(status=ACTIVE, worker_name=self.worker_name,
                           first_heartbeat=now, last_heartbeat=now,
                           group_names=self.group_names)
-                self.worker_status = ACTIVE, 1  # activating the process
+                self.worker_status = [ACTIVE, 1]  # activating the process
             else:
                 if mybackedstatus.status == DISABLED:
-                    self.worker_status = DISABLED, self.worker_status[
-                        1]  # keep sleeping
+                    # keep sleeping
+                    self.worker_status[0] = DISABLED
                     if self.worker_status[1] == MAXHIBERNATION:
                         logger.debug('........recording heartbeat')
                         db(sw.worker_name == self.worker_name).update(
                             last_heartbeat=now)
-
                 elif mybackedstatus.status == TERMINATE:
-                    self.worker_status = TERMINATE, self.worker_status[1]
+                    self.worker_status[0] = TERMINATE
                     logger.debug("Waiting to terminate the current task")
                     self.give_up()
                     return
                 elif mybackedstatus.status == KILL:
-                    self.worker_status = KILL, self.worker_status[1]
+                    self.worker_status[0] = KILL
                     self.die()
-
                 else:
-                    logger.debug('........recording heartbeat')
+                    logger.debug('........recording heartbeat (%s)', self.worker_status[0])
                     db(sw.worker_name == self.worker_name).update(
                         last_heartbeat=now, status=ACTIVE)
-                    self.worker_status = ACTIVE, 1  # re-activating the process
+                    self.worker_status[1] = 1  # re-activating the process
+                    if self.worker_status[0] <> RUNNING:
+                        self.worker_status[0] = ACTIVE
 
             self.do_assign_tasks = False
+
             if counter % 5 == 0:
                 try:
                     # delete inactive workers
+                    expiration = now - datetime.timedelta(seconds=self.heartbeat * 3)
+                    departure = now - datetime.timedelta(
+                        seconds=self.heartbeat * 3 * MAXHIBERNATION)
                     logger.debug(
                         '    freeing workers that have not sent heartbeat')
                     inactive_workers = db(
@@ -755,21 +775,31 @@ class Scheduler(MetaScheduler):
     def being_a_ticker(self):
         db = self.db_thread
         sw = db.scheduler_worker
-        ticker = db((sw.worker_name != self.worker_name) & (
-            sw.is_ticker == True) & (sw.status == ACTIVE)).select().first()
+        all_active = db(
+            (sw.worker_name != self.worker_name) & (sw.status == ACTIVE)
+        ).select()
+        ticker = all_active.find(lambda row: row.is_ticker is True).first()
+        not_busy = self.worker_status[0] == ACTIVE
         if not ticker:
-            db(sw.worker_name == self.worker_name).update(is_ticker=True)
-            db(sw.worker_name != self.worker_name).update(is_ticker=False)
-            logger.info("TICKER: I'm a ticker (%s)" % self.worker_name)
+            if not_busy:
+                #only if this worker isn't busy, otherwise wait for a free one
+                db(sw.worker_name == self.worker_name).update(is_ticker=True)
+                db(sw.worker_name != self.worker_name).update(is_ticker=False)
+                logger.info("TICKER(%s): I'm a ticker", self.worker_name)
+            else:
+                #giving up, only if I'm not alone
+                if len(all_active) > 1:
+                    db(sw.worker_name == self.worker_name).update(is_ticker=False)
+                else:
+                    not_busy = True
             db.commit()
-            return True
+            return not_busy
         else:
             logger.info(
                 "%s is a ticker, I'm a poor worker" % ticker.worker_name)
             return False
 
-    def assign_tasks(self):
-        db = self.db
+    def assign_tasks(self, db):
         sw, st = db.scheduler_worker, db.scheduler_task
         now = self.now()
         all_workers = db(sw.status == ACTIVE).select()
@@ -789,8 +819,15 @@ class Scheduler(MetaScheduler):
         db(st.status.belongs(
             (QUEUED, ASSIGNED)))(st.stop_time < now).update(status=EXPIRED)
 
-        all_available = db(st.status.belongs((QUEUED, ASSIGNED)))((st.times_run < st.repeats) | (st.repeats == 0))(st.start_time <= now)((st.stop_time == None) | (st.stop_time > now))(st.next_run_time <= now)(st.enabled == True)
-        limit = len(all_workers) * (50 / len(wkgroups))
+        all_available = db(
+            (st.status.belongs((QUEUED, ASSIGNED))) &
+            ((st.times_run < st.repeats) | (st.repeats == 0)) &
+            (st.start_time <= now) &
+            ((st.stop_time == None) | (st.stop_time > now)) &
+            (st.next_run_time <= now) &
+            (st.enabled == True)
+        )
+        limit = len(all_workers) * (50 / (len(wkgroups) or 1))
         #if there are a moltitude of tasks, let's figure out a maximum of tasks per worker.
         #this can be adjusted with some added intelligence (like esteeming how many tasks will
         #a worker complete before the ticker reassign them around, but the gain is quite small
@@ -804,11 +841,13 @@ class Scheduler(MetaScheduler):
 
         #let's freeze it up
         db.commit()
+        x = 0
         for group in wkgroups.keys():
-            tasks = all_available(st.group_name==group).select(
-                limitby=(0, limit), orderby=st.next_run_time)
+            tasks = all_available(st.group_name == group).select(
+                limitby=(0, limit), orderby = st.next_run_time)
             #let's break up the queue evenly among workers
             for task in tasks:
+                x += 1
                 gname = task.group_name
                 ws = wkgroups.get(gname)
                 if ws:
@@ -818,8 +857,10 @@ class Scheduler(MetaScheduler):
                         if w['c'] < counter:
                             myw = i
                         counter = w['c']
-                    d = dict(status=ASSIGNED,
-                             assigned_worker_name=wkgroups[gname]['workers'][myw]['name'])
+                    d = dict(
+                        status=ASSIGNED,
+                        assigned_worker_name=wkgroups[gname]['workers'][myw]['name']
+                    )
                     if not task.task_name:
                         d['task_name'] = task.function_name
                     task.update_record(**d)
@@ -827,14 +868,17 @@ class Scheduler(MetaScheduler):
 
         db.commit()
         #I didn't report tasks but I'm working nonetheless!!!!
-        if len(tasks) > 0:
+        if x > 0:
             self.empty_runs = 0
-        logger.info('TICKER: workers are %s' % len(all_workers))
-        logger.info('TICKER: tasks are %s' % len(tasks))
+        #I'll be greedy only if tasks assigned are equal to the limit
+        # (meaning there could be others ready to be assigned)
+        self.greedy = x >= limit and True or False
+        logger.info('TICKER(%s): workers are %s', self.worker_name, len(all_workers))
+        logger.info('TICKER(%s): tasks are %s', self.worker_name, x)
 
     def sleep(self):
         time.sleep(self.heartbeat * self.worker_status[1])
-                   # should only sleep until next available task
+        # should only sleep until next available task
 
     def queue_task(self, function, pargs=[], pvars={}, **kwargs):
         """
@@ -907,10 +951,10 @@ class Scheduler(MetaScheduler):
             orderby = ~st.id | ~sr.id
         row = self.db(q).select(
             *fields,
-            orderby=orderby,
-            left=left,
-            limitby=(0, 1)
-        ).first()
+            **dict(orderby=orderby,
+                   left=left,
+                   limitby=(0, 1))
+             ).first()
         if output:
             row.result = row.scheduler_run.result and \
                 loads(row.scheduler_run.result,

gluon/serializers.py

@@ -114,4 +114,4 @@ def rss(feed):
                            description=str(entry.get('description', '')),
                            pubDate=entry.get('created_on', now)
                            ) for entry in feed.get('entries', [])])
-    return rss2.dumps(rss)
+    return rss.to_xml(encoding='utf-8')

gluon/shell.py

@@ -220,7 +220,13 @@ def run(
     _env.update(exec_pythonrc())
     if startfile:
         try:
-            execfile(startfile, _env)
+            ccode = None
+            if startfile.endswith('.pyc'):
+                ccode = read_pyc(startfile)
+                exec ccode in _env
+            else:
+                execfile(startfile, _env)
+
             if import_models:
                 BaseAdapter.close_all_instances('commit')
         except Exception, e:

gluon/sqlhtml.py

@@ -263,16 +263,19 @@ class ListWidget(StringWidget):
 jQuery.fn.grow_input = function() {
   return this.each(function() {
     var ul = this;
-    jQuery(ul).find(":text").after('<a href="javascript:void(0)">+</a>').keypress(function (e) { return (e.which == 13) ? pe(ul) : true; }).next().click(function(){ pe(ul) });
+    jQuery(ul).find(":text").after('<a href="javascript:void(0)">+</a>&nbsp;<a href="javascript:void(0)">-</a>').keypress(function (e) { return (e.which == 13) ? pe(ul, e) : true; }).next().click(function(e){ pe(ul, e) }).next().click(function(e){ rl(ul, e)});
   });
 };
-function pe(ul) {
+function pe(ul, e) {
   var new_line = ml(ul);
   rel(ul);
-  new_line.appendTo(ul);
+  new_line.insertAfter(jQuery(e.target).parent());
   new_line.find(":text").focus();
   return false;
 }
+function rl(ul, e) {
+  jQuery(e.target).parent().remove();
+}
 function ml(ul) {
   var line = jQuery(ul).find("li:first").clone(true);
   line.find(':text').val('');
@@ -657,7 +660,7 @@ class AutocompleteWidget(object):
             attr['value'] = record and record[self.fields[0].name]
             attr['_onblur'] = "jQuery('#%(div_id)s').delay(1000).fadeOut('slow');" % \
                 dict(div_id=div_id, u='F' + self.keyword)
-            attr['_onkeyup'] = "jQuery('#%(key3)s').val('');var e=event.which?event.which:event.keyCode; function %(u)s(){jQuery('#%(id)s').val(jQuery('#%(key)s :selected').text());jQuery('#%(key3)s').val(jQuery('#%(key)s').val())}; if(e==39) %(u)s(); else if(e==40) {if(jQuery('#%(key)s option:selected').next().length)jQuery('#%(key)s option:selected').attr('selected',null).next().attr('selected','selected'); %(u)s();} else if(e==38) {if(jQuery('#%(key)s option:selected').prev().length)jQuery('#%(key)s option:selected').attr('selected',null).prev().attr('selected','selected'); %(u)s();} else if(jQuery('#%(id)s').val().length>=%(min_length)s) jQuery.get('%(url)s?%(key)s='+escape(jQuery('#%(id)s').val()),function(data){if(data=='')jQuery('#%(key3)s').val('');else{jQuery('#%(id)s').next('.error').hide();jQuery('#%(div_id)s').html(data).show().focus();jQuery('#%(div_id)s select').css('width',jQuery('#%(id)s').css('width'));jQuery('#%(key3)s').val(jQuery('#%(key)s').val());jQuery('#%(key)s').change(%(u)s);jQuery('#%(key)s').click(%(u)s);};}); else jQuery('#%(div_id)s').fadeOut('slow');" % \
+            attr['_onkeyup'] = "jQuery('#%(key3)s').val('');var e=event.which?event.which:event.keyCode; function %(u)s(){jQuery('#%(id)s').val(jQuery('#%(key)s :selected').text());jQuery('#%(key3)s').val(jQuery('#%(key)s').val())}; if(e==39) %(u)s(); else if(e==40) {if(jQuery('#%(key)s option:selected').next().length)jQuery('#%(key)s option:selected').attr('selected',null).next().attr('selected','selected'); %(u)s();} else if(e==38) {if(jQuery('#%(key)s option:selected').prev().length)jQuery('#%(key)s option:selected').attr('selected',null).prev().attr('selected','selected'); %(u)s();} else if(jQuery('#%(id)s').val().length>=%(min_length)s) jQuery.get('%(url)s?%(key)s='+encodeURIComponent(jQuery('#%(id)s').val()),function(data){if(data=='')jQuery('#%(key3)s').val('');else{jQuery('#%(id)s').next('.error').hide();jQuery('#%(div_id)s').html(data).show().focus();jQuery('#%(div_id)s select').css('width',jQuery('#%(id)s').css('width'));jQuery('#%(key3)s').val(jQuery('#%(key)s').val());jQuery('#%(key)s').change(%(u)s);jQuery('#%(key)s').click(%(u)s);};}); else jQuery('#%(div_id)s').fadeOut('slow');" % \
                 dict(url=self.url, min_length=self.min_length,
                      key=self.keyword, id=attr['_id'], key2=key2, key3=key3,
                      name=name, div_id=div_id, u='F' + self.keyword)
@@ -670,7 +673,7 @@ class AutocompleteWidget(object):
             attr['_name'] = field.name
             attr['_onblur'] = "jQuery('#%(div_id)s').delay(1000).fadeOut('slow');" % \
                 dict(div_id=div_id, u='F' + self.keyword)
-            attr['_onkeyup'] = "var e=event.which?event.which:event.keyCode; function %(u)s(){jQuery('#%(id)s').val(jQuery('#%(key)s').val())}; if(e==39) %(u)s(); else if(e==40) {if(jQuery('#%(key)s option:selected').next().length)jQuery('#%(key)s option:selected').attr('selected',null).next().attr('selected','selected'); %(u)s();} else if(e==38) {if(jQuery('#%(key)s option:selected').prev().length)jQuery('#%(key)s option:selected').attr('selected',null).prev().attr('selected','selected'); %(u)s();} else if(jQuery('#%(id)s').val().length>=%(min_length)s) jQuery.get('%(url)s?%(key)s='+escape(jQuery('#%(id)s').val()),function(data){jQuery('#%(id)s').next('.error').hide();jQuery('#%(div_id)s').html(data).show().focus();jQuery('#%(div_id)s select').css('width',jQuery('#%(id)s').css('width'));jQuery('#%(key)s').change(%(u)s);jQuery('#%(key)s').click(%(u)s);}); else jQuery('#%(div_id)s').fadeOut('slow');" % \
+            attr['_onkeyup'] = "var e=event.which?event.which:event.keyCode; function %(u)s(){jQuery('#%(id)s').val(jQuery('#%(key)s').val())}; if(e==39) %(u)s(); else if(e==40) {if(jQuery('#%(key)s option:selected').next().length)jQuery('#%(key)s option:selected').attr('selected',null).next().attr('selected','selected'); %(u)s();} else if(e==38) {if(jQuery('#%(key)s option:selected').prev().length)jQuery('#%(key)s option:selected').attr('selected',null).prev().attr('selected','selected'); %(u)s();} else if(jQuery('#%(id)s').val().length>=%(min_length)s) jQuery.get('%(url)s?%(key)s='+encodeURIComponent(jQuery('#%(id)s').val()),function(data){jQuery('#%(id)s').next('.error').hide();jQuery('#%(div_id)s').html(data).show().focus();jQuery('#%(div_id)s select').css('width',jQuery('#%(id)s').css('width'));jQuery('#%(key)s').change(%(u)s);jQuery('#%(key)s').click(%(u)s);}); else jQuery('#%(div_id)s').fadeOut('slow');" % \
                 dict(url=self.url, min_length=self.min_length,
                      key=self.keyword, id=attr['_id'], div_id=div_id, u='F' + self.keyword)
             if self.min_length == 0:
@@ -751,7 +754,13 @@ def formstyle_bootstrap(form, fields):
                 # flag submit button
                 _submit = True
                 controls['_class'] = 'btn btn-primary'
+            if controls['_type'] == 'file':
+                controls['_class'] = 'input-file'
 
+        # For password fields, which are wrapped in a CAT object.
+        if isinstance(controls, CAT) and isinstance(controls[0], INPUT):
+            controls[0].add_class('input-xlarge')
+                
         if isinstance(controls, SELECT):
             controls.add_class('input-xlarge')
 
@@ -1308,8 +1317,8 @@ class SQLFORM(FORM):
                         not OptionsWidget.has_options(field):
                     field.widget = self.widgets.list.widget
                 if field.widget and fieldname in request_vars:
-                    if fieldname in self.vars:
-                        value = self.vars[fieldname]
+                    if fieldname in self.request_vars:
+                        value = self.request_vars[fieldname]
                     elif self.record:
                         value = self.record[fieldname]
                     else:
@@ -1322,7 +1331,8 @@ class SQLFORM(FORM):
                     parent = self.field_parent[row_id]
                     if parent:
                         parent.components = [widget]
-                        parent._traverse(False, hideerror)
+                        if self.errors.get(fieldname):
+                            parent._traverse(False, hideerror)
                     self.custom.widget[fieldname] = widget
             self.accepted = ret
             return ret
@@ -1553,18 +1563,26 @@ class SQLFORM(FORM):
             return smart_query(fields, key)
 
     @staticmethod
-    def search_menu(fields, search_options=None):
+    def search_menu(fields,
+                    search_options=None,
+                    prefix='w2p'
+                    ):
         T = current.T
+        panel_id='%s_query_panel' % prefix
+        fields_id='%s_query_fields' % prefix
+        keywords_id='%s_keywords' % prefix
+        field_id='%s_field' % prefix
+        value_id='%s_value' % prefix
         search_options = search_options or {
-            'string': ['=', '!=', '<', '>', '<=', '>=', 'starts with', 'contains'],
-            'text': ['=', '!=', '<', '>', '<=', '>=', 'starts with', 'contains'],
+            'string': ['=', '!=', '<', '>', '<=', '>=', 'starts with', 'contains', 'in', 'not in'],
+            'text': ['=', '!=', '<', '>', '<=', '>=', 'starts with', 'contains', 'in', 'not in'],
             'date': ['=', '!=', '<', '>', '<=', '>='],
             'time': ['=', '!=', '<', '>', '<=', '>='],
             'datetime': ['=', '!=', '<', '>', '<=', '>='],
-            'integer': ['=', '!=', '<', '>', '<=', '>='],
+            'integer': ['=', '!=', '<', '>', '<=', '>=', 'in', 'not in'],
             'double': ['=', '!=', '<', '>', '<=', '>='],
-            'id': ['=', '!=', '<', '>', '<=', '>='],
-            'reference': ['=', '!=', '<', '>', '<=', '>='],
+            'id': ['=', '!=', '<', '>', '<=', '>=', 'in', 'not in'],
+            'reference': ['=', '!=', '<', '>', '<=', '>=', 'in', 'not in'],
             'boolean': ['=', '!=']}
         if fields[0]._db._adapter.dbengine == 'google:datastore':
             search_options['string'] = ['=', '!=', '<', '>', '<=', '>=']
@@ -1581,58 +1599,62 @@ class SQLFORM(FORM):
                 label = isinstance(
                     field.label, str) and T(field.label) or field.label
                 selectfields.append(OPTION(label, _value=str(field)))
-                operators = SELECT(*[T(option) for option in options])
+                operators = SELECT(*[OPTION(T(option), _value=option) for option in options])
                 if field.type == 'boolean':
                     value_input = SELECT(
                         OPTION(T("True"), _value="T"),
                         OPTION(T("False"), _value="F"),
-                        _id="w2p_value_" + name)
+                        _id="%s_%s" % (value_id,name))
                 else:
                     value_input = INPUT(_type='text',
-                                        _id="w2p_value_" + name,
+                                        _id="%s_%s" % (value_id,name),
                                         _class=field.type)
                 new_button = INPUT(
                     _type="button", _value=T('New'), _class="btn",
-                    _onclick="w2p_build_query('new','%s')" % field)
+                    _onclick="%s_build_query('new','%s')" % (prefix,field))
                 and_button = INPUT(
                     _type="button", _value=T('And'), _class="btn",
-                    _onclick="w2p_build_query('and','%s')" % field)
+                    _onclick="%s_build_query('and','%s')" % (prefix, field))
                 or_button = INPUT(
                     _type="button", _value=T('Or'), _class="btn",
-                    _onclick="w2p_build_query('or','%s')" % field)
+                    _onclick="%s_build_query('or','%s')" % (prefix, field))
                 close_button = INPUT(
                     _type="button", _value=T('Close'), _class="btn",
-                    _onclick="jQuery('#w2p_query_panel').slideUp()")
+                    _onclick="jQuery('#%s').slideUp()" % panel_id)
 
                 criteria.append(DIV(
                     operators, value_input, new_button,
                     and_button, or_button, close_button,
-                    _id='w2p_field_%s' % name,
+                    _id='%s_%s' % (field_id, name),
                         _class='w2p_query_row hidden',
                         _style='display:inline'))
 
         criteria.insert(0, SELECT(
-            _id="w2p_query_fields",
-                _onchange="jQuery('.w2p_query_row').hide();jQuery('#w2p_field_'+jQuery('#w2p_query_fields').val().replace('.','-')).show();",
+            _id=fields_id,
+                _onchange="jQuery('.w2p_query_row').hide();jQuery('#%s_'+jQuery('#%s').val().replace('.','-')).show();" % (field_id,fields_id),
                 _style='float:left',
                 *selectfields))
 
         fadd = SCRIPT("""
-        jQuery('#w2p_query_panel input,#w2p_query_panel select').css(
+        jQuery('#%(fields_id)s input,#%(fields_id)s select').css(
             'width','auto');
-        jQuery(function(){web2py_ajax_fields('#w2p_query_panel');});
-        function w2p_build_query(aggregator,a) {
+        jQuery(function(){web2py_ajax_fields('#%(fields_id)s');});
+        function %(prefix)s_build_query(aggregator,a) {
           var b=a.replace('.','-');
-          var option = jQuery('#w2p_field_'+b+' select').val();
-          var value = jQuery('#w2p_value_'+b).val().replace('"','\\\\"');
+          var option = jQuery('#%(field_id)s_'+b+' select').val();
+          var value = jQuery('#%(value_id)s_'+b).val().replace('"','\\\\"');
           var s=a+' '+option+' "'+value+'"';
-          var k=jQuery('#web2py_keywords');
+          var k=jQuery('#%(keywords_id)s');
           var v=k.val();
           if(aggregator=='new') k.val(s); else k.val((v?(v+' '+ aggregator +' '):'')+s);
         }
-        """)
+        """ % dict(
+                prefix=prefix,fields_id=fields_id,keywords_id=keywords_id,
+                field_id=field_id,value_id=value_id
+                )
+        )
         return CAT(
-            DIV(_id="w2p_query_panel", _class='hidden', *criteria), fadd)
+            DIV(_id=panel_id, _class='hidden', *criteria), fadd)
 
     @staticmethod
     def grid(query,
@@ -1675,6 +1697,8 @@ class SQLFORM(FORM):
              createargs={},
              editargs={},
              viewargs={},
+             buttons_placement = 'right',
+             links_placement = 'right'
              ):
 
         # jQuery UI ThemeRoller classes (empty if ui is disabled)
@@ -1722,19 +1746,26 @@ class SQLFORM(FORM):
         request = current.request
         session = current.session
         response = current.response
-        wenabled = (not user_signature or (session.auth and session.auth.user))
+        logged = session.auth and session.auth.user
+        wenabled = (not user_signature or logged)
         create = wenabled and create
         editable = wenabled and editable
         deletable = wenabled and deletable
 
         def url(**b):
             b['args'] = args + b.get('args', [])
+            localvars = request.vars.copy()
+            localvars.update(b.get('vars', {}))
+            b['vars'] = localvars
             b['hash_vars'] = False
             b['user_signature'] = user_signature
             return URL(**b)
 
         def url2(**b):
             b['args'] = request.args + b.get('args', [])
+            localvars = request.vars.copy()
+            localvars.update(b.get('vars', {}))
+            b['vars'] = localvars
             b['hash_vars'] = False
             b['user_signature'] = user_signature
             return URL(**b)
@@ -1746,13 +1777,11 @@ class SQLFORM(FORM):
         # - url has valid signature (vars are not signed, only path_info)
         # = url does not contain 'create','delete','edit' (readonly)
         if user_signature:
-            if not(
+            if not (
                 '/'.join(str(a) for a in args) == '/'.join(request.args) or
-                URL.verify(request, user_signature=user_signature,
-                           hash_vars=False) or not (
-                    'create' in request.args or
-                    'delete' in request.args or
-                    'edit' in request.args)):
+                URL.verify(request,user_signature=user_signature,
+                           hash_vars=False) or                
+                (request.args(len(args))=='view' and not logged)):
                 session.flash = T('not authorized')
                 redirect(referrer)
 
@@ -1824,6 +1853,14 @@ class SQLFORM(FORM):
                         buttons.append(link(record))
             return buttons
 
+        def linsert(lst, i, x):
+            """
+            a = [1,2]
+            linsert(a, 1, [0,3])
+            a = [1, 0, 3, 2]
+            """
+            lst[i:i] = x
+
         formfooter = DIV(
             _class='form_footer row_buttons %(header)s %(cornerbottom)s' % ui)
 
@@ -1850,7 +1887,7 @@ class SQLFORM(FORM):
 
         elif details and len(request.args) > 2 and request.args[-3] == 'view':
             table = db[request.args[-2]]
-            record = table(request.args[-1]) or redirect(URL('error'))
+            record = table(request.args[-1]) or redirect(referrer)
             sqlformargs.update(viewargs)
             view_form = SQLFORM(
                 table, record, upload=upload, ignore_rw=ignore_rw,
@@ -1903,6 +1940,9 @@ class SQLFORM(FORM):
                 (ExporterTSV, 'TSV (Excel compatible, hidden cols)'),
             tsv=(ExporterTSV, 'TSV (Excel compatible)'))
         if not exportclasses is None:
+            """
+            remember: allow to set exportclasses=dict(csv=False) to disable the csv format
+            """
             exportManager.update(exportclasses)
 
         export_type = request.vars._export_type
@@ -1919,23 +1959,22 @@ class SQLFORM(FORM):
                     if sign == '~':
                         orderby = ~orderby
 
-            table_fields = [f for f in fields if f._tablename in tablenames]
-            if export_type in ('csv_with_hidden_cols', 'tsv_with_hidden_cols'):
+            expcolumns = columns
+            if export_type.endswith('with_hidden_cols'):
+                expcolumns = [f for f in fields if f._tablename in tablenames]
+            if export_type in exportManager and exportManager[export_type]:
                 if request.vars.keywords:
                     try:
                         dbset = dbset(SQLFORM.build_query(
                             fields, request.vars.get('keywords', '')))
-                        rows = dbset.select(cacheable=True)
+                        rows = dbset.select(cacheable=True, *expcolumns)
                     except Exception, e:
                         response.flash = T('Internal Error')
                         rows = []
                 else:
-                    rows = dbset.select(cacheable=True)
-            else:
-                rows = dbset.select(left=left, orderby=orderby,
-                                    cacheable=True, *columns)
+                    rows = dbset.select(left=left, orderby=orderby,
+                                    cacheable=True, *expcolumns)
 
-            if export_type in exportManager:
                 value = exportManager[export_type]
                 clazz = value[0] if hasattr(value, '__getitem__') else value
                 oExp = clazz(rows)
@@ -1970,13 +2009,18 @@ class SQLFORM(FORM):
             if isinstance(search_widget, dict):
                 search_widget = search_widget[tablename]
             if search_widget == 'default':
-                search_menu = SQLFORM.search_menu(sfields)
+                prefix = formname == 'web2py_grid' and 'w2p' or 'w2p_%s' % formname
+                search_menu = SQLFORM.search_menu(sfields, prefix=prefix)
+                spanel_id = '%s_query_fields' % prefix
+                sfields_id = '%s_query_panel' % prefix
+                skeywords_id = '%s_keywords' % prefix
                 search_widget = lambda sfield, url: CAT(FORM(
                     INPUT(_name='keywords', _value=request.vars.keywords,
-                          _id='web2py_keywords', _onfocus="jQuery('#w2p_query_fields').change();jQuery('#w2p_query_panel').slideDown();"),
+                          _id=skeywords_id,
+                          _onfocus="jQuery('#%s').change();jQuery('#%s').slideDown();" % (spanel_id, sfields_id)),
                     INPUT(_type='submit', _value=T('Search'), _class="btn"),
                     INPUT(_type='submit', _value=T('Clear'), _class="btn",
-                          _onclick="jQuery('#web2py_keywords').val('');"),
+                          _onclick="jQuery('#%s').val('');" % skeywords_id),
                     _method="GET", _action=url), search_menu)
             form = search_widget and search_widget(sfields, url()) or ''
             console.append(add)
@@ -2000,6 +2044,9 @@ class SQLFORM(FORM):
                 c = 'count(*)'
                 nrows = dbset.select(c, left=left, cacheable=True,
                                      groupby=groupby).first()[c]
+            elif dbset._db._adapter.dbengine=='google:datastore':
+                #if we don't set a limit, this can timeout for a large table
+                nrows = dbset.db._adapter.count(dbset.query, limit=1000)
             else:
                 nrows = dbset.count()
         except:
@@ -2017,9 +2064,9 @@ class SQLFORM(FORM):
                 else:
                     orderby = (order[:1] == '~' and ~sort_field) or sort_field
 
-        head = TR(_class=ui.get('header'))
+        headcols = []
         if selectable:
-            head.append(TH(_class=ui.get('default')))
+            headcols.append(TH(_class=ui.get('default')))
         for field in fields:
             if columns and not str(field) in columns:
                 continue
@@ -2037,22 +2084,88 @@ class SQLFORM(FORM):
                 header = A(header, marker, _href=url(vars=dict(
                     keywords=request.vars.keywords or '',
                     order=key)), _class=trap_class())
-            head.append(TH(header, _class=ui.get('default')))
+            headcols.append(TH(header, _class=ui.get('default')))
 
+        toadd = []
         if links and links_in_grid:
             for link in links:
                 if isinstance(link, dict):
-                    head.append(TH(link['header'], _class=ui.get('default')))
+                    toadd.append(TH(link['header'], _class=ui.get('default')))
+            if links_placement in ['right', 'both']:
+                headcols.extend(toadd)
+            if links_placement in ['left', 'both']:
+                linsert(headcols, 0, toadd)
 
         # Include extra column for buttons if needed.
         include_buttons_column = (details or editable or deletable or
                                   (links and links_in_grid and
                                    not all([isinstance(link, dict) for link in links])))
         if include_buttons_column:
-            head.append(TH(_class=ui.get('default')))
+            if buttons_placement in ['right', 'both']:
+                headcols.append(TH(_class=ui.get('default','')))
+            if buttons_placement in ['left', 'both']:
+                headcols.insert(0, TH(_class=ui.get('default','')))
+
+        head = TR(*headcols, **dict(_class=ui.get('header')))
+
+        cursor = True
+        #figure out what page we are one to setup the limitby
+        if paginate and dbset._db._adapter.dbengine=='google:datastore':
+            cursor = request.vars.cursor or True
+            limitby = (0, paginate)
+            try: page = int(request.vars.page or 1)-1
+            except ValueError: page = 0
+        elif paginate and paginate<nrows:
+            try: page = int(request.vars.page or 1)-1
+            except ValueError: page = 0
+            limitby = (paginate*page,paginate*(page+1))
+        else:
+            limitby = None
+
+        try:
+            table_fields = [f for f in fields if f._tablename in tablenames]
+            if dbset._db._adapter.dbengine=='google:datastore':
+                rows = dbset.select(left=left,orderby=orderby,
+                                    groupby=groupby,limitby=limitby,
+                                    reusecursor=cursor,
+                                    cacheable=True,*table_fields)
+                next_cursor = dbset._db.get('_lastcursor', None)
+            else:
+                rows = dbset.select(left=left,orderby=orderby,
+                                    groupby=groupby,limitby=limitby,
+                                    cacheable=True,*table_fields)
+        except SyntaxError:
+            rows = None
+            next_cursor = None
+            error = T("Query Not Supported")
+        except Exception, e:
+            rows = None
+            next_cursor = None
+            error = T("Query Not Supported: %s")%e
+
+        message = error
+        if not message and nrows:
+            if dbset._db._adapter.dbengine=='google:datastore' and nrows>=1000:
+                message = T('at least %(nrows)s records found') % dict(nrows=nrows)
+            else:
+                message = T('%(nrows)s records found') % dict(nrows=nrows)
+        console.append(DIV(message,_class='web2py_counter'))
 
         paginator = UL()
-        if paginate and paginate < nrows:
+        if paginate and dbset._db._adapter.dbengine=='google:datastore':
+            #this means we may have a large table with an unknown number of rows.
+            try:
+                page = int(request.vars.page or 1)-1
+            except ValueError:
+                page = 0
+            paginator.append(LI('page %s'%(page+1)))
+            if next_cursor:
+                d = dict(page=page+2, cursor=next_cursor)
+                if order: d['order']=order
+                if request.vars.keywords: d['keywords']=request.vars.keywords
+                paginator.append(LI(
+                    A('next',_href=url(vars=d),_class=trap_class())))
+        elif paginate and paginate<nrows:
             npages, reminder = divmod(nrows, paginate)
             if reminder:
                 npages += 1
@@ -2060,7 +2173,6 @@ class SQLFORM(FORM):
                 page = int(request.vars.page or 1) - 1
             except ValueError:
                 page = 0
-            limitby = (paginate * page, paginate * (page + 1))
 
             def self_link(name, p):
                 d = dict(page=p + 1)
@@ -2088,38 +2200,15 @@ class SQLFORM(FORM):
         else:
             limitby = None
 
-        try:
-            table_fields = [f for f in fields if f._tablename in tablenames]
-            rows = dbset.select(left=left, orderby=orderby,
-                                groupby=groupby, limitby=limitby,
-                                *table_fields)
-        except SyntaxError:
-            rows = None
-            error = T("Query Not Supported")
-        if nrows:
-            message = error or T('%(nrows)s records found') % dict(nrows=nrows)
-            console.append(DIV(message, _class='web2py_counter'))
-
         if rows:
             htmltable = TABLE(THEAD(head))
             tbody = TBODY()
             numrec = 0
             for row in rows:
-                if numrec % 2 == 0:
-                    classtr = 'even'
-                else:
-                    classtr = 'odd'
-                numrec += 1
+                trcols = []
                 id = row[field_id]
-                if id:
-                    rid = id
-                    if callable(rid):  # can this ever be callable?
-                        rid = rid(row)
-                    tr = TR(_id=rid, _class='%s %s' % (classtr, 'with_id'))
-                else:
-                    tr = TR(_class=classtr)
                 if selectable:
-                    tr.append(
+                    trcols.append(
                         INPUT(_type="checkbox", _name="records", _value=id,
                                     value=request.vars.records))
                 for field in fields:
@@ -2157,15 +2246,21 @@ class SQLFORM(FORM):
                         value = truncate_string(value, maxlength)
                     elif not isinstance(value, DIV):
                         value = field.formatter(value)
-                    tr.append(TD(value))
+                    trcols.append(TD(value))
                 row_buttons = TD(_class='row_buttons')
                 if links and links_in_grid:
+                    toadd = []
                     for link in links:
                         if isinstance(link, dict):
-                            tr.append(TD(link['body'](row)))
+                            toadd.append(TD(link['body'](row)))
                         else:
                             if link(row):
                                 row_buttons.append(link(row))
+                    if links_placement in ['right', 'both']:
+                        trcols.extend(toadd)
+                    if links_placement in ['left', 'both']:
+                        linsert(trcols, 0, toadd)
+
                 if include_buttons_column:
                     if details and (not callable(details) or details(row)):
                         row_buttons.append(gridbutton(
@@ -2180,7 +2275,24 @@ class SQLFORM(FORM):
                             'buttondelete', 'Delete',
                             callback=url(args=['delete', tablename, id]),
                             delete='tr'))
-                    tr.append(row_buttons)
+                    if buttons_placement in ['right', 'both']:
+                        trcols.append(row_buttons)
+                    if buttons_placement in ['left', 'both']:
+                        trcols.insert(0, row_buttons)
+                if numrec % 2 == 0:
+                    classtr = 'even'
+                else:
+                    classtr = 'odd'
+                numrec += 1
+                if id:
+                    rid = id
+                    if callable(rid):  # can this ever be callable?
+                        rid = rid(row)
+                    tr = TR(*trcols, **dict(
+                            _id=rid,
+                            _class='%s %s' % (classtr, 'with_id')))
+                else:
+                    tr = TR(*trcols, **dict(_class=classtr))
                 tbody.append(tr)
             htmltable.append(tbody)
             htmltable = DIV(htmltable, _style='width:100%;overflow-x:auto')
@@ -2198,6 +2310,8 @@ class SQLFORM(FORM):
         if csv and nrows:
             export_links = []
             for k, v in sorted(exportManager.items()):
+                if not v:
+                    continue
                 label = v[1] if hasattr(v, "__getitem__") else k
                 link = url2(vars=dict(
                     order=request.vars.order or '',
@@ -2250,7 +2364,7 @@ class SQLFORM(FORM):
             table: pagination, search, view, edit, delete,
                    children, parent, etc.
 
-        constraints is a dict {'table',query} that limits which
+        constraints is a dict {'table':query} that limits which
         records can be accessible
         links is a dict like
            {'tablename':[lambda row: A(....), ...]}
@@ -2311,7 +2425,6 @@ class SQLFORM(FORM):
                             name = format % record
                     except TypeError:
                         name = id
-                    nameLink = 'view'
                     breadcrumbs.append(
                         LI(A(T(db[referee]._plural),
                              _class=trap_class(),
@@ -2352,9 +2465,8 @@ class SQLFORM(FORM):
         check = {}
         id_field_name = table._id.name
         for rfield in table._referenced_by:
-            if rfield.readable:
-                check[rfield.tablename] = \
-                    check.get(rfield.tablename, []) + [rfield.name]
+            check[rfield.tablename] = \
+                check.get(rfield.tablename, []) + [rfield.name]
         if isinstance(linked_tables, dict):
             linked_tables = linked_tables.get(table._tablename, [])
         for tablename in sorted(check):
@@ -2550,7 +2662,7 @@ class SQLTABLE(TABLE):
                 (tablename, fieldname) = colname.split('.')
                 try:
                     field = sqlrows.db[tablename][fieldname]
-                except KeyError:
+                except (KeyError, AttributeError):
                     field = None
                 if tablename in record \
                         and isinstance(record, Row) \
@@ -2561,7 +2673,7 @@ class SQLTABLE(TABLE):
                 else:
                     raise SyntaxError('something wrong in Rows object')
                 r_old = r
-                if not field:
+                if not field or isinstance(field, (Field.Virtual, Field.Lazy)):
                     pass
                 elif linkto and field.type == 'id':
                     try:
@@ -2740,7 +2852,6 @@ class ExporterTSV(ExportClass):
         writer = csv.writer(out, delimiter='\t')
         import codecs
         final.write(codecs.BOM_UTF16)
-        colnames = [a.split('.') for a in self.rows.colnames]
         writer.writerow(
             [unicode(col).encode("utf8") for col in self.rows.colnames])
         data = out.getvalue().decode("utf8")

gluon/storage.py

@@ -70,8 +70,10 @@ class Storage(dict):
         []
         """
         value = self.get(key, [])
-        return value if not value else \
-            value if isinstance(value, (list, tuple)) else [value]
+        if value is None or isinstance(value, (list, tuple)):
+            return value
+        else:
+            return [value]
 
     def getfirst(self, key, default=None):
         """

gluon/template.py

@@ -16,9 +16,13 @@ Contributors:
 
 import os
 import cgi
-import cStringIO
 import logging
 from re import compile, sub, escape, DOTALL
+try:
+    import cStringIO as StringIO
+except:
+    from io import StringIO
+
 try:
     # have web2py
     from restricted import RestrictedError
@@ -791,13 +795,13 @@ def get_parsed(text):
 
 class DummyResponse():
     def __init__(self):
-        self.body = cStringIO.StringIO()
+        self.body = StringIO.StringIO()
 
     def write(self, data, escape=True):
         if not escape:
             self.body.write(str(data))
-        elif hasattr(data, 'xml') and callable(data.xml):
-            self.body.write(data.xml())
+        elif hasattr(data, 'as_html') and callable(data.as_html):
+            self.body.write(data.as_html())
         else:
             # make it a string
             if not isinstance(data, (str, unicode)):
@@ -870,7 +874,7 @@ def render(content="hello world",
     # save current response class
     if context and 'response' in context:
         old_response_body = context['response'].body
-        context['response'].body = cStringIO.StringIO()
+        context['response'].body = StringIO.StringIO()
     else:
         old_response_body = None
         context['response'] = Response()
@@ -887,7 +891,7 @@ def render(content="hello world",
             stream = open(filename, 'rb')
             close_stream = True
         elif content:
-            stream = cStringIO.StringIO(content)
+            stream = StringIO.StringIO(content)
 
     # Execute the template.
     code = str(TemplateParser(stream.read(

gluon/tests/test_dal.py

@@ -13,7 +13,10 @@ else:
 
 import unittest
 import datetime
-import cStringIO
+try:
+    import cStringIO as StringIO
+except:
+    from io import StringIO
 from dal import DAL, Field, Table, SQLALL
 
 ALLOWED_DATATYPES = [
@@ -555,11 +558,11 @@ class TestImportExportFields(unittest.TestCase):
                 id = db.person.insert(name=str(k))
                 db.pet.insert(friend=id,name=str(k))
         db.commit()
-        stream = cStringIO.StringIO()
+        stream = StringIO.StringIO()
         db.export_to_csv_file(stream)
         db(db.pet).delete()
         db(db.person).delete()
-        stream = cStringIO.StringIO(stream.getvalue())
+        stream = StringIO.StringIO(stream.getvalue())
         db.import_from_csv_file(stream)
         assert db(db.person.id==db.pet.friend)(db.person.name==db.pet.name).count()==10
         db.pet.drop()
@@ -579,9 +582,9 @@ class TestImportExportUuidFields(unittest.TestCase):
                 id = db.person.insert(name=str(k),uuid=str(k))
                 db.pet.insert(friend=id,name=str(k))
         db.commit()
-        stream = cStringIO.StringIO()
+        stream = StringIO.StringIO()
         db.export_to_csv_file(stream)
-        stream = cStringIO.StringIO(stream.getvalue())
+        stream = StringIO.StringIO(stream.getvalue())
         db.import_from_csv_file(stream)
         assert db(db.person).count()==10
         assert db(db.person.id==db.pet.friend)(db.person.name==db.pet.name).count()==20

gluon/tests/test_languages.py

@@ -57,20 +57,18 @@ try:
     class TestTranslations(unittest.TestCase):
 
         def setUp(self):
-            self.request = Storage()
             if os.path.isdir('gluon'):
-                self.request.folder = 'applications/welcome'
+                self.langpath = 'applications/welcome/languages'
             else:
-                self.request.folder = os.path.realpath(
-                    '../../applications/welcome')
-            self.request.env = Storage()
-            self.request.env.http_accept_language = 'en'
+                self.langpath = os.path.realpath(
+                    '../../applications/welcome/languages')
+            self.http_accept_language = 'en'
 
         def tearDown(self):
             pass
 
         def test_plain(self):
-            T = languages.translator(self.request)
+            T = languages.translator(self.langpath, self.http_accept_language)
             self.assertEqual(str(T('Hello World')),
                              'Hello World')
             self.assertEqual(str(T('Hello World## comment')),

gluon/tests/test_routes.py

@@ -105,13 +105,16 @@ class TestRoutes(unittest.TestCase):
         self.assertEqual(filter_url(
             'http://domain.com/abc/def/ghi/j%20kl'), "/abc/def/ghi ['j_kl']")
         self.assertEqual(filter_url('http://domain.com/welcome/static/path/to/static'), "%s/applications/welcome/static/path/to/static" % root)
+        # no more necessary since explcit check for directory traversal attacks
+        """
         self.assertRaises(HTTP, filter_url, 'http://domain.com/welcome/static/bad/path/to/st~tic')
         try:
             # 2.7+ only
             self.assertRaisesRegexp(HTTP, "400.*BAD REQUEST \[invalid path\]", filter_url, 'http://domain.com/welcome/static/bad/path/to/st~tic')
         except AttributeError:
             pass
-        # outgoing
+        """
+        # outgoing        
         self.assertEqual(filter_url('http://domain.com/init/default/index',
                          out=True), '/init/default/index')
         self.assertEqual(filter_url('http://domain.com/init/default/index/arg1', out=True), '/init/default/index/arg1')

gluon/tests/test_web.py

@@ -40,6 +40,7 @@ class TestWeb(unittest.TestCase):
 
         # check registration and login were successful
         client.get('index')
+        print client.text
         self.assertTrue('Welcome Homer' in client.text)
 
         client = WebClient('http://127.0.0.1:8000/admin/default/')

gluon/tools.py

@@ -769,8 +769,12 @@ class Recaptcha(DIV):
             del self.request_vars.recaptcha_response_field
             self.request_vars.captcha = ''
             return True
-        self.errors['captcha'] = self.error_message
-        return False
+        else:
+            # In case we get an error code, store it so we can get an error message
+            # from the /api/challenge URL as described in the reCAPTCHA api docs.
+            self.error = return_values[1]
+            self.errors['captcha'] = self.error_message
+            return False
 
     def xml(self):
         public_key = self.public_key
@@ -1138,6 +1142,7 @@ class Auth(object):
             login_next = url_index,
             login_onvalidation = [],
             login_onaccept = [],
+            login_onfail = [],
             login_methods = [self],
             login_form = self,
             logout_next = url_index,
@@ -1365,13 +1370,13 @@ class Auth(object):
                                  archive_names='%(tablename)s_archive',
                                  current_record='current_record'):
         """
-        to enable full record vernionioning (including auth tables):
+        to enable full record versioning (including auth tables):
 
         auth = Auth(db)
         auth.define_tables(signature=True)
         # define our own tables
         db.define_table('mything',Field('name'),auth.signature)
-        auth.enable_record_vernining(tables=db)
+        auth.enable_record_versioning(tables=db)
 
         tables can be the db (all table) or a list of tables.
         only tables with modified_by and modified_on fiels (as created
@@ -1386,7 +1391,7 @@ class Auth(object):
 
         Important: If you use auth.enable_record_versioning,
         do not use auth.archive or you will end up with duplicates.
-        auth.archive does explicitely what enable_record_versioning
+        auth.archive does explicitly what enable_record_versioning
         does automatically.
 
         """
@@ -1489,7 +1494,7 @@ class Auth(object):
                      IS_NOT_IN_DB(db, '%s.username' % settings.table_user_name)]
                 if not settings.username_case_sensitive:
                     is_unique_username.insert(1, IS_LOWER())
-                table = db.define_table(
+                db.define_table(
                     settings.table_user_name,
                     Field('first_name', length=128, default='',
                           label=self.messages.label_first_name,
@@ -1522,7 +1527,7 @@ class Auth(object):
                         fake_migrate=fake_migrate,
                         format='%(username)s'))
             else:
-                table = db.define_table(
+                db.define_table(
                     settings.table_user_name,
                     Field('first_name', length=128, default='',
                           label=self.messages.label_first_name,
@@ -1555,7 +1560,7 @@ class Auth(object):
         if not settings.table_group_name in db.tables:
             extra_fields = settings.extra_fields.get(
                 settings.table_group_name, []) + signature_list
-            table = db.define_table(
+            db.define_table(
                 settings.table_group_name,
                 Field('role', length=512, default='',
                       label=self.messages.label_role,
@@ -1573,7 +1578,7 @@ class Auth(object):
         if not settings.table_membership_name in db.tables:
             extra_fields = settings.extra_fields.get(
                 settings.table_membership_name, []) + signature_list
-            table = db.define_table(
+            db.define_table(
                 settings.table_membership_name,
                 Field('user_id', reference_table_user,
                       label=self.messages.label_user_id),
@@ -1587,7 +1592,7 @@ class Auth(object):
         if not settings.table_permission_name in db.tables:
             extra_fields = settings.extra_fields.get(
                 settings.table_permission_name, []) + signature_list
-            table = db.define_table(
+            db.define_table(
                 settings.table_permission_name,
                 Field('group_id', reference_table_group,
                       label=self.messages.label_group_id),
@@ -1605,7 +1610,7 @@ class Auth(object):
                         settings.table_permission_name, migrate),
                     fake_migrate=fake_migrate))
         if not settings.table_event_name in db.tables:
-            table = db.define_table(
+            db.define_table(
                 settings.table_event_name,
                 Field('time_stamp', 'datetime',
                       default=current.request.now,
@@ -1629,7 +1634,7 @@ class Auth(object):
         now = current.request.now
         if settings.cas_domains:
             if not settings.table_cas_name in db.tables:
-                table = db.define_table(
+                db.define_table(
                     settings.table_cas_name,
                     Field('user_id', reference_table_user, default=None,
                           label=self.messages.label_user_id),
@@ -1688,7 +1693,7 @@ class Auth(object):
             user_id = None  # user unknown
         vars = vars or {}
         self.table_event().insert(
-            description=description % vars,
+            description=str(description % vars),
             origin=origin, user_id=user_id)
 
     def get_or_create_user(self, keys, update_fields=['email']):
@@ -1758,11 +1763,17 @@ class Auth(object):
         """
         login the user = db.auth_user(id)
         """
-        user = Storage(self.table_user()._filter_fields(user, id=True))
-        if 'password' in user:
-            del user.password
+        from gluon.settings import global_settings
+        if global_settings.web2py_runtime_gae:
+            user = Row(self.db.auth_user._filter_fields(user, id=True))
+            delattr(user,'password')
+        else:
+            user = Row(user)
+            for key,value in user.items():
+                if callable(value) or key=='password':
+                    delattr(user,key)
         current.session.auth = Storage(
-            user=user,
+            user = user,
             last_visit=current.request.now,
             expiration=self.settings.expiration,
             hmac_key=web2py_uuid())
@@ -1773,8 +1784,6 @@ class Auth(object):
         """
         logins user as specified by usernname (or email) and password
         """
-        request = current.request
-        session = current.session
         table_user = self.table_user()
         if self.settings.login_userfield:
             userfield = self.settings.login_userfield
@@ -1826,14 +1835,15 @@ class Auth(object):
                              created_on=request.now,
                              renew=interactivelogin)
             service = session._cas_service
+            query_sep = '&' if '?' in service else '?'
             del session._cas_service
             if 'warn' in request.vars and not interactivelogin:
                 response.headers[
-                    'refresh'] = "5;URL=%s" % service + "?ticket=" + ticket
+                    'refresh'] = "5;URL=%s" % service + query_sep + "ticket=" + ticket
                 return A("Continue to %s" % service,
-                         _href=service + "?ticket=" + ticket)
+                         _href=service + query_sep + "ticket=" + ticket)
             else:
-                redirect(service + "?ticket=" + ticket)
+                redirect(service + query_sep + "ticket=" + ticket)
         if self.is_logged_in() and not 'renew' in request.vars:
             return allow_access()
         elif not self.is_logged_in() and 'gateway' in request.vars:
@@ -1949,7 +1959,9 @@ class Auth(object):
             onaccept = self.settings.login_onaccept
         if log is DEFAULT:
             log = self.messages.login_log
-
+        
+        onfail = self.settings.login_onfail
+        
         user = None  # default
 
         # do we use our own login form, or from a central source?
@@ -2062,8 +2074,10 @@ class Auth(object):
                                    request.post_vars)
                     # invalid login
                     session.flash = self.messages.invalid_login
+                    callback(onfail, None)
                     redirect(
-                        self.url(args=request.args, vars=request.get_vars))
+                        self.url(args=request.args, vars=request.get_vars),
+                        client_side=True)
 
         else:
             # use a central authentication server
@@ -2079,7 +2093,7 @@ class Auth(object):
             else:
                 # we need to pass through login again before going on
                 next = self.url(self.settings.function, args='login')
-                redirect(cas.login_url(next))
+                redirect(cas.login_url(next), client_side=True)
 
         # process authenticated users
         if user:
@@ -2102,14 +2116,16 @@ class Auth(object):
                 if next == session._auth_next:
                     session._auth_next = None
                 next = replace_id(next, form)
-                redirect(next)
+                redirect(next, client_side=True)
+                
             table_user[username].requires = old_requires
             return form
         elif user:
             callback(onaccept, None)
+
         if next == session._auth_next:
             del session._auth_next
-        redirect(next)
+        redirect(next, client_side=True)
 
     def logout(self, next=DEFAULT, onlogout=DEFAULT, log=DEFAULT):
         """
@@ -2138,7 +2154,8 @@ class Auth(object):
 
         current.session.auth = None
         current.session.flash = self.messages.logged_out
-        redirect(next)
+        if not next is None:
+            redirect(next)
 
     def register(
         self,
@@ -2160,7 +2177,7 @@ class Auth(object):
         response = current.response
         session = current.session
         if self.is_logged_in():
-            redirect(self.settings.logged_url)
+            redirect(self.settings.logged_url, client_side=True)
         if next is DEFAULT:
             next = self.next or self.settings.register_next
         if onvalidation is DEFAULT:
@@ -2276,7 +2293,7 @@ class Auth(object):
                 next = self.url(args=request.args)
             else:
                 next = replace_id(next, form)
-            redirect(next)
+            redirect(next, client_side=True)
         return form
 
     def is_logged_in(self):
@@ -2524,7 +2541,7 @@ class Auth(object):
                 raise Exception
         except Exception:
             session.flash = self.messages.invalid_reset_password
-            redirect(next)
+            redirect(next, client_side=True)
         passfield = self.settings.password_field
         form = SQLFORM.factory(
             Field('new_password', 'password',
@@ -2549,7 +2566,7 @@ class Auth(object):
             session.flash = self.messages.password_changed
             if self.settings.login_after_password_change:
                 self.login_user(user)
-            redirect(next)
+            redirect(next, client_side=True)
         return form
 
     def request_reset_password(
@@ -2607,10 +2624,10 @@ class Auth(object):
             user = table_user(email=form.vars.email)
             if not user:
                 session.flash = self.messages.invalid_email
-                redirect(self.url(args=request.args))
+                redirect(self.url(args=request.args), client_side=True)
             elif user.registration_key in ('pending', 'disabled', 'blocked'):
                 session.flash = self.messages.registration_pending
-                redirect(self.url(args=request.args))
+                redirect(self.url(args=request.args), client_side=True)
             if self.email_reset_password(user):
                 session.flash = self.messages.email_sent
             else:
@@ -2621,7 +2638,7 @@ class Auth(object):
                 next = self.url(args=request.args)
             else:
                 next = replace_id(next, form)
-            redirect(next)
+            redirect(next, client_side=True)
         # old_requires = table_user.email.requires
         return form
 
@@ -2666,10 +2683,9 @@ class Auth(object):
         """
 
         if not self.is_logged_in():
-            redirect(self.settings.login_url)
+            redirect(self.settings.login_url, client_side=True)
         db = self.db
         table_user = self.table_user()
-        usern = self.settings.table_user_name
         s = db(table_user.id == self.user.id)
 
         request = current.request
@@ -2717,7 +2733,7 @@ class Auth(object):
                     next = self.url(args=request.args)
                 else:
                     next = replace_id(next, form)
-            redirect(next)
+                redirect(next, client_side=True)
         return form
 
     def profile(
@@ -2737,7 +2753,7 @@ class Auth(object):
 
         table_user = self.table_user()
         if not self.is_logged_in():
-            redirect(self.settings.login_url)
+            redirect(self.settings.login_url, client_side=True)
         passfield = self.settings.password_field
         table_user[passfield].writable = False
         request = current.request
@@ -2773,7 +2789,7 @@ class Auth(object):
                 next = self.url(args=request.args)
             else:
                 next = replace_id(next, form)
-            redirect(next)
+            redirect(next, client_side=True)
         return form
 
     def is_impersonating(self):
@@ -2883,7 +2899,9 @@ class Auth(object):
                 user = user or self.user
                 if requires_login:
                     if not user:
-                        if not otherwise is None:
+                        if current.request.ajax:
+                            raise HTTP(401)
+                        elif not otherwise is None:
                             if callable(otherwise):
                                 return otherwise()
                             redirect(otherwise)
@@ -2990,10 +3008,12 @@ class Auth(object):
         return self.id_group(self.user_group_role(user_id))
 
     def user_group_role(self, user_id=None):
+        if not self.settings.create_user_groups:
+            return None
         if user_id:
             user = self.table_user()[user_id]
         else:
-            user = self.user
+            user = self.user        
         return self.settings.create_user_groups % user
 
     def has_membership(self, group_id=None, user_id=None, role=None):
@@ -3284,19 +3304,30 @@ class Auth(object):
              manage_permissions=False,
              force_prefix='',
              restrict_search=False,
-             resolve=True):
+             resolve=True,
+             extra=None,
+             menugroups=None):
         if not hasattr(self, '_wiki'):
             self._wiki = Wiki(self, render=render,
                               manage_permissions=manage_permissions,
                               force_prefix=force_prefix,
                               restrict_search=restrict_search,
-                              env=env)
+                              env=env, extra=extra or {},
+                              menugroups=menugroups)
         else:
             self._wiki.env.update(env or {})
         # if resolve is set to True, process request as wiki call
         # resolve=False allows initial setup without wiki redirection
+        wiki = None
         if resolve:
-            return self._wiki.read(slug)['content'] if slug else self._wiki()
+            action = str(current.request.args(0)).startswith("_")
+            if slug and not action:
+                wiki = self._wiki.read(slug)['content']
+            else:
+                wiki = self._wiki()
+            if isinstance(wiki, basestring):
+                wiki = XML(wiki)
+            return wiki
 
 
 class Crud(object):
@@ -3465,10 +3496,12 @@ class Crud(object):
             deletable = self.settings.update_deletable
         if message is DEFAULT:
             message = self.messages.record_updated
+        if not 'hidden' in attributes:
+            attributes['hidden'] = {}
+        attributes['hidden']['_next'] = next
         form = SQLFORM(
             table,
             record,
-            hidden=dict(_next=next),
             showid=self.settings.showid,
             submit_button=self.messages.submit_button,
             delete_label=self.messages.delete_label,
@@ -3476,7 +3509,7 @@ class Crud(object):
             upload=self.settings.download_url,
             formstyle=self.settings.formstyle,
             separator=self.settings.label_separator,
-            **attributes
+            **attributes # contains hidden
             )
         self.accepted = False
         self.deleted = False
@@ -4139,12 +4172,13 @@ class Service(object):
                 return '<NULL>'
             return value
         if args and args[0] in self.run_procedures:
+            import types
             r = universal_caller(self.run_procedures[args[0]],
                                  *args[1:], **dict(request.vars))
             s = cStringIO.StringIO()
             if hasattr(r, 'export_to_csv_file'):
                 r.export_to_csv_file(s)
-            elif r and isinstance(r[0], (dict, Storage)):
+            elif r and not isinstance(r, types.GeneratorType) and isinstance(r[0], (dict, Storage)):
                 import csv
                 writer = csv.writer(s)
                 writer.writerow(r[0].keys())
@@ -4228,7 +4262,7 @@ class Service(object):
         if not method in methods:
             return return_error(id, 100, 'method "%s" does not exist' % method)
         try:
-            s = methods[method](*params)
+            s = methods[method](**params)
             if hasattr(s, 'as_list'):
                 s = s.as_list()
             return return_response(id, s)
@@ -4581,16 +4615,23 @@ class PluginManager(object):
 
 
 class Expose(object):
-    def __init__(self, base=None, basename='base'):
+    def __init__(self, base=None, basename='base', extensions=None, allow_download=True):
+        """
+        extensions: an optional list of file extensions for filtering displayed files:
+        ['.py', '.jpg']
+        allow_download: whether to allow downloading selected files
+        """
         current.session.forget()
         base = base or os.path.join(current.request.folder, 'static')
         self.basename = basename
-        args = self.args = current.request.raw_args and \
-            current.request.raw_args.split('/') or []
-        filename = os.path.join(base, *args)
+        self.args = current.request.raw_args and \
+            [arg for arg in current.request.raw_args.split('/') if arg] or []
+        filename = os.path.join(base, *self.args)
+        if not os.path.exists(filename):
+            raise HTTP(404, "FILE NOT FOUND")
         if not os.path.normpath(filename).startswith(base):
             raise HTTP(401, "NOT AUTHORIZED")
-        if not os.path.isdir(filename):
+        if allow_download and not os.path.isdir(filename):
             current.response.headers['Content-Type'] = contenttype(filename)
             raise HTTP(200, open(filename, 'rb'), **current.response.headers)
         self.path = path = os.path.join(filename, '*')
@@ -4598,23 +4639,24 @@ class Expose(object):
                             if os.path.isdir(f) and not self.isprivate(f)]
         self.filenames = [f[len(path) - 1:] for f in sorted(glob.glob(path))
                             if not os.path.isdir(f) and not self.isprivate(f)]
+        if extensions:
+            self.filenames = [f for f in self.filenames if os.path.splitext(f)[-1] in extensions]
 
     def breadcrumbs(self, basename):
         path = []
         span = SPAN()
         span.append(A(basename, _href=URL()))
-        span.append('/')
-        args = current.request.raw_args and \
-            current.request.raw_args.split('/') or []
-        for arg in args:
+        for arg in self.args:
+            span.append('/')
             path.append(arg)
             span.append(A(arg, _href=URL(args='/'.join(path))))
-            span.append('/')
         return span
 
     def table_folders(self):
-        return TABLE(*[TR(TD(A(folder, _href=URL(args=self.args + [folder]))))
-                           for folder in self.folders])
+        if self.folders:
+            return SPAN(H3('Folders'), TABLE(*[TR(TD(A(folder, _href=URL(args=self.args + [folder]))))
+                           for folder in self.folders]))
+        return ''
 
     @staticmethod
     def isprivate(f):
@@ -4622,21 +4664,21 @@ class Expose(object):
 
     @staticmethod
     def isimage(f):
-        return f.rsplit('.')[-1].lower() in ('png', 'jpg', 'jpeg', 'gif', 'tiff')
+        return os.path.splitext(f)[-1].lower() in ('.png', '.jpg', '.jpeg', '.gif', '.tiff')
 
     def table_files(self, width=160):
-        return TABLE(*[TR(TD(A(f, _href=URL(args=self.args + [f]))),
+        if self.filenames:
+            return SPAN(H3('Files'), TABLE(*[TR(TD(A(f, _href=URL(args=self.args + [f]))),
                           TD(IMG(_src=URL(args=self.args + [f]),
                                  _style='max-width:%spx' % width)
                                  if width and self.isimage(f) else ''))
-                           for f in self.filenames])
+                           for f in self.filenames]))
+        return ''
 
     def xml(self):
         return DIV(
             H2(self.breadcrumbs(self.basename)),
-            H3('Folders'),
             self.table_folders(),
-            H3('Files'),
             self.table_files()).xml()
 
 
@@ -4645,7 +4687,8 @@ class Wiki(object):
     rows_page = 25
 
     def markmin_render(self, page):
-        html = MARKMIN(page.body, url=True, environment=self.env,
+        html = MARKMIN(page.body, extra=self.extra, 
+                       url=True, environment=self.env,
                        autolinks=lambda link: expand_one(link, {})).xml()
         html += DIV(_class='w2p_wiki_tags',
                     *[A(t.strip(), _href=URL(args='_search', vars=dict(q=t)))
@@ -4656,7 +4699,7 @@ class Wiki(object):
         html = page.body
         # @///function -> http://..../function
         html = replace_at_urls(html, URL)
-        # http://...jpg -> <img src="http://...jpg/> or oembed
+        # http://...jpg -> <img src="http://...jpg/> or embed
         html = replace_autolinks(html, lambda link: expand_one(link, {}))
         # @{component:name} -> <script>embed component name</script>
         html = replace_components(html, self.env)
@@ -4674,7 +4717,7 @@ class Wiki(object):
 
     def __init__(self, auth, env=None, render='markmin',
                  manage_permissions=False, force_prefix='',
-                 restrict_search=False):
+                 restrict_search=False, extra=None, menugroups=None):
         self.env = env or {}
         self.env['component'] = Wiki.component
         if render == 'markmin':
@@ -4683,6 +4726,7 @@ class Wiki(object):
             render = self.html_render
         self.render = render
         self.auth = auth
+        self.menugroups = menugroups
         if self.auth.user:
             self.force_prefix = force_prefix % self.auth.user
         else:
@@ -4690,6 +4734,7 @@ class Wiki(object):
         self.host = current.request.env.http_host
         perms = self.manage_permissions = manage_permissions
         self.restrict_search = restrict_search
+        self.extra = extra or {}
         db = auth.db
         table_definitions = [
             ('wiki_page', {
@@ -4797,6 +4842,16 @@ class Wiki(object):
 
     def can_search(self):
         return True
+
+    def can_see_menu(self):
+        if self.menugroups is None:
+            return True
+        if self.auth.user:
+            groups = self.auth.user_groups.values()
+            if any(t in self.menugroups for t in groups):
+                return True
+        return False
+
     ### END POLICY
 
     def __call__(self):
@@ -4809,7 +4864,7 @@ class Wiki(object):
         elif not zero or not zero.startswith('_'):
             return self.read(zero)
         elif zero == '_edit':
-            return self.edit(request.args(1) or 'index')
+            return self.edit(request.args(1) or 'index',request.args(2) or 0)
         elif zero == '_editmedia':
             return self.editmedia(request.args(1) or 'index')
         elif zero == '_create':
@@ -4884,7 +4939,7 @@ class Wiki(object):
             raise HTTP(401, "Not Authorized")
         return True
 
-    def edit(self, slug):
+    def edit(self,slug,from_template=0):
         auth = self.auth
         db = auth.db
         page = db.wiki_page(slug=slug)
@@ -4905,7 +4960,7 @@ class Wiki(object):
                 db.wiki_page.body.default = \
                     '- Menu Item > @////index\n- - Submenu > http://web2py.com'
             else:
-                db.wiki_page.body.default = '## %s\n\npage content' % title_guess
+                db.wiki_page.body.default = db(db.wiki_page.id==from_template).select(db.wiki_page.body)[0].body if int(from_template) > 0 else '## %s\n\npage content' % title_guess
         vars = current.request.post_vars
         if vars.body:
             vars.body = vars.body.replace('://%s' % self.host, '://HOSTNAME')
@@ -4924,12 +4979,20 @@ class Wiki(object):
             pagecontent.css('font-family',
                             'Monaco,Menlo,Consolas,"Courier New",monospace');
             var prevbutton = $('<button class="btn nopreview">Preview</button>');
+            var mediabutton = $('<button class="btn nopreview">Media</button>');
             var preview = $('<div id="preview"></div>').hide();
+            var previewmedia = $('<div id="previewmedia"></div>');
             var table = $('form');
-            prevbutton.insertBefore(table);
             preview.insertBefore(table);
-            prevbutton.on('click', function(e) {
-                e.preventDefault();
+            prevbutton.insertBefore(table);
+            mediabutton.insertBefore(table);
+            previewmedia.insertBefore(table);
+            mediabutton.toggle(function() {
+                web2py_component('%(urlmedia)s', 'previewmedia');
+            }, function() {
+                previewmedia.empty();
+            });
+            prevbutton.click(function() {
                 if (prevbutton.hasClass('nopreview')) {
                     prevbutton.addClass('preview').removeClass(
                         'nopreview').html('Edit Source');
@@ -4942,7 +5005,7 @@ class Wiki(object):
                 }
             })
         })
-        """ % dict(url=URL(args=('_preview')))
+        """ % dict(url=URL(args=('_preview')), urlmedia=URL(extension='load',args=('_editmedia'),vars=dict(embedded=1)))
         return dict(content=TAG[''](form, SCRIPT(script)))
 
     def editmedia(self, slug):
@@ -4958,9 +5021,21 @@ class Wiki(object):
                       row.filename.split('.')[-1]))
         self.auth.db.wiki_media.wiki_page.default = page.id
         self.auth.db.wiki_media.wiki_page.writable = False
+        links = []
+        csv = True
+        if current.request.vars.embedded:
+            script = "var c = $('#wiki_page_body'); c.val(c.val() + $('%s').text()); return false;"
+            fragment = self.auth.db.wiki_media.id.represent
+            csv = False
+            links=[
+                lambda row:
+                    A('copy into source', _href='#', _onclick=script % (fragment(row.id, row)))
+                    ]
         content = SQLFORM.grid(
             self.auth.db.wiki_media.wiki_page == page.id,
             orderby=self.auth.db.wiki_media.title,
+            links = links,
+            csv = csv,
             args=['_editmedia', slug],
             user_signature=False)
         return dict(content=content)
@@ -4969,13 +5044,21 @@ class Wiki(object):
         if not self.can_edit():
             return self.not_authorized()
         db = self.auth.db
-        form = FORM(INPUT(_name='slug', value=current.request.args(1),
-                          requires=(IS_SLUG(),
-                                    IS_NOT_IN_DB(db, db.wiki_page.slug))),
-                    INPUT(_type='submit',
-                          _value=current.T('Create Page from Slug')))
+        slugs=db(db.wiki_page.id>0).select(db.wiki_page.id,db.wiki_page.slug)
+        options=[OPTION(row.slug,_value=row.id) for row in slugs] 
+        options.insert(0, OPTION('',_value=''))
+        form = SQLFORM.factory(Field("slug", default=current.request.args(1),
+                                     requires=(IS_SLUG(),
+                                     IS_NOT_IN_DB(db,db.wiki_page.slug))),
+                               Field("from_template", "reference wiki_page",
+                                     requires=IS_EMPTY_OR(IS_IN_DB(db, db.wiki_page, '%(slug)s')),
+                                     comment=current.T("Choose Template or empty for new Page")),
+                                     _class="well span6")
+        form.element("[type=submit]").attributes["_value"] = current.T("Create Page from Slug")
+
         if form.process().accepted:
-            redirect(URL(args=('_edit', form.vars.slug)))
+             # form.vars.from_template = 0 if not form.vars.from_template else form.vars.from_template
+             redirect(URL(args=('_edit',form.vars.slug,form.vars.from_template or 0))) # added param
         return dict(content=form)
 
     def pages(self):
@@ -5032,7 +5115,7 @@ class Wiki(object):
                 subtree = []
                 tree[base] = subtree
                 parent.append((current.T(title), False, link, subtree))
-        if True:
+        if self.can_see_menu():
             submenu = []
             menu.append((current.T('[Wiki]'), None, None, submenu))
             if URL() == URL(controller, function):
@@ -5059,12 +5142,14 @@ class Wiki(object):
 
             submenu.append((current.T('Create New Page'), None,
                             URL(controller, function, args=('_create'))))
-        if self.can_manage():
-            submenu.append((current.T('Manage Pages'), None,
+            # Moved next if to inside self.auth.user check
+            if self.can_manage():
+                submenu.append((current.T('Manage Pages'), None,
                             URL(controller, function, args=('_pages'))))
-            submenu.append((current.T('Edit Menu'), None,
+                submenu.append((current.T('Edit Menu'), None,
                             URL(controller, function, args=('_edit', 'wiki-menu'))))
-        submenu.append((current.T('Search Pages'), None,
+            # Also moved inside self.auth.user check
+            submenu.append((current.T('Search Pages'), None,
                         URL(controller, function, args=('_search'))))
         return menu
 

gluon/utils.py

@@ -9,7 +9,6 @@ License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
 This file specifically includes utilities for security.
 """
 
-import string
 import threading
 import struct
 import hashlib
@@ -19,22 +18,34 @@ import random
 import time
 import os
 import re
+import sys
 import logging
 import socket
-import cPickle
 import base64
 import zlib
 
+python_version = sys.version_info[0]
+
+if python_version == 2:
+    import cPickle as pickle
+else:
+    import pickle
+
+
 try:
     from Crypto.Cipher import AES
 except ImportError:
-    from contrib import aes as AES
+    import contrib.aes as AES
 
 try:
     from contrib.pbkdf2 import pbkdf2_hex
     HAVE_PBKDF2 = True
 except ImportError:
-    HAVE_PBKDF2 = False
+    try:
+        from .pbkdf2 import pbkdf2_hex
+        HAVE_PBKDF2 = True
+    except (ImportError, ValueError):
+        HAVE_PBKDF2 = False
 
 logger = logging.getLogger("web2py")
 
@@ -115,7 +126,7 @@ def pad(s, n=32, padchar='.'):
 def secure_dumps(data, encryption_key, hash_key=None, compression_level=None):
     if not hash_key:
         hash_key = hashlib.sha1(encryption_key).hexdigest()
-    dump = cPickle.dumps(data)
+    dump = pickle.dumps(data)
     if compression_level:
         dump = zlib.compress(dump, compression_level)
     key = pad(encryption_key[:32])
@@ -141,8 +152,8 @@ def secure_loads(data, encryption_key, hash_key=None, compression_level=None):
         data = data.rstrip(' ')
         if compression_level:
             data = zlib.decompress(data)
-        return cPickle.loads(data)
-    except (TypeError, cPickle.UnpicklingError):
+        return pickle.loads(data)
+    except (TypeError, pickle.UnpicklingError):
         return None
 
 ### compute constant CTOKENS
@@ -173,7 +184,10 @@ def initialize_urandom():
             # try to add process-specific entropy
             frandom = open('/dev/urandom', 'wb')
             try:
-                frandom.write(''.join(chr(t) for t in ctokens))
+                if python_version == 2:
+                    frandom.write(''.join(chr(t) for t in ctokens)) # python 2
+                else:
+                    frandom.write(bytes([]).join(bytes([t]) for t in ctokens)) # python 3
             finally:
                 frandom.close()
         except IOError:
@@ -185,8 +199,11 @@ def initialize_urandom():
             """Cryptographically secure session management is not possible on your system because
 your system does not provide a cryptographically secure entropy source.
 This is not specific to web2py; consider deploying on a different operating system.""")
-    unpacked_ctokens = struct.unpack('=QQ', string.join(
-        (chr(x) for x in ctokens), ''))
+    if python_version == 2:
+        packed = ''.join(chr(x) for x in ctokens) # python 2
+    else:
+        packed = bytes([]).join(bytes([x]) for x in ctokens) # python 3
+    unpacked_ctokens = struct.unpack('=QQ', packed)
     return unpacked_ctokens, have_urandom
 UNPACKED_CTOKENS, HAVE_URANDOM = initialize_urandom()
 
@@ -243,14 +260,14 @@ def is_valid_ip_address(address):
     # deal with special cases
     if address.lower() in ('127.0.0.1', 'localhost', '::1', '::ffff:127.0.0.1'):
         return True
-    elif address.lower() in ('unkown', ''):
+    elif address.lower() in ('unknown', ''):
         return False
     elif address.count('.') == 3:  # assume IPv4
         if address.startswith('::ffff:'):
             address = address[7:]
         if hasattr(socket, 'inet_aton'):  # try validate using the OS
             try:
-                addr = socket.inet_aton(address)
+                socket.inet_aton(address)
                 return True
             except socket.error:  # invalid address
                 return False
@@ -261,9 +278,22 @@ def is_valid_ip_address(address):
             return False
     elif hasattr(socket, 'inet_pton'):  # assume IPv6, try using the OS
         try:
-            addr = socket.inet_pton(socket.AF_INET6, address)
+            socket.inet_pton(socket.AF_INET6, address)
             return True
         except socket.error:  # invalid address
             return False
     else:  # do not know what to do? assume it is a valid address
         return True
+
+
+def is_loopback_ip_address(ip):
+    """Determines whether the IP address appears to be a loopback address.
+
+    This assumes that the IP is valid.  The IPv6 check is limited to '::1'.
+
+    """
+    if not ip:
+        return False
+    if ip.count('.') == 3:  # IPv4
+        return ip.startswith('127') or ip.startswith('::ffff:127')
+    return ip == '::1'  # IPv6

gluon/validators.py

@@ -575,7 +575,10 @@ class IS_NOT_IN_DB(Validator):
         self.record_id = id
 
     def __call__(self, value):
-        value = str(value)
+        if isinstance(value,unicode):
+            value = value.encode('utf8')
+        else:
+            value = str(value)
         if not value.strip():
             return (value, translate(self.error_message))
         if value in self.allowed_override:
@@ -778,6 +781,8 @@ class IS_FLOAT_IN_RANGE(Validator):
         return (value, self.error_message)
 
     def formatter(self, value):
+        if value is None:
+            return None
         return str2dec(value).replace('.', self.dot)
 
 
@@ -882,6 +887,8 @@ class IS_DECIMAL_IN_RANGE(Validator):
         return (value, self.error_message)
 
     def formatter(self, value):
+        if value is None:
+            return None
         return str2dec(value).replace('.', self.dot)
 
 
@@ -2170,6 +2177,8 @@ class IS_DATE(Validator):
             return (value, translate(self.error_message) % self.extremes)
 
     def formatter(self, value):
+        if value is None:
+            return None
         format = self.format
         year = value.year
         y = '%.4i' % year
@@ -2228,6 +2237,8 @@ class IS_DATETIME(Validator):
             return (value, translate(self.error_message) % self.extremes)
 
     def formatter(self, value):
+        if value is None:
+            return None
         format = self.format
         year = value.year
         y = '%.4i' % year
@@ -2622,6 +2633,13 @@ class LazyCrypt(object):
         """
         compares the current lazy crypted password with a stored password
         """
+
+        # LazyCrypt objects comparison
+        if isinstance(stored_password, self.__class__):
+            return ((self is stored_password) or
+                   ((self.crypt.key == stored_password.crypt.key) and
+                   (self.password == stored_password.password)))
+
         if self.crypt.key:
             if ':' in self.crypt.key:
                 key = self.crypt.key.split(':')[1]
@@ -2847,6 +2865,8 @@ class IS_STRONG(object):
 
     def __call__(self, value):
         failures = []
+        if value and len(value) == value.count('*') > 4:
+            return (value, None)
         if self.entropy is not None:
             entropy = calc_entropy(value)
             if entropy < self.entropy:

gluon/widget.py

@@ -28,12 +28,13 @@ import main
 from fileutils import w2p_pack, read_file, write_file
 from settings import global_settings
 from shell import run, test
+from utils import is_valid_ip_address, is_loopback_ip_address
 
 try:
     import Tkinter
     import tkMessageBox
     import contrib.taskbar_widget
-    from winservice import web2py_windows_service_handler
+    from winservice import register_service_handler, Web2pyService
     have_winservice = True
 except:
     have_winservice = False
@@ -99,9 +100,27 @@ class IO(object):
             self.buffer.write(data)
 
 
-def try_start_browser(url):
-    """ Try to start the default browser """
+def get_url(host, path='/', proto='http', port=80):
+    if ':' in host:
+        host = '[%s]' % host
+    else:
+        host = host.replace('0.0.0.0', '127.0.0.1')
+    if path.startswith('/'):
+        path = path[1:]
+    if proto.endswith(':'):
+        proto = proto[:-1]
+    if not port or port == 80:
+        port = ''
+    else:
+        port = ':%s' % port
+    return '%s://%s%s/%s' % (proto, host, port, path)
 
+
+def start_browser(url, startup=False):
+    if startup:
+        print 'please visit:'
+        print '\t', url
+        print 'starting browser...'
     try:
         import webbrowser
         webbrowser.open(url)
@@ -109,15 +128,6 @@ def try_start_browser(url):
         print 'warning: unable to detect your browser'
 
 
-def start_browser(proto, ip, port):
-    """ Starts the default browser """
-    print 'please visit:'
-    url = '%s://%s:%s' % (proto, ip, port)
-    print '\t', url
-    print 'starting browser...'
-    try_start_browser(url)
-
-
 def presentation(root):
     """ Draw the splash screen """
 
@@ -185,7 +195,7 @@ class web2pyDialog(object):
         httplog = os.path.join(self.options.folder, 'httpserver.log')
 
         # Building the Menu
-        item = lambda: try_start_browser(httplog)
+        item = lambda: start_browser(httplog)
         servermenu.add_command(label='View httpserver.log',
                                command=item)
 
@@ -206,7 +216,7 @@ class web2pyDialog(object):
         helpmenu = Tkinter.Menu(self.menu, tearoff=0)
 
         # Home Page
-        item = lambda: try_start_browser('http://www.web2py.com')
+        item = lambda: start_browser('http://www.web2py.com/')
         helpmenu.add_command(label='Home Page',
                              command=item)
 
@@ -236,7 +246,8 @@ class web2pyDialog(object):
         self.ips = {}
         self.selected_ip = Tkinter.StringVar()
         row = 0
-        ips = [('127.0.0.1', 'Local')] + \
+        ips = [('127.0.0.1', 'Local (IPv4)')] + \
+            ([('::1', 'Local (IPv6)')] if socket.has_ipv6 else []) + \
             [(ip, 'Public') for ip in options.ips] + \
             [('0.0.0.0', 'Public')]
         for ip, legend in ips:
@@ -405,10 +416,9 @@ class web2pyDialog(object):
                           if os.path.exists('applications/%s/__init__.py' % arq)]
         self.pagesmenu.delete(0, len(available_apps))
         for arq in available_apps:
-            url = self.url + '/' + arq
-            start_browser = lambda u = url: try_start_browser(u)
+            url = self.url + arq
             self.pagesmenu.add_command(label=url,
-                                       command=start_browser)
+                                       command=lambda u=url: start_browser(u))
 
     def quit(self, justHide=False):
         """ Finish the program execution """
@@ -448,8 +458,7 @@ class web2pyDialog(object):
 
         ip = self.selected_ip.get()
 
-        regexp = '\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'
-        if ip and not re.compile(regexp).match(ip):
+        if not is_valid_ip_address(ip):
             return self.error('invalid host ip address')
 
         try:
@@ -463,7 +472,7 @@ class web2pyDialog(object):
         else:
             proto = 'http'
 
-        self.url = '%s://%s:%s' % (proto, ip, port)
+        self.url = get_url(ip, proto=proto, port=port)
         self.connect_pages()
         self.button_start.configure(state='disabled')
 
@@ -500,7 +509,9 @@ class web2pyDialog(object):
         self.button_stop.configure(state='normal')
 
         if not options.taskbar:
-            thread.start_new_thread(start_browser, (proto, ip, port))
+            thread.start_new_thread(start_browser,
+                                    (get_url(ip, proto=proto, port=port),),
+                                    dict(startup=True))
 
         self.password.configure(state='readonly')
         [ip.configure(state='disabled') for ip in self.ips.values()]
@@ -583,11 +594,13 @@ def console():
 
     parser.description = description
 
+    msg = ('IP address of the server (e.g., 127.0.0.1 or ::1); '
+           'Note: This value is ignored when using the \'interfaces\' option.')
     parser.add_option('-i',
                       '--ip',
                       default='127.0.0.1',
                       dest='ip',
-                      help='ip address of the server (127.0.0.1)')
+                      help=msg)
 
     parser.add_option('-p',
                       '--port',
@@ -596,8 +609,8 @@ def console():
                       type='int',
                       help='port of server (8000)')
 
-    msg = 'password to be used for administration'
-    msg += ' (use -a "<recycle>" to reuse the last password))'
+    msg = ('password to be used for administration '
+           '(use -a "<recycle>" to reuse the last password))')
     parser.add_option('-a',
                       '--password',
                       default='<ask>',
@@ -616,11 +629,13 @@ def console():
                       dest='ssl_private_key',
                       help='file that contains ssl private key')
 
+    msg = ('Use this file containing the CA certificate to validate X509 '
+           'certificates from clients')
     parser.add_option('--ca-cert',
                       action='store',
                       dest='ssl_ca_certificate',
                       default=None,
-                      help='Use this file containing the CA certificate to validate X509 certificates from clients')
+                      help=msg)
 
     parser.add_option('-d',
                       '--pid_filename',
@@ -707,8 +722,8 @@ def console():
                       default=False,
                       help='disable all output')
 
-    msg = 'set debug output level (0-100, 0 means all, 100 means none;'
-    msg += ' default is 30)'
+    msg = ('set debug output level (0-100, 0 means all, 100 means none; '
+           'default is 30)')
     parser.add_option('-D',
                       '--debug',
                       dest='debuglevel',
@@ -716,18 +731,18 @@ def console():
                       type='int',
                       help=msg)
 
-    msg = 'run web2py in interactive shell or IPython (if installed) with'
-    msg += ' specified appname (if app does not exist it will be created).'
-    msg += ' APPNAME like a/c/f (c,f optional)'
+    msg = ('run web2py in interactive shell or IPython (if installed) with '
+           'specified appname (if app does not exist it will be created). '
+           'APPNAME like a/c/f (c,f optional)')
     parser.add_option('-S',
                       '--shell',
                       dest='shell',
                       metavar='APPNAME',
                       help=msg)
 
-    msg = 'run web2py in interactive shell or bpython (if installed) with'
-    msg += ' specified appname (if app does not exist it will be created).'
-    msg += '\n Use combined with --shell'
+    msg = ('run web2py in interactive shell or bpython (if installed) with '
+           'specified appname (if app does not exist it will be created).\n'
+           'Use combined with --shell')
     parser.add_option('-B',
                       '--bpython',
                       action='store_true',
@@ -743,8 +758,8 @@ def console():
                       dest='plain',
                       help=msg)
 
-    msg = 'auto import model files; default is False; should be used'
-    msg += ' with --shell option'
+    msg = ('auto import model files; default is False; should be used '
+           'with --shell option')
     parser.add_option('-M',
                       '--import_models',
                       action='store_true',
@@ -752,8 +767,8 @@ def console():
                       dest='import_models',
                       help=msg)
 
-    msg = 'run PYTHON_FILE in web2py environment;'
-    msg += ' should be used with --shell option'
+    msg = ('run PYTHON_FILE in web2py environment; '
+           'should be used with --shell option')
     parser.add_option('-R',
                       '--run',
                       dest='run',
@@ -761,11 +776,11 @@ def console():
                       default='',
                       help=msg)
 
-    msg = 'run scheduled tasks for the specified apps: expects a list of '
-    msg += 'app names as -K app1,app2,app3 '
-    msg += 'or a list of app:groups as -K app1:group1:group2,app2:group1 '
-    msg += 'to override specific group_names. (only strings, no spaces '
-    msg += 'allowed. Requires a scheduler defined in the models'
+    msg = ('run scheduled tasks for the specified apps: expects a list of '
+           'app names as -K app1,app2,app3 '
+           'or a list of app:groups as -K app1:group1:group2,app2:group1 '
+           'to override specific group_names. (only strings, no spaces '
+           'allowed. Requires a scheduler defined in the models')
     parser.add_option('-K',
                       '--scheduler',
                       dest='scheduler',
@@ -780,8 +795,8 @@ def console():
                       dest='with_scheduler',
                       help=msg)
 
-    msg = 'run doctests in web2py environment; ' +\
-        'TEST_PATH like a/c/f (c,f optional)'
+    msg = ('run doctests in web2py environment; '
+           'TEST_PATH like a/c/f (c,f optional)')
     parser.add_option('-T',
                       '--test',
                       dest='test',
@@ -850,12 +865,14 @@ def console():
                       dest='nogui',
                       help='text-only, no GUI')
 
+    msg = ('should be followed by a list of arguments to be passed to script, '
+           'to be used with -S, -A must be the last option')
     parser.add_option('-A',
                       '--args',
                       action='store',
                       dest='args',
                       default=None,
-                      help='should be followed by a list of arguments to be passed to script, to be used with -S, -A must be the last option')
+                      help=msg)
 
     parser.add_option('--no-banner',
                       action='store_true',
@@ -863,7 +880,10 @@ def console():
                       dest='nobanner',
                       help='Do not print header banner')
 
-    msg = 'listen on multiple addresses: "ip:port:cert:key:ca_cert;ip2:port2:cert2:key2:ca_cert2;..." (:cert:key optional; no spaces)'
+    msg = ('listen on multiple addresses: '
+           '"ip1:port1:key1:cert1:ca_cert1;ip2:port2:key2:cert2:ca_cert2;..." '
+           '(:key:cert:ca_cert optional; no spaces; IPv6 addresses must be in '
+           'square [] brackets)')
     parser.add_option('--interfaces',
                       action='store',
                       dest='interfaces',
@@ -890,9 +910,9 @@ def console():
     global_settings.cmd_args = args
 
     try:
-        options.ips = [
-            ip for ip in socket.gethostbyname_ex(socket.getfqdn())[2]
-            if ip != '127.0.0.1']
+        options.ips = list(set([
+            ip[4][0] for ip in socket.getaddrinfo(socket.getfqdn(), 0)
+            if not is_loopback_ip_address(ip[4][0])]))
     except socket.gaierror:
         options.ips = []
 
@@ -911,7 +931,6 @@ def console():
 
     if options.cronjob:
         global_settings.cronjob = True  # tell the world
-        options.run = False   # don't start cron jobs
         options.plain = True    # cronjobs use a plain shell
         options.nobanner = True
         options.nogui = True
@@ -919,15 +938,22 @@ def console():
     options.folder = os.path.abspath(options.folder)
 
     #  accept --interfaces in the form
-    #  "ip:port:cert:key;ip2:port2;ip3:port3:cert3:key3"
-    #  (no spaces; optional cert:key indicate SSL)
+    #  "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3"
+    #  (no spaces; optional key:cert indicate SSL)
     if isinstance(options.interfaces, str):
-        options.interfaces = [
-            interface.split(':') for interface in options.interfaces.split(';')]
-        for interface in options.interfaces:
-            interface[1] = int(interface[1])    # numeric port
-        options.interfaces = [
-            tuple(interface) for interface in options.interfaces]
+        interfaces = options.interfaces.split(';')
+        options.interfaces = []
+        for interface in interfaces:
+            if interface.startswith('['):  # IPv6
+                ip, if_remainder = interface.split(']', 1)
+                ip = ip[1:]
+                if_remainder = if_remainder[1:].split(':')
+                if_remainder[0] = int(if_remainder[0])  # numeric port
+                options.interfaces.append(tuple([ip] + if_remainder))
+            else:  # IPv4
+                interface = interface.split(':')
+                interface[1] = int(interface[1])  # numeric port
+                options.interfaces.append(tuple(interface))
 
     #  accepts --scheduler in the form
     #  "app:group1,group2,app2:group1"
@@ -948,14 +974,6 @@ def console():
         if not os.path.exists('applications/__init__.py'):
             write_file('applications/__init__.py', '')
 
-        if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'):
-            try:
-                w2p_pack('welcome.w2p', 'applications/welcome')
-                os.unlink('NEWINSTALL')
-            except:
-                msg = "New installation: unable to create welcome.w2p file"
-                sys.stderr.write(msg)
-
     return (options, args)
 
 
@@ -1007,6 +1025,8 @@ def start_schedulers(options):
         processes.append(p)
         print "Currently running %s scheduler processes" % (len(processes))
         p.start()
+        ##to avoid bashing the db at the same time
+        time.sleep(0.7)
         print "Processes started"
     for p in processes:
         try:
@@ -1092,6 +1112,22 @@ def start(cron=True):
             pass
         return
 
+    # ## if -W install/start/stop web2py as service
+    if options.winservice:
+        if os.name == 'nt':
+            if have_winservice:
+                register_service_handler(
+                    argv=['', options.winservice],
+                    opt_file=options.config,
+                    cls=Web2pyService)
+            else:
+                print 'Error: Missing python module winservice'
+                sys.exit(1)
+        else:
+            print 'Error: Windows services not supported on this platform'
+            sys.exit(1)
+        return
+
     # ## if -H cron is enabled in this *process*
     # ## if --softcron use softcron
     # ## use hardcron in all other cases
@@ -1103,20 +1139,6 @@ def start(cron=True):
         global_settings.web2py_crontype = 'hard'
         newcron.hardcron(options.folder).start()
 
-    # ## if -W install/start/stop web2py as service
-    if options.winservice:
-        if os.name == 'nt':
-            if have_winservice:
-                web2py_windows_service_handler(['', options.winservice],
-                                               options.config)
-            else:
-                print 'Error: Missing python module winservice'
-                sys.exit(1)
-        else:
-            print 'Error: Windows services not supported on this platform'
-            sys.exit(1)
-        return
-
     # ## if no password provided and havetk start Tk interface
     # ## or start interface if we want to put in taskbar (system tray)
 
@@ -1186,14 +1208,21 @@ end tell
 
     # ## start server
 
-    (ip, port) = (options.ip, int(options.port))
+    # Use first interface IP and port if interfaces specified, since the
+    # interfaces option overrides the IP (and related) options.
+    if not options.interfaces:
+        (ip, port) = (options.ip, int(options.port))
+    else:
+        first_if = options.interfaces[0]
+        (ip, port) = first_if[0], first_if[1]
 
     # Check for non default value for ssl inputs
     if (len(options.ssl_certificate) > 0) or (len(options.ssl_private_key) > 0):
         proto = 'https'
     else:
         proto = 'http'
-    url = '%s://%s:%s' % (proto, ip, port)
+
+    url = get_url(ip, proto=proto, port=port)
 
     if not options.nobanner:
         print 'please visit:'

gluon/winservice.py

@@ -28,6 +28,7 @@ import servicemanager
 import _winreg
 from fileutils import up
 
+
 __all__ = ['web2py_windows_service_handler']
 
 
@@ -152,20 +153,59 @@ class Web2pyService(Service):
         time.sleep(1)
 
 
-def web2py_windows_service_handler(argv=None, opt_file='options'):
+class Web2pyCronService(Web2pyService):
+
+    _svc_name_ = 'web2py_cron'
+    _svc_display_name_ = 'web2py Cron Service'
+    _exe_args_ = 'options'
+
+    def start(self):
+        import newcron
+        import global_settings
+        self.log('web2py server starting')
+        if not self.chdir():
+            return
+        if len(sys.argv) == 2:
+            opt_mod = sys.argv[1]
+        else:
+            opt_mod = self._exe_args_
+        options = __import__(opt_mod, [], [], '')
+        global_settings.global_settings.web2py_crontype = 'external'
+        if options.scheduler:   # -K                         
+            apps = [app.strip() for app in options.scheduler.split(
+                    ',') if check_existent_app(options, app.strip())]
+        else:
+            apps = None
+        self.extcron = newcron.extcron(options.folder, apps=apps)
+        try:
+            self.extcron.start()
+        except:
+            # self.server.stop()
+            self.extcron = None
+            raise
+
+    def stop(self):
+        self.log('web2py cron stopping')
+        if not self.chdir():
+            return
+        if self.extcron:
+            self.extcron.join()
+
+def register_service_handler(argv=None, opt_file='options', cls=Web2pyService):
     path = os.path.dirname(__file__)
     web2py_path = up(path)
     if web2py_path.endswith('.zip'):  # in case bianry distro 'library.zip'
         web2py_path = os.path.dirname(web2py_path)
     os.chdir(web2py_path)
     classstring = os.path.normpath(
-        os.path.join(web2py_path, 'gluon.winservice.Web2pyService'))
+        os.path.join(web2py_path, 'gluon.winservice.'+cls.__name__))
     if opt_file:
-        Web2pyService._exe_args_ = opt_file
-        win32serviceutil.HandleCommandLine(Web2pyService,
-                                           serviceClassString=classstring, argv=['', 'install'])
-    win32serviceutil.HandleCommandLine(Web2pyService,
-                                       serviceClassString=classstring, argv=argv)
+        cls._exe_args_ = opt_file
+        win32serviceutil.HandleCommandLine(
+            cls, serviceClassString=classstring, argv=['', 'install'])
+    win32serviceutil.HandleCommandLine(
+        cls, serviceClassString=classstring, argv=argv)
 
 if __name__ == '__main__':
-    web2py_windows_service_handler()
+    register_service_handler(cls=Web2pyService)
+    register_service_handler(cls=Web2pyCronService)

router.example.py

@@ -89,8 +89,8 @@
 #         domains = None,
 #         map_hyphen = False,
 #         acfe_match = r'\w+$',                 # legal app/ctlr/fcn/ext
-#         file_match = r'([-+=@$%\w]+[./]?)+$', # legal static subpath
-#         args_match = r'([\w@ -]+[=.]?)+$',    # legal arg in args
+#         file_match = r'([-+=@$%\w]|(?<=[-+=@$%\w])[./])*$',   # legal static subpath
+#         args_match = r'([\w@ -]|(?<=[\w@ -])[.=])*$',         # legal arg in args
 #     )
 #
 #  See rewrite.map_url_in() and rewrite.map_url_out() for implementation details.

scripts/extract_sqlite_models.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+'''
+Create the web2py model code needed to access your sqlite legacy db.
+
+Usage:
+python extract_sqlite_models.py
+
+Access your tables with:
+legacy_db(legacy_db.mytable.id>0).select()
+
+
+extract_sqlite_models.py -- Copyright (C) Michele Comitini
+This code is distributed with web2py.
+
+The regexp code and the dictionary type map was extended from
+extact_mysql_models.py that comes with web2py. extact_mysql_models.py is Copyright (C) Falko Krause.
+
+'''
+import re
+import sys
+import sqlite3
+
+data_type_map = dict(
+    varchar='string',
+    int='integer',
+    integer='integer',
+    tinyint='integer',
+    smallint='integer',
+    mediumint='integer',
+    bigint='integer',
+    float='double',
+    double='double',
+    char='string',
+    decimal='integer',
+    date='date',
+    time='time',
+    timestamp='datetime',
+    datetime='datetime',
+    binary='blob',
+    blob='blob',
+    tinyblob='blob',
+    mediumblob='blob',
+    longblob='blob',
+    text='text',
+    tinytext='text',
+    mediumtext='text',
+    longtext='text',
+    bit='boolean',
+    nvarchar='text',
+    numeric='decimal(30,15)',
+    real='decimal(30,15)',
+)
+
+def get_foreign_keys(sql_lines):
+    fks = dict()
+    for line in sql_lines[1:-1]:
+        hit = re.search(r'FOREIGN\s+KEY\s+\("(\S+)"\)\s+REFERENCES\s+"(\S+)"\s+\("(\S+)"\)', line)
+        if hit:
+            fks[hit.group(1)] =  hit.groups()[1:]
+
+    return fks
+
+def sqlite(database_name):
+    conn = sqlite3.connect(database_name)
+    c = conn.cursor()
+    r = c.execute(r"select name,sql from sqlite_master where type='table' and not name like '\_%' and not lower(name) like 'sqlite_%'")
+    tables = r.fetchall()
+    connection_string = "legacy_db = DAL('sqlite://%s')" % database_name.split('/')[-1]
+    legacy_db_table_web2py_code = []
+    for table_name, sql_create_stmnt in tables:
+        if table_name.startswith('_'):
+            continue
+        if 'CREATE' in sql_create_stmnt:  # check if the table exists
+            #remove garbage lines from sql statement
+            sql_lines = sql_create_stmnt.split('\n')
+            sql_lines = [x for x in sql_lines if not(
+                x.startswith('--') or x.startswith('/*') or x == '')]
+            #generate the web2py code from the create statement
+            web2py_table_code = ''
+            fields = []
+            fks = get_foreign_keys(sql_lines)
+            for line in sql_lines[1:-1]:
+                if re.search('KEY', line) or re.search('PRIMARY', line) or re.search('"ID"', line) or line.startswith(')'):
+                    continue
+                hit = re.search(r'"(\S+)"\s+(\w+(\(\S+\))?),?( .*)?', line)
+                if hit is not None:
+                    name, d_type = hit.group(1), hit.group(2)
+                    d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)
+                    name = unicode(re.sub('`', '', name))
+                    if name in fks.keys():
+                        if fks[name][1].lower() == 'id':
+                            field_type = 'reference %s' % (fks[name][0])
+                        else:
+                            field_type = 'reference %s.%s' % (fks[name][0], fks[name][1])
+                    else:
+                        field_type = data_type_map[d_type]
+                    web2py_table_code += "\n    Field('%s','%s')," % (
+                        name, field_type)
+            web2py_table_code = "legacy_db.define_table('%s',%s\n    migrate=False)" % (table_name, web2py_table_code)
+            legacy_db_table_web2py_code.append(web2py_table_code)
+    #----------------------------------------
+    #write the legacy db to file
+    legacy_db_web2py_code = connection_string + "\n\n"
+    legacy_db_web2py_code += "\n\n#--------\n".join(
+        legacy_db_table_web2py_code)
+    return legacy_db_web2py_code
+
+if len(sys.argv) < 2:
+    print 'USAGE:\n\n    extract_mysql_models.py data_basename\n\n'
+else:
+    print "# -*- coding: utf-8 -*-"
+    print sqlite(sys.argv[1])

scripts/sessions2trash.py

@@ -23,6 +23,7 @@ Typical usage:
 """
 
 from __future__ import with_statement
+from gluon import current
 from gluon.storage import Storage
 from optparse import OptionParser
 import cPickle
@@ -93,12 +94,10 @@ class SessionSetDb(SessionSet):
     def get(self):
         """Return list of SessionDb instances for existing sessions."""
         sessions = []
-        tablename = 'web2py_session'
-        from gluon import current
-        (record_id_name, table, record_id, unique_key) = \
-            current.response._dbtable_and_field
-        for row in table._db(table.id > 0).select():
-            sessions.append(SessionDb(row))
+        table = current.response.session_db_table
+        if table:
+            for row in table._db(table.id > 0).select():
+                sessions.append(SessionDb(row))
         return sessions
 
 
@@ -121,9 +120,7 @@ class SessionDb(object):
         self.row = row
 
     def delete(self):
-        from gluon import current
-        (record_id_name, table, record_id, unique_key) = \
-            current.response._dbtable_and_field
+        table = current.response.session_db_table
         self.row.delete_record()
         table._db.commit()
 

scripts/setup-web2py-heroku.sh

@@ -0,0 +1,16 @@
+read -p "Choose your admin password?" passwd
+sudo pip install virtualenv
+sudo pip install psycopg2
+virtualenv venv --distribute
+source venv/bin/activate
+pip freeze > requirements.txt
+echo "web: python web2py.py -a '$passwd' -i 0.0.0.0 -p \$PORT" > Procfile
+git init
+git add .
+git add Procfile
+git commit -a -m "first commit"
+heroku create
+git push heroku master
+heroku addons:add heroku-postgresql:dev
+heroku scale web=1
+heroku open

scripts/setup-web2py-nginx-uwsgi-ubuntu.sh

@@ -4,7 +4,6 @@ echo 'Requires Ubuntu 12.04 and installs Nginx + uWSGI + Web2py'
 # Get Web2py Admin Password
 echo -e "Web2py Admin Password: \c "
 read  PW
-
 # Upgrade and install needed software
 apt-get update
 apt-get -y upgrade
@@ -12,24 +11,31 @@ apt-get -y dist-upgrade
 apt-get autoremove
 apt-get autoclean
 apt-get -y install nginx-full
-apt-get -y install uwsgi uwsgi-plugin-python
-
+apt-get -y install build-essential python-dev libxml2-dev python-pip
+pip install --upgrade pip
+pip install --upgrade uwsgi
 # Create configuration file /etc/nginx/sites-available/web2py
 echo 'server {
         listen          80;
         server_name     $hostname;
+        #to enable correct use of response.static_version
+        #location ~* /(\w+)/static(?:/_[\d]+\.[\d]+\.[\d]+)?/(.*)$ {
+        #    alias /home/www-data/web2py/applications/$1/static/$2;
+        #    expires max;
+        #}
         location ~* /(\w+)/static/ {
-           root /home/www-data/web2py/applications/;
+            root /home/www-data/web2py/applications/;
+            #remove next comment on production
+            #expires max;
         }
-         location / {
-                #uwsgi_pass      127.0.0.1:9001;
-                uwsgi_pass      unix:///run/uwsgi/app/web2py/web2py.socket;
-                include         uwsgi_params;
-                uwsgi_param     UWSGI_SCHEME $scheme;
-                uwsgi_param     SERVER_SOFTWARE    nginx/$nginx_version;
+        location / {
+            #uwsgi_pass      127.0.0.1:9001;
+            uwsgi_pass      unix:///tmp/web2py.socket;
+            include         uwsgi_params;
+            uwsgi_param     UWSGI_SCHEME $scheme;
+            uwsgi_param     SERVER_SOFTWARE    nginx/$nginx_version;
         }
 }
-
 server {
         listen          443;
         server_name     $hostname;
@@ -37,11 +43,11 @@ server {
         ssl_certificate         /etc/nginx/ssl/web2py.crt;
         ssl_certificate_key     /etc/nginx/ssl/web2py.key;
         location / {
-                #uwsgi_pass      127.0.0.1:9001;
-                uwsgi_pass      unix:///run/uwsgi/app/web2py/web2py.socket;
-                include         uwsgi_params;
-                uwsgi_param     UWSGI_SCHEME $scheme;
-                uwsgi_param     SERVER_SOFTWARE    nginx/$nginx_version;
+            #uwsgi_pass      127.0.0.1:9001;
+            uwsgi_pass      unix:///tmp/web2py.socket;
+            include         uwsgi_params;
+            uwsgi_param     UWSGI_SCHEME $scheme;
+            uwsgi_param     SERVER_SOFTWARE    nginx/$nginx_version;
         }
 
 }' >/etc/nginx/sites-available/web2py
@@ -54,17 +60,18 @@ openssl genrsa -out web2py.key 1024
 openssl req -batch -new -key web2py.key -out web2py.csr
 openssl x509 -req -days 1780 -in web2py.csr -signkey web2py.key -out web2py.crt
 
-# Create configuration file /etc/uwsgi/apps-available/web2py.xml
+# Prepare folders for uwsgi
+sudo mkdir /etc/uwsgi
+sudo mkdir /var/log/uwsgi
+
+# Create configuration file /etc/uwsgi/web2py.xml
 echo '<uwsgi>
-    <plugin>python</plugin>
-    <socket>/run/uwsgi/app/web2py/web2py.socket</socket>
+    <socket>/tmp/web2py.socket</socket>
     <pythonpath>/home/www-data/web2py/</pythonpath>
-    <app mountpoint="/">
-        <script>wsgihandler</script>
-    </app>
+    <mount>/=wsgihandler:application</mount>
     <master/>
     <processes>4</processes>
-    <harakiri>60</harakiri> 
+    <harakiri>60</harakiri>
     <reload-mercy>8</reload-mercy>
     <cpu-affinity>1</cpu-affinity>
     <stats>/tmp/stats.socket</stats>
@@ -72,11 +79,30 @@ echo '<uwsgi>
     <limit-as>512</limit-as>
     <reload-on-as>256</reload-on-as>
     <reload-on-rss>192</reload-on-rss>
+    <uid>www-data</uid>
+    <gid>www-data</gid>
+    <cron>0 0 -1 -1 -1 python /home/www-data/web2py/web2py.py -Q -S welcome -M -R scripts/sessions2trash.py -A -o</cron>
     <no-orphans/>
-    <vacuum/>    
-</uwsgi>' >/etc/uwsgi/apps-available/web2py.xml
-ln -s /etc/uwsgi/apps-available/web2py.xml /etc/uwsgi/apps-enabled/web2py.xml
+</uwsgi>' >/etc/uwsgi/web2py.xml
 
+#Create a configuration file for uwsgi in emperor-mode
+#for Upstart in /etc/init/uwsgi-emperor.conf
+echo '# Emperor uWSGI script
+
+description "uWSGI Emperor"
+start on runlevel [2345]
+stop on runlevel [06]
+##
+#remove the comments in the next section to enable static file compression for the welcome app
+#in that case, turn on gzip_static on; on /etc/nginx/nginx.conf
+##
+#pre-start script
+#    python /home/www-data/web2py/web2py.py -S welcome -R scripts/zip_static_files.py
+#    chown -R www-data:www-data /home/www-data/web2py/*
+#end script
+respawn
+exec uwsgi --master --die-on-term --emperor /etc/uwsgi --logto /var/log/uwsgi/uwsgi.log
+' > /etc/init/uwsgi-emperor.conf
 # Install Web2py
 apt-get -y install unzip
 mkdir /home/www-data
@@ -84,9 +110,18 @@ cd /home/www-data
 wget http://web2py.com/examples/static/web2py_src.zip
 unzip web2py_src.zip
 rm web2py_src.zip
+# Download latest version of sessions2trash.py
+wget http://web2py.googlecode.com/hg/scripts/sessions2trash.py -O /home/www-data/web2py/scripts/sessions2trash.py
 chown -R www-data:www-data web2py
 cd /home/www-data/web2py
 sudo -u www-data python -c "from gluon.main import save_password; save_password('$PW',443)"
-/etc/init.d/uwsgi restart
+start uwsgi-emperor
 /etc/init.d/nginx restart
 
+## you can reload uwsgi with
+# restart uwsgi-emperor
+## and stop it with
+# stop uwsgi-emperor
+## to reload web2py only (without restarting uwsgi)
+# touch /etc/uwsgi/web2py.xml
+

scripts/sync_languages.py

@@ -7,14 +7,10 @@ import sys
 import shutil
 import os
 
-from gluon.languages import findT, utf8_repr
+from gluon.languages import findT
 
 sys.path.insert(0, '.')
 
-file = sys.argv[1]
-apps = sys.argv[2:]
-
-
 def sync_language(d, data):
     ''' this function makes sure a translated string will be prefered over an untranslated
     string when syncing languages between apps. when both are translated, it prefers the
@@ -37,35 +33,46 @@ def sync_language(d, data):
 
     return d
 
-d = {}
-for app in apps:
-    path = 'applications/%s/' % app
-    findT(path, file)
-    langfile = open(os.path.join(path, 'languages', '%s.py' % file))
+def sync_main(file, apps):
+    d = {}
+    for app in apps:
+        path = 'applications/%s/' % app
+        findT(path, file)
+        langfile = open(os.path.join(path, 'languages', '%s.py' % file))
+        try:
+            data = eval(langfile.read())
+        finally:
+            langfile.close()
+    
+        d = sync_language(d, data)
+
+
+    path = 'applications/%s/' % apps[-1]
+    file1 = os.path.join(path, 'languages', '%s.py' % file)
+    
+    f = open(file1, 'w')
     try:
-        data = eval(langfile.read())
+        f.write('# coding: utf8\n')
+        f.write('{\n')
+        keys = d.keys()
+        keys.sort()
+        for key in keys:
+            f.write("'''%s''':'''%s''',\n" % (key.replace("'", "\\'"), str(d[key].replace("'", "\\'"))))
+        f.write('}\n')
     finally:
-        langfile.close()
+        f.close()
+    
+    oapps = reversed(apps[:-1])
+    for app in oapps:
+        path2 = 'applications/%s/' % app
+        file2 = os.path.join(path2, 'languages', '%s.py' % file)
+        if file1 != file2:
+            shutil.copyfile(file1, file2)
 
-    d = sync_language(d, data)
+if __name__ == "__main__":
 
-path = 'applications/%s/' % apps[-1]
-file1 = os.path.join(path, 'languages', '%s.py' % file)
+    file = sys.argv[1]
+    apps = sys.argv[2:]
 
-f = open(file1, 'w')
-try:
-    f.write('# coding: utf8\n')
-    f.write('{\n')
-    keys = d.keys()
-    keys.sort()
-    for key in keys:
-        f.write('%s:%s,\n' % (utf8_repr(key), utf8_repr(str(d[key]))))
-    f.write('}\n')
-finally:
-    f.close()
-
-oapps = reversed(apps[:-1])
-for app in oapps:
-    path2 = 'applications/%s/' % app
-    file2 = os.path.join(path2, 'languages', '%s.py' % file)
-    shutil.copyfile(file1, file2)
+    sync_main(file, apps)
+