R-2.9.9

fix web2py.js with show_if and grid

CHANGELOG

@@ -1,10 +1,13 @@
-## 2.9.6
+## 2.9.6 - 2.9.8
 
+- fixed support of GAE + SQL
 - fixed a typo in the license of some login_methods code. It is now LGPL consistently with the rest of the web2py code. This change applied to all previous web2py versions.
 - support for SAML2 (with pysaml2)
 - Sphinx documentation (thanks Niphlod)
 - improved scheduler (thanks Niphlod)
 - increased security
+- better cache.dick (thanks Leonel)
+- sessions are stored in subfolders for speed
 - postgres support for "INSERT ... RETURING ..."
 - ldap support for Certificate Authority (thanks Maggs and Shane)
 - improved support for S/Mime X.509 (thanks Gyuris)

Makefile

@@ -30,20 +30,20 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.9.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.9.9-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
 	rm -f routes.py 
-	rm -f applications/*/sessions/*
-	rm -f applications/*/errors/* | echo 'too many files'
-	rm -f applications/*/cache/*
-	rm -f applications/admin/databases/*                 
-	rm -f applications/welcome/databases/*               
-	rm -f applications/examples/databases/*             
-	rm -f applications/admin/uploads/*                 
-	rm -f applications/welcome/uploads/*               
-	rm -f applications/examples/uploads/* 
+	rm -rf applications/*/sessions/*
+	rm -rf applications/*/errors/* | echo 'too many files'
+	rm -rf applications/*/cache/*
+	rm -rf applications/admin/databases/*                 
+	rm -rf applications/welcome/databases/*               
+	rm -rf applications/examples/databases/*             
+	rm -rf applications/admin/uploads/*                 
+	rm -rf applications/welcome/uploads/*               
+	rm -rf applications/examples/uploads/* 
 	### NO MORE make epydoc
 	# make epydoc
 	### make welcome layout and appadmin the default

README.markdown

@@ -6,7 +6,6 @@ It is written and programmable in Python. LGPLv3 License
 
 Learn more at http://web2py.com
 
-
 ## Google App Engine deployment
 
     cp examples/app.yaml ./
@@ -14,6 +13,10 @@ Learn more at http://web2py.com
     
 Then edit ./app.yaml and replace "yourappname" with yourappname.
 
+## Documentation (readthedocs.org)
+
+[![Docs Status](https://readthedocs.org/projects/web2py/badge/?version=latest)](http://web2py.rtfd.org/)
+
 ## Tests
 
 [![Build Status](https://travis-ci.org/web2py/web2py.png)](https://travis-ci.org/web2py/web2py)

VERSION

@@ -1 +1 @@
-Version 2.9.6-stable+timestamp.2014.09.01.20.55.31
+Version 2.9.9-stable+timestamp.2014.09.08.08.12.34

applications/admin/static/js/web2py.js

@@ -545,8 +545,11 @@
       };
       $('[data-show-trigger]', target).each(function () {
         var name = $(this).attr('data-show-trigger');
-        if(!triggers[name]) triggers[name] = [];
-        triggers[name].push($(this).attr('id'));
+        // The field exists only when creating/editing a row
+        if ($('#' + name).length) {
+            if(!triggers[name]) triggers[name] = [];
+            triggers[name].push($(this).attr('id'));
+        }
       });
       for(var name in triggers) {
         $('#' + name, target).change(show_if).keyup(show_if);

applications/examples/static/js/web2py.js

@@ -545,8 +545,11 @@
       };
       $('[data-show-trigger]', target).each(function () {
         var name = $(this).attr('data-show-trigger');
-        if(!triggers[name]) triggers[name] = [];
-        triggers[name].push($(this).attr('id'));
+        // The field exists only when creating/editing a row
+        if ($('#' + name).length) {
+            if(!triggers[name]) triggers[name] = [];
+            triggers[name].push($(this).attr('id'));
+        }
       });
       for(var name in triggers) {
         $('#' + name, target).change(show_if).keyup(show_if);

applications/welcome/static/js/web2py.js

@@ -545,8 +545,11 @@
       };
       $('[data-show-trigger]', target).each(function () {
         var name = $(this).attr('data-show-trigger');
-        if(!triggers[name]) triggers[name] = [];
-        triggers[name].push($(this).attr('id'));
+        // The field exists only when creating/editing a row
+        if ($('#' + name).length) {
+            if(!triggers[name]) triggers[name] = [];
+            triggers[name].push($(this).attr('id'));
+        }
       });
       for(var name in triggers) {
         $('#' + name, target).change(show_if).keyup(show_if);

gluon/compileapp.py

@@ -38,6 +38,7 @@ import marshal
 import shutil
 import imp
 import logging
+import types
 logger = logging.getLogger("web2py")
 from gluon import rewrite
 from custom_import import custom_import_install
@@ -211,7 +212,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
             request.env.path_info
         other_request.cid = target
         other_request.env.http_web2py_component_element = target
-        other_request.restful = request.restful  # Needed when you call LOAD() on a controller who has some actions decorates with @request.restful()
+        other_request.restful = types.MethodType(request.restful.im_func, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful()
         other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
 
         other_environment = copy.copy(current.globalenv)  # NASTY

gluon/dal.py

@@ -198,7 +198,6 @@ if PYTHON_VERSION[:2] < (2, 7):
 else:
     from collections import OrderedDict
 
-
 CALLABLETYPES = (types.LambdaType, types.FunctionType,
                  types.BuiltinFunctionType,
                  types.MethodType, types.BuiltinMethodType)
@@ -4587,24 +4586,28 @@ class CubridAdapter(MySQLAdapter):
 ######## GAE MySQL ##########
 class DatabaseStoredFile:
 
-    web2py_filesystem = False
+    web2py_filesystems = set()
 
     def escape(self, obj):
         return self.db._adapter.escape(obj)
 
+    @staticmethod
+    def try_create_web2py_filesystem(db):
+        if not db._uri in DatabaseStoredFile.web2py_filesystems:
+            if db._adapter.dbengine == 'mysql':
+                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
+            elif db._adapter.dbengine in ('postgres', 'sqlite'):
+                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
+            db.executesql(sql)
+            DatabaseStoredFile.web2py_filesystems.add(db._uri)
+ 
     def __init__(self, db, filename, mode):
         if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'):
             raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now")
         self.db = db
         self.filename = filename
         self.mode = mode
-        if not self.web2py_filesystem:
-            if db._adapter.dbengine == 'mysql':
-                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
-            elif db._adapter.dbengine in ('postgres', 'sqlite'):
-                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
-            self.db.executesql(sql)
-            DatabaseStoredFile.web2py_filesystem = True
+        DatabaseStoredFile.try_create_web2py_filesystem(db)
         self.p = 0
         self.data = ''
         if mode in ('r', 'rw', 'a'):
@@ -4655,6 +4658,9 @@ class DatabaseStoredFile:
     def exists(db, filename):
         if exists(filename):
             return True
+
+        DatabaseStoredFile.try_create_web2py_filesystem(db)
+
         query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
         try:
             if db.executesql(query):
@@ -5137,35 +5143,35 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
             return [GAEF(first.name, '!=', self.represent(second, first.type), lambda a, b:a!=b)]
         else:
             if not second is None:
-                second = Key.from_path(first._tablename, long(second))
+                second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '!=', second, lambda a, b:a!=b)]
 
     def LT(self, first, second=None):
         if first.type != 'id':
             return [GAEF(first.name, '<', self.represent(second, first.type), lambda a, b:a<b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '<', second, lambda a, b:a<b)]
 
     def LE(self, first, second=None):
         if first.type != 'id':
             return [GAEF(first.name, '<=', self.represent(second, first.type), lambda a, b:a<=b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '<=', second, lambda a, b:a<=b)]
 
     def GT(self, first, second=None):
         if first.type != 'id' or second==0 or second == '0':
             return [GAEF(first.name, '>', self.represent(second, first.type), lambda a, b:a>b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '>', second, lambda a, b:a>b)]
 
     def GE(self, first, second=None):
         if first.type != 'id':
             return [GAEF(first.name, '>=', self.represent(second, first.type), lambda a, b:a>=b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '>=', second, lambda a, b:a>=b)]
 
     def INVERT(self, first):

gluon/fileutils.py

@@ -20,7 +20,7 @@ import datetime
 import logging
 from http import HTTP
 from gzip import open as gzopen
-
+from recfile import generate
 
 __all__ = [
     'parse_version',
@@ -400,6 +400,8 @@ def get_session(request, other_application='admin'):
         session_id = request.cookies['session_id_' + other_application].value
         session_filename = os.path.join(
             up(request.folder), other_application, 'sessions', session_id)
+        if not os.path.exists(session_filename):
+            session_filename = generate(session_filename)        
         osession = storage.load_storage(session_filename)
     except Exception, e:
         osession = storage.Storage()

gluon/globals.py

@@ -25,6 +25,7 @@ from gluon.serializers import json, custom_json
 import gluon.settings as settings
 from gluon.utils import web2py_uuid, secure_dumps, secure_loads
 from gluon.settings import global_settings
+from gluon import recfile
 import hashlib
 import portalocker
 import cPickle
@@ -165,7 +166,6 @@ class Request(Storage):
     - is_local
     - is_https
     - restful()
-    - settings
     """
 
     def __init__(self, env):
@@ -825,7 +825,7 @@ class Session(Storage):
                                      'sessions', response.session_id)
                     try:
                         response.session_file = \
-                            open(response.session_filename, 'rb+')
+                            recfile.open(response.session_filename, 'rb+')
                         portalocker.lock(response.session_file,
                                          portalocker.LOCK_EX)
                         response.session_locked = True
@@ -1147,7 +1147,7 @@ class Session(Storage):
                 session_folder = os.path.dirname(response.session_filename)
                 if not os.path.exists(session_folder):
                     os.mkdir(session_folder)
-                response.session_file = open(response.session_filename, 'wb')
+                response.session_file = recfile.open(response.session_filename, 'wb')
                 portalocker.lock(response.session_file, portalocker.LOCK_EX)
                 response.session_locked = True
             if response.session_file:

gluon/recfile.py

@@ -0,0 +1,63 @@
+import os, uuid
+
+def generate(filename, depth=2, base=512):
+    if os.path.sep in filename:
+        path, filename = os.path.split(filename)
+    else:
+        path = None
+    dummyhash = sum(ord(c)*256**(i % 4) for i,c in enumerate(filename)) % base**depth
+    folders = []
+    for level in range(depth-1,-1,-1):
+        code, dummyhash = divmod(dummyhash, base**level)
+        folders.append("%03x" % code)
+    folders.append(filename)
+    if path:
+        folders.insert(0,path)
+    return os.path.join(*folders)
+
+def exists(filename, path=None):
+    if os.path.exists(filename):
+        return True
+    if path is None:
+        path, filename = os.path.split(filename)
+    fullfilename = os.path.join(path, generate(filename))
+    if os.path.exists(fullfilename):
+        return True
+    return False
+
+def remove(filename, path=None):
+    if os.path.exists(filename):
+        return os.unlink(filename)
+    if path is None:
+        path, filename = os.path.split(filename)
+    fullfilename = os.path.join(path, generate(filename))
+    if os.path.exists(fullfilename):
+        return os.unlink(fullfilename)
+    raise IOError
+
+def open(filename, mode="r", path=None):
+    if not path:
+        path, filename = os.path.split(filename)
+    fullfilename = None
+    if not mode.startswith('w'):
+        fullfilename = os.path.join(path, filename)
+        if not os.path.exists(fullfilename):
+            fullfilename = None
+    if not fullfilename:
+        fullfilename = os.path.join(path, generate(filename))
+        if mode.startswith('w') and not os.path.exists(os.path.dirname(fullfilename)):
+            os.makedirs(os.path.dirname(fullfilename))
+    return file(fullfilename, mode)
+
+def test():
+    if not os.path.exists('tests'):
+        os.mkdir('tests')
+    for k in range(20):
+        filename = os.path.join('tests',str(uuid.uuid4())+'.test')
+        open(filename, "w").write('test')
+        assert open(filename, "r").read()=='test'
+        if exists(filename):
+            remove(filename)
+
+if __name__ == '__main__':
+    test()

gluon/scheduler.py

@@ -96,7 +96,7 @@ IDENTIFIER = "%s#%s" % (socket.gethostname(),os.getpid())
 logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
 from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
-from gluon import IS_INT_IN_RANGE, IS_DATETIME
+from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
 from gluon.utils import web2py_uuid
 from gluon.storage import Storage
 
@@ -671,7 +671,10 @@ class Scheduler(MetaScheduler):
         db.define_table(
             'scheduler_task_deps',
             Field('job_name', default='job_0'),
-            Field('task_parent', 'reference scheduler_task'),
+            Field('task_parent', 'integer',
+                requires=IS_IN_DB(db, 'scheduler_task.id',
+                                      '%(task_name)s')
+            ),
             Field('task_child', 'reference scheduler_task'),
             Field('can_visit', 'boolean', default=False),
             migrate=self.__get_migrate('scheduler_task_deps', migrate)
@@ -1311,7 +1314,7 @@ class Scheduler(MetaScheduler):
         """
         from gluon.dal import Query
         sr, st = self.db.scheduler_run, self.db.scheduler_task
-        if isinstance(ref, int):
+        if isinstance(ref, (int, long)):
             q = st.id == ref
         elif isinstance(ref, str):
             q = st.uuid == ref
@@ -1362,7 +1365,7 @@ class Scheduler(MetaScheduler):
             Experimental
         """
         st, sw = self.db.scheduler_task, self.db.scheduler_worker
-        if isinstance(ref, int):
+        if isinstance(ref, (int, long)):
             q = st.id == ref
         elif isinstance(ref, str):
             q = st.uuid == ref

gluon/sqlhtml.py

@@ -1289,7 +1289,7 @@ class SQLFORM(FORM):
             xfields.append(
                 (self.FIELDKEY_DELETE_RECORD + SQLFORM.ID_ROW_SUFFIX,
                  LABEL(
-                        T(delete_label), separator,
+                        T(delete_label), sep,
                         _for=self.FIELDKEY_DELETE_RECORD,
                         _id=self.FIELDKEY_DELETE_RECORD + \
                             SQLFORM.ID_LABEL_SUFFIX),
@@ -2111,6 +2111,8 @@ class SQLFORM(FORM):
                 field_id = groupby #take the field passed as groupby
             elif groupby and isinstance(groupby, Expression):
                 field_id = groupby.first #take the first groupby field
+                while not(isinstance(field_id, Field)): # Navigate to the first Field of the expression
+                    field_id = field_id.first
         table = field_id.table
         tablename = table._tablename
         if not any(str(f) == str(field_id) for f in fields):

gluon/storage.py

@@ -13,6 +13,7 @@ Provides:
 """
 
 import cPickle
+import copy_reg
 import gluon.portalocker as portalocker
 
 __all__ = ['List', 'Storage', 'Settings', 'Messages',
@@ -129,6 +130,12 @@ class Storage(dict):
         values = self.getlist(key)
         return values[-1] if values else default
 
+
+def pickle_storage(s):
+    return Storage, (dict(s),)
+
+copy_reg.pickle(Storage, pickle_storage)
+
 PICKABLE = (str, int, long, float, bool, list, dict, tuple, set)
 
 

gluon/template.py

@@ -279,15 +279,19 @@ class TemplateParser(object):
         self.context = context
 
         # allow optional alternative delimiters
-        if delimiters is None:
-            delimiters = context.get('response', {})\
-                .get('app_settings',{}).get('template_delimiters')
+        
         if delimiters != self.default_delimiters:
-            escaped_delimiters = (escape(elimiters[0]),
+            escaped_delimiters = (escape(delimiters[0]),
                                   escape(delimiters[1]))
             self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters, DOTALL)
-        else:
-            delimiters = self.default_delimiters
+        elif hasattr(context.get('response', None), 'delimiters'):
+            if context['response'].delimiters != self.default_delimiters:
+                delimiters = context['response'].delimiters
+                escaped_delimiters = (
+                    escape(delimiters[0]),
+                    escape(delimiters[1]))
+                self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters,
+                                     DOTALL)
         self.delimiters = delimiters
 
         # Create a root level Content that everything will go into.

gluon/tools.py

@@ -2710,7 +2710,8 @@ class Auth(object):
             extra_fields = [
                 Field("password_two", "password", requires=IS_EQUAL_TO(
                         request.post_vars.get(passfield,None),
-                        error_message=self.messages.mismatched_password))]
+                        error_message=self.messages.mismatched_password),
+                        label=current.T("Confirm Password"))]
         else:
             extra_fields = []
         form = SQLFORM(table_user,
@@ -3007,7 +3008,7 @@ class Auth(object):
 
         if self.settings.prevent_password_reset_attacks:
             key = request.vars.key
-            if not key and len(request.args)>1:
+            if not key and len(request.args)>0:
                 key = request.args[-1]
             if key:
                 session._reset_password_key = key

scripts/sessions2trash.py

@@ -29,6 +29,12 @@ Typical usage:
 """
 
 from __future__ import with_statement
+
+import sys
+import os
+print os.path.join(*__file__.split(os.sep)[:-2] or ['.'])
+sys.path.append(os.path.join(*__file__.split(os.sep)[:-2] or ['.']))
+
 from gluon import current
 from gluon.storage import Storage
 from optparse import OptionParser
@@ -37,6 +43,7 @@ import datetime
 import os
 import stat
 import time
+import glob
 
 EXPIRATION_MINUTES = 60
 SLEEP_MINUTES = 5
@@ -157,6 +164,9 @@ class SessionFile(object):
     def delete(self):
         try:
             os.unlink(self.filename)
+            path = os.path.dirname(filename)
+            if not path.endswith('sessions') and len(os.listdir(path))==0:
+                os.rmdir(path)
         except:
             pass
 
@@ -191,10 +201,11 @@ def single_loop(expiration=None, force=False, verbose=False):
         except:
             expiration = EXPIRATION_MINUTES * 60
 
-    set_db = SessionSetDb(expiration, force, verbose)
     set_files = SessionSetFiles(expiration, force, verbose)
-    set_db.trash()
     set_files.trash()
+    set_db = SessionSetDb(expiration, force, verbose)
+    set_db.trash()
+
 
 def main():
     """Main processing."""