R-2.9.7

fix custom view delimiters

CHANGELOG

@@ -1,10 +1,13 @@
-## 2.9.6
+## 2.9.6-2.9.7
 
+- fixed support of GAE + SQL
 - fixed a typo in the license of some login_methods code. It is now LGPL consistently with the rest of the web2py code. This change applied to all previous web2py versions.
 - support for SAML2 (with pysaml2)
 - Sphinx documentation (thanks Niphlod)
 - improved scheduler (thanks Niphlod)
 - increased security
+- better cache.dick (thanks Leonel)
+- sessions are stored in subfolders for speed
 - postgres support for "INSERT ... RETURING ..."
 - ldap support for Certificate Authority (thanks Maggs and Shane)
 - improved support for S/Mime X.509 (thanks Gyuris)

Makefile

@@ -30,20 +30,20 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.9.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.9.7-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
 	rm -f routes.py 
-	rm -f applications/*/sessions/*
-	rm -f applications/*/errors/* | echo 'too many files'
-	rm -f applications/*/cache/*
-	rm -f applications/admin/databases/*                 
-	rm -f applications/welcome/databases/*               
-	rm -f applications/examples/databases/*             
-	rm -f applications/admin/uploads/*                 
-	rm -f applications/welcome/uploads/*               
-	rm -f applications/examples/uploads/* 
+	rm -rf applications/*/sessions/*
+	rm -rf applications/*/errors/* | echo 'too many files'
+	rm -rf applications/*/cache/*
+	rm -rf applications/admin/databases/*                 
+	rm -rf applications/welcome/databases/*               
+	rm -rf applications/examples/databases/*             
+	rm -rf applications/admin/uploads/*                 
+	rm -rf applications/welcome/uploads/*               
+	rm -rf applications/examples/uploads/* 
 	### NO MORE make epydoc
 	# make epydoc
 	### make welcome layout and appadmin the default

VERSION

@@ -1 +1 @@
-Version 2.9.6-stable+timestamp.2014.09.01.20.55.31
+Version 2.9.7-stable+timestamp.2014.09.04.22.37.07

gluon/cache.py

@@ -20,20 +20,27 @@ caching will be provided by the GAE memcache
 (see gluon.contrib.gae_memcache)
 """
 import time
-import portalocker
-import shelve
+import shutil
 import thread
 import os
+import sys
 import logging
 import re
 import hashlib
 import datetime
+import tempfile
+from gluon import recfile
 try:
     from gluon import settings
     have_settings = True
 except ImportError:
     have_settings = False
 
+try:
+   import cPickle as pickle
+except:
+   import pickle
+
 logger = logging.getLogger("web2py.cache")
 
 __all__ = ['Cache', 'lazy_cache']
@@ -122,7 +129,7 @@ class CacheAbstract(object):
         Auxiliary function called by `clear` to search and clear cache entries
         """
         r = re.compile(regex)
-        for (key, value) in storage.items():
+        for key in storage:
             if r.match(str(key)):
                 del storage[key]
                 break
@@ -242,7 +249,6 @@ class CacheOnDisk(CacheAbstract):
 
     This is implemented as a shelve object and it is shared by multiple web2py
     processes (and threads) as long as they share the same filesystem.
-    The file is locked when accessed.
 
     Disk cache provides persistance when web2py is started/stopped but it slower
     than `CacheInRam`
@@ -250,137 +256,158 @@ class CacheOnDisk(CacheAbstract):
     Values stored in disk cache must be pickable.
     """
 
-    def _close_shelve_and_unlock(self):
-        try:
-            if self.storage:
-                self.storage.close()
-        except ValueError:
-            pass
-        finally:
-            self.storage = None
-            if self.locker and self.locked:
-                portalocker.unlock(self.locker)
-                self.locker.close()
-                self.locked = False
+    class PersistentStorage(object):
+        """
+        Implements a key based storage in disk.
+        """
+        def __init__(self, folder):
+            self.folder = folder
+            # Check the best way to do atomic file replacement.
+            if sys.version_info >= (3, 3):
+                self.replace = os.replace
+            elif sys.platform == "win32":
+                import ctypes
+                from ctypes import wintypes
+                ReplaceFile = ctypes.windll.kernel32.ReplaceFileW
+                ReplaceFile.restype = wintypes.BOOL
+                ReplaceFile.argtypes = [
+                    wintypes.LPWSTR,
+                    wintypes.LPWSTR,
+                    wintypes.LPWSTR,
+                    wintypes.DWORD,
+                    wintypes.LPVOID,
+                    wintypes.LPVOID,
+                    ]
 
-    def _open_shelve_and_lock(self):
-        """Open and return a shelf object, obtaining an exclusive lock
-        on self.locker first. Replaces the close method of the
-        returned shelf instance with one that releases the lock upon
-        closing."""
+                def replace_windows(src, dst):
+                    if not ReplaceFile(dst, src, None, 0, 0, 0):
+                        os.rename(src, dst)
 
-        storage = None
-        locker = None
-        locked = False
-        try:
-            locker = locker = open(self.locker_name, 'a')
-            portalocker.lock(locker, portalocker.LOCK_EX)
-            locked = True
+                self.replace = replace_windows
+            else:
+                # POSIX rename() is always atomic
+                self.replace = os.rename
+
+
+        def __setitem__(self, key, value):
+            tmp_name, tmp_path = tempfile.mkstemp(dir=self.folder)
+            tmp = os.fdopen(tmp_name, 'wb')
             try:
-                storage = shelve.open(self.shelve_name)
-            except:
-                logger.error('corrupted cache file %s, will try rebuild it'
-                             % self.shelve_name)
-                storage = None
-            if storage is None:
-                if os.path.exists(self.shelve_name):
-                    os.unlink(self.shelve_name)
-                storage = shelve.open(self.shelve_name)
-            if not CacheAbstract.cache_stats_name in storage.keys():
-                storage[CacheAbstract.cache_stats_name] = {
-                    'hit_total': 0, 'misses': 0}
-            storage.sync()
-        except Exception, e:
-            if storage:
-                storage.close()
-                storage = None
-            if locked:
-                portalocker.unlock(locker)
-                locker.close()
-            locked = False
-            raise RuntimeError(
-                'unable to create/re-create cache file %s' % self.shelve_name)
-        self.locker = locker
-        self.locked = locked
-        self.storage = storage
-        return storage
+                pickle.dump((time.time(), value), tmp, pickle.HIGHEST_PROTOCOL)
+            finally:
+                tmp.close()
+            fullfilename = os.path.join(self.folder, recfile.generate(key))
+            if not os.path.exists(os.path.dirname(fullfilename)):
+                os.makedirs(os.path.dirname(fullfilename))
+            self.replace(tmp_path, fullfilename)
+
+
+        def __getitem__(self, key):
+            if recfile.exists(key, path=self.folder):
+                timestamp, value = pickle.load(recfile.open(key, 'rb', path=self.folder))
+                return value
+            else:
+                raise KeyError
+
+        def __contains__(self, key):
+            return recfile.exists(key, path=self.folder)
+
+
+        def __delitem__(self, key):
+            recfile.remove(key, path=self.folder)
+
+
+        def __iter__(self):
+            for dirpath, dirnames, filenames in os.walk(self.folder):
+                for filename in filenames:
+                    yield filename
+
+
+        def get(self, key, default=None):
+            try:
+                return self[key]
+            except KeyError:
+                return default
+
+
+        def clear(self):
+            for key in self:
+                del self[key]
 
     def __init__(self, request=None, folder=None):
         self.initialized = False
         self.request = request
         self.folder = folder
-        self.storage = {}
+        self.storage = None
+
 
     def initialize(self):
         if self.initialized:
             return
         else:
             self.initialized = True
+
         folder = self.folder
         request = self.request
 
         # Lets test if the cache folder exists, if not
         # we are going to create it
-        folder = folder or os.path.join(request.folder, 'cache')
+        folder = os.path.join(folder or request.folder, 'cache')
 
         if not os.path.exists(folder):
             os.mkdir(folder)
 
-        ### we need this because of a possible bug in shelve that may
-        ### or may not lock
-        self.locker_name = os.path.join(folder, 'cache.lock')
-        self.shelve_name = os.path.join(folder, 'cache.shelve')
+        self.storage = CacheOnDisk.PersistentStorage(folder)
+        
+        if not CacheAbstract.cache_stats_name in self.storage:
+            self.storage[CacheAbstract.cache_stats_name] = {'hit_total': 0, 'misses': 0}
 
-    def clear(self, regex=None):
-        self.initialize()
-        storage = self._open_shelve_and_lock()
-        try:
-            if regex is None:
-                storage.clear()
-            else:
-                self._clear(storage, regex)
-            storage.sync()
-        finally:
-            self._close_shelve_and_unlock()
 
     def __call__(self, key, f,
                  time_expire=DEFAULT_TIME_EXPIRE):
         self.initialize()
+
         dt = time_expire
-        storage = self._open_shelve_and_lock()
-        try:
-            item = storage.get(key, None)
-            storage[CacheAbstract.cache_stats_name]['hit_total'] += 1
-            if item and f is None:
-                del storage[key]
-                storage.sync()
-            now = time.time()
-            if f is None:
-                value = None
-            elif item and (dt is None or item[0] > now - dt):
-                value = item[1]
-            else:
-                value = f()
-                storage[key] = (now, value)
-                storage[CacheAbstract.cache_stats_name]['misses'] += 1
-                storage.sync()
-        finally:
-            self._close_shelve_and_unlock()
+        item = self.storage.get(key)
+        self.storage[CacheAbstract.cache_stats_name]['hit_total'] += 1
+
+        if item and f is None:
+            del self.storage[key]
+
+        if f is None:
+            return None
+
+        now = time.time()
+
+        if item and ((dt is None) or (item[0] > now - dt)):
+            value = item[1]
+        else:
+            value = f()
+            self.storage[key] = (now, value)
+            self.storage[CacheAbstract.cache_stats_name]['misses'] += 1
 
         return value
 
+    def clear(self, regex=None):
+        self.initialize()
+        storage = self.storage
+        if regex is None:
+            storage.clear()
+        else:
+            self._clear(storage, regex)
+
+        if not CacheAbstract.cache_stats_name in storage:
+            storage[CacheAbstract.cache_stats_name] = {
+                'hit_total': 0, 'misses': 0}
+
+
     def increment(self, key, value=1):
         self.initialize()
-        storage = self._open_shelve_and_lock()
-        try:
-            if key in storage:
-                value = storage[key][1] + value
-            storage[key] = (time.time(), value)
-            storage.sync()
-        finally:
-            self._close_shelve_and_unlock()
+        self.storage[key] += value
         return value
 
+
+
 class CacheAction(object):
     def __init__(self, func, key, time_expire, cache, cache_model):
         self.__name__ = func.__name__

gluon/compileapp.py

@@ -38,6 +38,7 @@ import marshal
 import shutil
 import imp
 import logging
+import types
 logger = logging.getLogger("web2py")
 from gluon import rewrite
 from custom_import import custom_import_install
@@ -211,7 +212,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
             request.env.path_info
         other_request.cid = target
         other_request.env.http_web2py_component_element = target
-        other_request.restful = request.restful  # Needed when you call LOAD() on a controller who has some actions decorates with @request.restful()
+        other_request.restful = types.MethodType(request.restful.im_func, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful()
         other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
 
         other_environment = copy.copy(current.globalenv)  # NASTY

gluon/dal.py

@@ -198,7 +198,6 @@ if PYTHON_VERSION[:2] < (2, 7):
 else:
     from collections import OrderedDict
 
-
 CALLABLETYPES = (types.LambdaType, types.FunctionType,
                  types.BuiltinFunctionType,
                  types.MethodType, types.BuiltinMethodType)
@@ -4587,24 +4586,28 @@ class CubridAdapter(MySQLAdapter):
 ######## GAE MySQL ##########
 class DatabaseStoredFile:
 
-    web2py_filesystem = False
+    web2py_filesystems = set()
 
     def escape(self, obj):
         return self.db._adapter.escape(obj)
 
+    @staticmethod
+    def try_create_web2py_filesystem(db):
+        if not db._uri in DatabaseStoredFile.web2py_filesystems:
+            if db._adapter.dbengine == 'mysql':
+                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
+            elif db._adapter.dbengine in ('postgres', 'sqlite'):
+                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
+            db.executesql(sql)
+            DatabaseStoredFile.web2py_filesystems.add(db._uri)
+ 
     def __init__(self, db, filename, mode):
         if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'):
             raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now")
         self.db = db
         self.filename = filename
         self.mode = mode
-        if not self.web2py_filesystem:
-            if db._adapter.dbengine == 'mysql':
-                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
-            elif db._adapter.dbengine in ('postgres', 'sqlite'):
-                sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
-            self.db.executesql(sql)
-            DatabaseStoredFile.web2py_filesystem = True
+        DatabaseStoredFile.try_create_web2py_filesystem(db)
         self.p = 0
         self.data = ''
         if mode in ('r', 'rw', 'a'):
@@ -4655,6 +4658,9 @@ class DatabaseStoredFile:
     def exists(db, filename):
         if exists(filename):
             return True
+
+        DatabaseStoredFile.try_create_web2py_filesystem(db)
+
         query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
         try:
             if db.executesql(query):
@@ -5137,35 +5143,35 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
             return [GAEF(first.name, '!=', self.represent(second, first.type), lambda a, b:a!=b)]
         else:
             if not second is None:
-                second = Key.from_path(first._tablename, long(second))
+                second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '!=', second, lambda a, b:a!=b)]
 
     def LT(self, first, second=None):
         if first.type != 'id':
             return [GAEF(first.name, '<', self.represent(second, first.type), lambda a, b:a<b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '<', second, lambda a, b:a<b)]
 
     def LE(self, first, second=None):
         if first.type != 'id':
             return [GAEF(first.name, '<=', self.represent(second, first.type), lambda a, b:a<=b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '<=', second, lambda a, b:a<=b)]
 
     def GT(self, first, second=None):
         if first.type != 'id' or second==0 or second == '0':
             return [GAEF(first.name, '>', self.represent(second, first.type), lambda a, b:a>b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '>', second, lambda a, b:a>b)]
 
     def GE(self, first, second=None):
         if first.type != 'id':
             return [GAEF(first.name, '>=', self.represent(second, first.type), lambda a, b:a>=b)]
         else:
-            second = Key.from_path(first._tablename, long(second))
+            second = self.keyfunc(first._tablename, long(second))
             return [GAEF(first.name, '>=', second, lambda a, b:a>=b)]
 
     def INVERT(self, first):

gluon/globals.py

@@ -25,6 +25,7 @@ from gluon.serializers import json, custom_json
 import gluon.settings as settings
 from gluon.utils import web2py_uuid, secure_dumps, secure_loads
 from gluon.settings import global_settings
+from gluon import recfile
 import hashlib
 import portalocker
 import cPickle
@@ -165,7 +166,6 @@ class Request(Storage):
     - is_local
     - is_https
     - restful()
-    - settings
     """
 
     def __init__(self, env):
@@ -825,7 +825,7 @@ class Session(Storage):
                                      'sessions', response.session_id)
                     try:
                         response.session_file = \
-                            open(response.session_filename, 'rb+')
+                            recfile.open(response.session_filename, 'rb+')
                         portalocker.lock(response.session_file,
                                          portalocker.LOCK_EX)
                         response.session_locked = True
@@ -1147,7 +1147,7 @@ class Session(Storage):
                 session_folder = os.path.dirname(response.session_filename)
                 if not os.path.exists(session_folder):
                     os.mkdir(session_folder)
-                response.session_file = open(response.session_filename, 'wb')
+                response.session_file = recfile.open(response.session_filename, 'wb')
                 portalocker.lock(response.session_file, portalocker.LOCK_EX)
                 response.session_locked = True
             if response.session_file:

gluon/recfile.py

@@ -0,0 +1,57 @@
+import os, uuid
+
+def generate(filename, depth=2, base=512):
+    dummyhash = sum(ord(c)*256**(i % 4) for i,c in enumerate(filename)) % base**depth
+    folders = []
+    for level in range(depth-1,-1,-1):
+        code, dummyhash = divmod(dummyhash, base**level)
+        folders.append("%03x" % code)
+    folders.append(filename)
+    return os.path.join(*folders)
+
+def exists(filename, path=None):
+    if os.path.exists(filename):
+        return True
+    if path is None:
+        path, filename = os.path.split(filename)
+    fullfilename = os.path.join(path, generate(filename))
+    if os.path.exists(fullfilename):
+        return True
+    return False
+
+def remove(filename, path=None):
+    if os.path.exists(filename):
+        return os.unlink(filename)
+    if path is None:
+        path, filename = os.path.split(filename)
+    fullfilename = os.path.join(path, generate(filename))
+    if os.path.exists(fullfilename):
+        return os.unlink(fullfilename)
+    raise IOError
+
+def open(filename, mode="r", path=None):
+    if not path:
+        path, filename = os.path.split(filename)
+    fullfilename = None
+    if not mode.startswith('w'):
+        fullfilename = os.path.join(path, filename)
+        if not os.path.exists(fullfilename):
+            fullfilename = None
+    if not fullfilename:
+        fullfilename = os.path.join(path, generate(filename))
+        if mode.startswith('w') and not os.path.exists(os.path.dirname(fullfilename)):
+            os.makedirs(os.path.dirname(fullfilename))
+    return file(fullfilename, mode)
+
+def test():
+    if not os.path.exists('tests'):
+        os.mkdir('tests')
+    for k in range(20):
+        filename = os.path.join('tests',str(uuid.uuid4())+'.test')
+        open(filename, "w").write('test')
+        assert open(filename, "r").read()=='test'
+        if exists(filename):
+            remove(filename)
+
+if __name__ == '__main__':
+    test()

gluon/scheduler.py

@@ -96,7 +96,7 @@ IDENTIFIER = "%s#%s" % (socket.gethostname(),os.getpid())
 logger = logging.getLogger('web2py.scheduler.%s' % IDENTIFIER)
 
 from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
-from gluon import IS_INT_IN_RANGE, IS_DATETIME
+from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB
 from gluon.utils import web2py_uuid
 from gluon.storage import Storage
 
@@ -671,7 +671,10 @@ class Scheduler(MetaScheduler):
         db.define_table(
             'scheduler_task_deps',
             Field('job_name', default='job_0'),
-            Field('task_parent', 'reference scheduler_task'),
+            Field('task_parent', 'integer',
+                requires=IS_IN_DB(db, 'scheduler_task.id',
+                                      '%(task_name)s')
+            ),
             Field('task_child', 'reference scheduler_task'),
             Field('can_visit', 'boolean', default=False),
             migrate=self.__get_migrate('scheduler_task_deps', migrate)
@@ -1311,7 +1314,7 @@ class Scheduler(MetaScheduler):
         """
         from gluon.dal import Query
         sr, st = self.db.scheduler_run, self.db.scheduler_task
-        if isinstance(ref, int):
+        if isinstance(ref, (int, long)):
             q = st.id == ref
         elif isinstance(ref, str):
             q = st.uuid == ref
@@ -1362,7 +1365,7 @@ class Scheduler(MetaScheduler):
             Experimental
         """
         st, sw = self.db.scheduler_task, self.db.scheduler_worker
-        if isinstance(ref, int):
+        if isinstance(ref, (int, long)):
             q = st.id == ref
         elif isinstance(ref, str):
             q = st.uuid == ref

gluon/sqlhtml.py

@@ -2111,6 +2111,8 @@ class SQLFORM(FORM):
                 field_id = groupby #take the field passed as groupby
             elif groupby and isinstance(groupby, Expression):
                 field_id = groupby.first #take the first groupby field
+                while not(isinstance(field_id, Field)): # Navigate to the first Field of the expression
+                    field_id = field_id.first
         table = field_id.table
         tablename = table._tablename
         if not any(str(f) == str(field_id) for f in fields):

gluon/template.py

@@ -279,15 +279,19 @@ class TemplateParser(object):
         self.context = context
 
         # allow optional alternative delimiters
-        if delimiters is None:
-            delimiters = context.get('response', {})\
-                .get('app_settings',{}).get('template_delimiters')
+        
         if delimiters != self.default_delimiters:
-            escaped_delimiters = (escape(elimiters[0]),
+            escaped_delimiters = (escape(delimiters[0]),
                                   escape(delimiters[1]))
             self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters, DOTALL)
-        else:
-            delimiters = self.default_delimiters
+        elif hasattr(context.get('response', None), 'delimiters'):
+            if context['response'].delimiters != self.default_delimiters:
+                delimiters = context['response'].delimiters
+                escaped_delimiters = (
+                    escape(delimiters[0]),
+                    escape(delimiters[1]))
+                self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters,
+                                     DOTALL)
         self.delimiters = delimiters
 
         # Create a root level Content that everything will go into.

gluon/tools.py

@@ -3007,7 +3007,7 @@ class Auth(object):
 
         if self.settings.prevent_password_reset_attacks:
             key = request.vars.key
-            if not key and len(request.args)>1:
+            if not key and len(request.args)>0:
                 key = request.args[-1]
             if key:
                 session._reset_password_key = key