Merge github.com:web2py/web2py

gluon/admin.py

@@ -60,7 +60,7 @@ def app_pack(app, request, raise_ex=False):
     """
     try:
         app_cleanup(app, request)
-        filename = apath('../deposit/%s.w2p' % app, request)
+        filename = apath('../deposit/web2py.app.%s.w2p' % app, request)
         w2p_pack(filename, apath(app, request))
         return filename
     except Exception, e:

gluon/cache.py

@@ -479,6 +479,17 @@ class Cache(object):
             return CacheAction(func,key,time_expire,self,cache_model)
         return tmp
 
+    @staticmethod
+    def with_prefix(cache_model, prefix):
+        """
+        allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix')
+        it will add prefix to all the cache keys used.
+        """        
+        return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\
+            cache_model(prefix + key, f, time_expire)
+
+
+
 def lazy_cache(key=None,time_expire=None,cache_model='ram'):
     """
     can be used to cache any function including in modules,

gluon/cfs.py

@@ -12,7 +12,7 @@ Functions required to execute app components
 FOR INTERNAL USE ONLY
 """
 
-import os
+from os import stat
 import thread
 import logging
 from fileutils import read_file
@@ -35,7 +35,7 @@ def getcfs(key, filename, filter=None):
     This is used on Google App Engine since pyc files cannot be saved.
     """
     try:
-        t = os.stat(filename).st_mtime
+        t = stat(filename).st_mtime
     except OSError:
         return filter() if callable(filter) else ''
     cfs_lock.acquire()

gluon/contrib/gae_memcache.py

@@ -47,10 +47,12 @@ class MemcacheClient(Client):
         self.set((time.time(), value))
         return value
 
-    def clear(self, key):
-        key = '%s/%s' % (self.request.application, key)
-        self.delete(key)
-
+    def clear(self, key = None):
+        if key:
+            key = '%s/%s' % (self.request.application, key)
+            self.delete(key)
+        else:
+            self.flush_all()
 
 
 

gluon/contrib/login_methods/oauth20_account.py

@@ -107,7 +107,7 @@ server for requests.  It can be used for the optional"scope" parameters for Face
             path_info = next
         else:
             path_info = r.env.path_info
-        uri = '%s://%s%s' %(url_scheme, http_host, path_info)
+        uri = '%s://%s%s' % (url_scheme, http_host, path_info)
         if r.get_vars and not next:
             uri += '?' + urlencode(r.get_vars)
         return uri

gluon/contrib/markmin/markmin2html.py

@@ -540,7 +540,7 @@ regex_strong=re.compile(r'\*\*(?P<t>[^\s*]+( +[^\s*]+)*)\*\*')
 regex_del=re.compile(r'~~(?P<t>[^\s*]+( +[^\s*]+)*)~~')
 regex_em=re.compile(r"''(?P<t>[^\s']+(?: +[^\s']+)*)''")
 regex_num=re.compile(r"^\s*[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?\s*$")
-regex_list=re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s+)?(.*)$')
+regex_list=re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s*)?(.*)$')
 regex_bq_headline=re.compile('^(?:(\.+|\++|\-+)(\.)?\s+)?(-{3}-*)$')
 regex_tq=re.compile('^(-{3}-*)(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[a-zA-Z][_a-zA-Z\-\d]*)\])?)?$')
 regex_proto = re.compile(r'(?<!["\w>/=])(?P<p>\w+):(?P<k>\w+://[\w\d\-+=?%&/:.]+)', re.M)
@@ -1165,17 +1165,17 @@ def render(text,
                         (lev, mtag, lineno)= parse_list(t2, p, ss, 'ol', lev, mtag, lineno)
                         lineno+=1
                         continue
-                    elif c0 == '-': # unordered list
-                        (lev, mtag, lineno) = parse_list(t2, p, ss, 'ul', lev, mtag, lineno)
-                        lineno+=1
-                        continue
+                    elif c0 == '-': # unordered list, table or blockquote
+                        if p or ss:
+                            (lev, mtag, lineno) = parse_list(t2, p, ss, 'ul', lev, mtag, lineno)
+                            lineno+=1
+                            continue
+                        else:
+                            (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
                     elif lev>0: # and c0 == '.' # paragraph in lists
                         (lev, mtag, lineno) = parse_point(t2, ss, lev, mtag, lineno)
                         lineno+=1
                         continue
-                else:
-                    if c0 == '-': # table or blockquote?
-                        (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
 
             if lev == 0 and (mtag == 'q' or s == META):
                 # new paragraph

gluon/contrib/memcache/__init__.py

@@ -17,7 +17,7 @@ DEFAULT_TIME_EXPIRE = 300 # seconds (must be the same as cache.ram)
 def MemcacheClient(*a, **b):
     if not hasattr(current,'__mc_instance'):
         current.__memcache_client = MemcacheClientObj(*a, **b)
-    return current.__memecache_client
+    return current.__memcache_client
 
 class MemcacheClientObj(Client):
 
@@ -61,7 +61,7 @@ class MemcacheClientObj(Client):
             if item:
                 if not isinstance(item,(list,tuple)):
                     value = item
-                elif (item[0] < now - dt): # value expired
+                elif (item[0] < now - time_expire): # value expired
                     item = None # value to be computed
                 else:
                     value = item[1]

gluon/contrib/minify/minify.py

@@ -11,6 +11,7 @@ Modified by: Massimo Di Pierro <massimo.dipierro@gmail.com>
 import cssmin
 import jsmin
 import os
+import hashlib
 
 def read_binary_file(filename):
     f = open(filename,'rb')
@@ -23,9 +24,9 @@ def write_binary_file(filename,data):
     f.write(data)
     f.close()
 
-def fix_links(css,static_path):    
-    return css.replace('../',static_path+'/')
-
+def fix_links(css,static_path):
+    return css.replace('../',static_path+'../')
+    
 def minify(files, path_info, folder, optimize_css, optimize_js,
            ignore_concat = [],
            ignore_minify = ['/jquery.js', '/anytime.js']):
@@ -56,6 +57,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
     new_files = []
     css = []
     js = []
+    processed = []
     for k,filename in enumerate(files):
         if not filename.startswith('/') or \
                 any(filename.endswith(x) for x in ignore_concat):
@@ -66,8 +68,20 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
                                     filename[len(static_path)+1:])        
 
         if filename.lower().endswith('.css'):
+            processed.append(filename)
+            spath_info, sfilename = path_info.split('/'), filename.split('/')
+            u = 0
+            for i,a in enumerate(sfilename):
+                try:
+                    if a != spath_info[i]:
+                        u = i
+                        break
+                except:
+                    pass
             if concat_css:
                 contents = read_binary_file(abs_filename)
+                replacement = '../'*len(spath_info[u:]) + '/'.join(sfilename[u:-1]) + '/'
+                contents = fix_links(contents, replacement)
                 if minify_css:
                     css.append(cssmin.cssmin(contents))
                 else:
@@ -75,6 +89,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
             else:
                 css.append(filename)
         elif filename.lower().endswith('.js'):
+            processed.append(filename)
             if concat_js:
                 contents = read_binary_file(abs_filename)            
                 if minify_js and not filename.endswith('.min.js') and \
@@ -84,19 +99,19 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
                     js.append(contents)
             else:
                 js.append(filename)
+    dest_key = hashlib.md5(repr(processed)).hexdigest()
     if css and concat_css:
         css = '\n\n'.join(contents for contents in css)
-        if inline_css:
-            css = ('css:inline',fix_links(css,static_path))
-        else:
+        if not inline_css:
             temppath = os.path.join(folder,'static',temp)
             if not os.path.exists(temppath): os.mkdir(temppath)
-            tempfile = os.path.join(temppath,'compressed.css')
+            dest = "compressed_%s.css" % dest_key
+            tempfile = os.path.join(temppath, dest)
             write_binary_file(tempfile,css)
-            css = path_info+'/compressed.css'
+            css = path_info+'/%s' % dest
         new_files.append(css)
     else:
-        new_files += css    
+        new_files += css
     if js and concat_js:
         js = '\n'.join(contents for contents in js)
         if inline_js:
@@ -104,9 +119,10 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
         else:
             temppath = os.path.join(folder,'static',temp)
             if not os.path.exists(temppath): os.mkdir(temppath)
-            tempfile = os.path.join(folder,'static',temp,'compressed.js')
+            dest = "compressed_%s.js" % dest_key
+            tempfile = os.path.join(folder,'static',temp,dest)
             write_binary_file(tempfile,js)
-            js = path_info+'/compressed.js'
+            js = path_info+'/%s' % dest
         new_files.append(js)
     else:
         new_files += js

gluon/dal.py

@@ -40,7 +40,7 @@ Example of usage:
 >>> # from dal import DAL, Field
 
 ### create DAL connection (and create DB if it doesn't exist)
->>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'), 
+>>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
 ... folder=None)
 
 ### define a table 'person' (create/alter as necessary)
@@ -185,6 +185,7 @@ SELECT_ARGS = set(
     ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
      'distinct', 'having', 'join','for_update', 'processor','cacheable'))
 
+
 ogetattr = object.__getattribute__
 osetattr = object.__setattr__
 exists = os.path.exists
@@ -241,7 +242,7 @@ REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contain
 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
 REGEX_QUOTES = re.compile("'[^']*'")
-REGEX_ALPHANUMERIC = re.compile('^[a-zA-Z]\w*$')
+REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
 
 # list of drivers will be built on the fly
 # and lists only what is available
@@ -329,7 +330,7 @@ if not 'google' in DRIVERS:
         DRIVERS.append('Sybase(Sybase)')
     except ImportError:
         logger.debug('no Sybase driver')
-        
+
     try:
         import kinterbasdb
         DRIVERS.append('Interbase(kinterbasdb)')
@@ -341,7 +342,7 @@ if not 'google' in DRIVERS:
         import fdb
         DRIVERS.append('Firbird(fdb)')
     except ImportError:
-        logger.debug('no Firebird driver fdb')    
+        logger.debug('no Firebird driver fdb')
 #####
     try:
         import firebirdsql
@@ -659,7 +660,7 @@ class BaseAdapter(ConnectionPool):
 
     def find_driver(self,adapter_args,uri=None):
         if hasattr(self,'driver') and self.driver!=None:
-            return 
+            return
         drivers_available = [driver for driver in self.drivers
                              if driver in globals()]
         if uri:
@@ -667,7 +668,7 @@ class BaseAdapter(ConnectionPool):
             request_driver = items[1] if len(items)>1 else None
         else:
             request_driver = None
-        request_driver = request_driver or adapter_args.get('driver')            
+        request_driver = request_driver or adapter_args.get('driver')
         if request_driver:
             if request_driver in drivers_available:
                 self.driver_name = request_driver
@@ -679,7 +680,7 @@ class BaseAdapter(ConnectionPool):
             self.driver = globals().get(self.driver_name)
         else:
             raise RuntimeError, "no driver available %s", self.drivers
-            
+
 
     def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
                  credential_decoder=IDENTITY, driver_args={},
@@ -798,7 +799,7 @@ class BaseAdapter(ConnectionPool):
                     else:
                         schema = parms[0]
                     ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
-                    ftype = ftype % dict(schema=schema, 
+                    ftype = ftype % dict(schema=schema,
                                          tablename=tablename,
                                          fieldname=field_name, srid=srid,
                                          dimension=dimension)
@@ -889,7 +890,7 @@ class BaseAdapter(ConnectionPool):
         else:
             table._dbt = pjoin(
                 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
-                    
+
         if table._dbt:
             table._loggername = pjoin(dbpath, 'sql.log')
             logfile = self.file_open(table._loggername, 'a')
@@ -1001,7 +1002,7 @@ class BaseAdapter(ConnectionPool):
                     query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
                 metadata_change = True
             elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
-                  and not (key in table.fields and 
+                  and not (key in table.fields and
                            isinstance(table[key].type, SQLCustomType)) \
                   and not sql_fields[key]['type'].startswith('reference')\
                   and not sql_fields[key]['type'].startswith('double')\
@@ -1072,6 +1073,9 @@ class BaseAdapter(ConnectionPool):
     def EXTRACT(self, first, what):
         return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
 
+    def EPOCH(self, first):
+        return self.EXTRACT(first, 'epoch')
+
     def AGGREGATE(self, first, what):
         return "%s(%s)" % (what, self.expand(first))
 
@@ -1272,7 +1276,7 @@ class BaseAdapter(ConnectionPool):
             first = expression.first
             second = expression.second
             op = expression.op
-            if not second is None: 
+            if not second is None:
                 return op(first, second)
             elif not first is None:
                 return op(first)
@@ -1453,6 +1457,8 @@ class BaseAdapter(ConnectionPool):
         having = args_get('having', False)
         limitby = args_get('limitby', False)
         for_update = args_get('for_update', False)
+        if not distinct is True and not distinct is False and not groupby:
+            distinct, groupby = False, distinct
         if self.can_select_for_update is False and for_update is True:
             raise SyntaxError, 'invalid select attribute: for_update'
         if distinct is True:
@@ -1579,14 +1585,14 @@ class BaseAdapter(ConnectionPool):
         """
         sql = self._select(query, fields, attributes)
         cache = attributes.get('cache', None)
-        if cache and attributes.get('cacheable',False):            
+        if cache and attributes.get('cacheable',False):
             del attributes['cache']
             (cache_model, time_expire) = cache
             key = self.uri + '/' + sql
             if len(key)>200: key = hashlib.md5(key).hexdigest()
             args = (sql,fields,attributes)
             return cache_model(
-                key, 
+                key,
                 lambda self=self,args=args:self._select_aux(*args),
                 time_expire)
         else:
@@ -1613,15 +1619,16 @@ class BaseAdapter(ConnectionPool):
         self.execute(self._count(query, distinct))
         return self.cursor.fetchone()[0]
 
-    def tables(self, query):
+    def tables(self, *queries):
         tables = set()
-        if isinstance(query, Field):
-            tables.add(query.tablename)
-        elif isinstance(query, (Expression, Query)):
-            if not query.first is None:
-                tables = tables.union(self.tables(query.first))
-            if not query.second is None:
-                tables = tables.union(self.tables(query.second))
+        for query in queries:
+            if isinstance(query, Field):
+                tables.add(query.tablename)
+            elif isinstance(query, (Expression, Query)):
+                if not query.first is None:
+                    tables = tables.union(self.tables(query.first))
+                if not query.second is None:
+                    tables = tables.union(self.tables(query.second))
         return list(tables)
 
     def commit(self):
@@ -1685,9 +1692,9 @@ class BaseAdapter(ConnectionPool):
             elif not isinstance(obj, (list, tuple)):
                 obj = [obj]
             if field_is_type('list:string'):
-                obj = [str(item) for item in obj]
+                obj = map(str,obj)
             else:
-                obj = [int(item) for item in obj]
+                obj = map(int,obj)
         if isinstance(obj, (list, tuple)):
             obj = bar_encode(obj)
         if obj is None:
@@ -1904,7 +1911,7 @@ class BaseAdapter(ConnectionPool):
         for (i,row) in enumerate(rows):
             new_row = Row()
             for (j,colname) in enumerate(colnames):
-                value = row[j]                
+                value = row[j]
                 tmp = tmps[j]
                 if tmp:
                     (tablename,fieldname,table,field,ft) = tmp
@@ -1918,14 +1925,14 @@ class BaseAdapter(ConnectionPool):
                     if field.filter_out:
                         value = field.filter_out(value)
                     colset[fieldname] = value
-                                        
+
                     # for backward compatibility
                     if ft=='id' and fieldname!='id' and \
                             not 'id' in table.fields:
                         colset['id'] = value
 
                     if ft == 'id' and not cacheable:
-                        # temporary hack to deal with 
+                        # temporary hack to deal with
                         # GoogleDatastoreAdapter
                         # references
                         if isinstance(self, GoogleDatastoreAdapter):
@@ -2025,8 +2032,11 @@ class SQLiteAdapter(BaseAdapter):
             'second': (17, 19),
             }
         try:
-            (i, j) = table[lookup]
-            return int(s[i:j])
+            if lookup != 'epoch':
+                (i, j) = table[lookup]
+                return int(s[i:j])
+            else:
+                return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
         except:
             return None
 
@@ -2278,6 +2288,9 @@ class MySQLAdapter(BaseAdapter):
         return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
                                         parameters[0], parameters[1])
 
+    def EPOCH(self, first):
+        return "UNIX_TIMESTAMP(%s)" % self.expand(first)
+
     def _drop(self,table,mode):
         # breaks db integrity but without this mysql does not drop table
         return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,
@@ -2641,9 +2654,9 @@ class NewPostgreSQLAdapter(PostgreSQLAdapter):
             elif not isinstance(obj, (list, tuple)):
                 obj = [obj]
             if field_is_type('list:string'):
-                obj = [str(item) for item in obj]
+                obj = map(str,obj)
             else:
-                obj = [int(item) for item in obj]
+                obj = map(int,obj)
             return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
         return BaseAdapter.represent(self, obj, fieldtype)
 
@@ -2982,7 +2995,7 @@ class MSSQLAdapter(BaseAdapter):
             # (in the form of arg1=value1&arg2=value2&...)
             # Default values (drivers like FreeTDS insist on uppercase parameter keys)
             argsdict = { 'DRIVER':'{SQL Server}' }
-            urlargs = m.group('urlargs') or ''            
+            urlargs = m.group('urlargs') or ''
             for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
                 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
             urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
@@ -3007,6 +3020,9 @@ class MSSQLAdapter(BaseAdapter):
             return rows[minimum:]
         return rows[minimum:maximum]
 
+    def EPOCH(self, first):
+        return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
+
     # GIS functions
 
     # No STAsGeoJSON in MSSQL
@@ -3154,7 +3170,7 @@ class SybaseAdapter(MSSQLAdapter):
                 logger.error('NdGpatch error')
                 raise e
         else:
-            m = self.REGEX_URI.match(uri)            
+            m = self.REGEX_URI.match(uri)
             if not m:
                 raise SyntaxError, \
                     "Invalid URI string in DAL: %s" % self.uri
@@ -3226,6 +3242,9 @@ class FireBirdAdapter(BaseAdapter):
     def RANDOM(self):
         return 'RAND()'
 
+    def EPOCH(self, first):
+        return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
+
     def NOT_NULL(self,default,field_type):
         return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
 
@@ -3250,7 +3269,7 @@ class FireBirdAdapter(BaseAdapter):
 
     def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
                  credential_decoder=IDENTITY, driver_args={},
-                 adapter_args={}): 
+                 adapter_args={}):
         self.db = db
         self.dbengine = "firebird"
         self.uri = uri
@@ -3281,7 +3300,7 @@ class FireBirdAdapter(BaseAdapter):
                            user = credential_decoder(user),
                            password = credential_decoder(password),
                            charset = charset)
-        
+
         def connect(driver_args=driver_args):
             return self.driver.connect(**driver_args)
         self.pool_connection(connect)
@@ -3341,7 +3360,7 @@ class FireBirdEmbeddedAdapter(FireBirdAdapter):
                            user=credential_decoder(user),
                            password=credential_decoder(password),
                            charset=charset)
-        
+
         def connect(driver_args=driver_args):
             return self.driver.connect(**driver_args)
         self.pool_connection(connect)
@@ -4008,8 +4027,8 @@ class GoogleSQLAdapter(UseDatabaseStoredFile,MySQLAdapter):
         self.execute("SET FOREIGN_KEY_CHECKS=1;")
         self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
 
-    def execute(self,a):
-        return self.log_execute(a.decode('utf8'))
+    def execute(self, command, *a, **b):
+        return self.log_execute(command.decode('utf8'), *a, **b)
 
 class NoSQLAdapter(BaseAdapter):
     can_select_for_update = False
@@ -4347,7 +4366,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
         if first.type != 'id':
             return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
         else:
-            second = [Key.from_path(first._tablename, i) for i in second]
+            second = [Key.from_path(first._tablename, int(i)) for i in second]
             return [GAEF(first.name,'in',second,lambda a,b:a in b)]
 
     def CONTAINS(self,first,second):
@@ -4389,7 +4408,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
             tablename = self.get_table(query)
         elif fields:
             tablename = fields[0].tablename
-            query = fields[0].table._id>0
+            query = fields[0].table._id != None
         else:
             raise SyntaxError, "Unable to determine a tablename"
 
@@ -4414,7 +4433,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
                 else:
                     projection.append(f.name)
 
-        # projection's can't include 'id'. 
+        # projection's can't include 'id'.
         # it will be added to the result later
         query_projection = [
             p for p in projection if \
@@ -4678,9 +4697,7 @@ class CouchDBAdapter(NoSQLAdapter):
     def _select(self,query,fields,attributes):
         if not isinstance(query,Query):
             raise SyntaxError, "Not Supported"
-        for key in set(attributes.keys())-set(('orderby','groupby','limitby',
-                                               'required','cache','left',
-                                               'distinct', 'having', 'processor')):
+        for key in set(attributes.keys())-SELECT_ARGS:
             raise SyntaxError, 'invalid select attribute: %s' % key
         new_fields=[]
         for item in fields:
@@ -4912,7 +4929,7 @@ class MongoDBAdapter(NoSQLAdapter):
             print "in expand and this is a query"
             # any query using 'id':=
             #   set name as _id (as per pymongo/mongodb primary key)
-            # convert second arg to an objectid field 
+            # convert second arg to an objectid field
             # (if its not already)
             # if second arg is 0 convert to objectid
             if isinstance(expression.first,Field) and \
@@ -4922,9 +4939,9 @@ class MongoDBAdapter(NoSQLAdapter):
                         not isinstance(expression.second,ObjectId):
                     if isinstance(expression.second,int):
                         try:
-                            # Because the reference field is by default 
-                            # an integer and therefore this must be an 
-                            # integer to be able to work with other 
+                            # Because the reference field is by default
+                            # an integer and therefore this must be an
+                            # integer to be able to work with other
                             # databases
                             expression.second = ObjectId(("%X" % expression.second))
                         except:
@@ -5212,27 +5229,27 @@ class MongoDBAdapter(NoSQLAdapter):
         return result
 
     def ADD(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '%s + %s' % (self.expand(first), self.expand(second, first.type))
 
     def SUB(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s - %s)' % (self.expand(first), self.expand(second, first.type))
 
     def MUL(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s * %s)' % (self.expand(first), self.expand(second, first.type))
 
     def DIV(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s / %s)' % (self.expand(first), self.expand(second, first.type))
 
     def MOD(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type))
 
     def AS(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '%s AS %s' % (self.expand(first), second)
 
     #We could implement an option that simulates a full featured SQL database. But I think the option should be set explicit or implemented as another library.
@@ -5344,31 +5361,31 @@ class MongoDBAdapter(NoSQLAdapter):
 
     #TODO javascript has math
     def ADD(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '%s + %s' % (self.expand(first), self.expand(second, first.type))
 
     #TODO javascript has math
     def SUB(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s - %s)' % (self.expand(first), self.expand(second, first.type))
 
     #TODO javascript has math
     def MUL(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s * %s)' % (self.expand(first), self.expand(second, first.type))
         #TODO javascript has math
 
     def DIV(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s / %s)' % (self.expand(first), self.expand(second, first.type))
     #TODO javascript has math
     def MOD(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type))
 
     #TODO javascript can do this
     def AS(self, first, second):
-        raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
+        raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry"
         return '%s AS %s' % (self.expand(first), second)
 
     #We could implement an option that simulates a full featured SQL database. But I think the option should be set explicit or implemented as another library.
@@ -5487,13 +5504,15 @@ class IMAPAdapter(NoSQLAdapter):
     # directly with set.update(deleted=True)
 
 
-    # This objects give access
+    # This object give access
     # to the adapter auto mailbox
     # mapped names (which native
     # mailbox has what table name)
 
-    db.mailboxes <dict> # tablename, server native name
-    db.mailbox_names <dict> # server native name, tablename
+    db.mailboxes <dict> # tablename, server native name pairs
+
+    # To retrieve a table native mailbox name use:
+    db.<table>.mailbox
 
     """
 
@@ -5684,7 +5703,7 @@ class IMAPAdapter(NoSQLAdapter):
             year = int(date_list[2])
             month = months.index(date_list[1])
             day = int(date_list[0])
-            hms = [int(value) for value in date_list[3].split(":")]
+            hms = map(int, date_list[3].split(":"))
             return datetime.datetime(year, month, day,
                                      hms[0], hms[1], hms[2]) + add
         elif isinstance(date, (datetime.datetime, datetime.date)):
@@ -5797,7 +5816,14 @@ class IMAPAdapter(NoSQLAdapter):
                             Field("attachments", "list:string", writable=False, readable=False),
                             )
 
-        return self.connection.mailbox_names
+            # Set a special _mailbox attribute for storing
+            # native mailbox names
+            self.db[mailbox_name].mailbox = \
+                self.connection.mailbox_names[mailbox_name]
+
+        # Set the db instance mailbox collections
+        self.db.mailboxes = self.connection.mailbox_names
+        return self.db.mailboxes
 
     def create_table(self, *args, **kwargs):
         # not implemented
@@ -6210,7 +6236,11 @@ class IMAPAdapter(NoSQLAdapter):
             raise Exception("Operation not supported")
         return result
 
-    def NE(self, first, second):
+    def NE(self, first, second=None):
+        if (second is None) and isinstance(first, Field):
+            # All records special table query
+            if first.type == "id":
+                return self.GE(first, 1)
         result = self.NOT(self.EQ(first, second))
         result =  result.replace("NOT NOT", "").strip()
         return result
@@ -6361,11 +6391,9 @@ def sqlhtml_validators(field):
             refs = None
             db, id = r._db, r._id
             if isinstance(db._adapter, GoogleDatastoreAdapter):
-                for i in xrange(0, len(ids), 30):
-                    if not refs:
-                        refs = db(id.belongs(ids[i:i+30])).select(id)
-                    else:
-                        refs = refs&db(id.belongs(ids[i:i+30])).select(id)
+                def count(values): return db(id.belongs(values)).select(id)
+                rx = range(0, len(ids), 30)
+                refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
             else:
                 refs = db(id.belongs(ids)).select(id)
             return (refs and ', '.join(str(f(r,x.id)) for x in refs) or '')
@@ -6455,7 +6483,7 @@ class Row(object):
 
     def values(self):
         return self.__dict__.values()
-    
+
     def __iter__(self):
         return self.__dict__.__iter__()
 
@@ -6648,7 +6676,7 @@ class DAL(object):
        db.define_table('tablename', Field('fieldname1'),
                                     Field('fieldname2'))
     """
-    
+
     @staticmethod
     def set_folder(folder):
         """
@@ -7133,7 +7161,11 @@ def index():
         return table
 
     def __contains__(self, tablename):
-        return tablename in self.tables
+        try:
+            return tablename in self.tables
+        except AttributeError:
+            # The instance has no .tables attribute yet
+            return False
 
     def get(self,key,default):
         return self.__dict__.get(key,default)
@@ -7171,7 +7203,7 @@ def index():
 
     def __call__(self, query=None, ignore_common_filters=None):
         if isinstance(query,Table):
-            query = query._id>0
+            query = query._id != None
         elif isinstance(query,Field):
             query = query!=None
         return Set(self, query, ignore_common_filters=ignore_common_filters)
@@ -7211,7 +7243,7 @@ def index():
         Added 2012-08-24 "fields" and "colnames" optional arguments. If either
         is provided, the results cursor returned by the DB driver will be
         converted to a DAL Rows object using the db._adapter.parse() method.
-        
+
         The "fields" argument is a list of DAL Field objects that match the
         fields returned from the DB. The Field objects should be part of one or
         more Table objects defined on the DAL object. The "fields" list can
@@ -7224,14 +7256,14 @@ def index():
         can be specified as a list of field names in tablename.fieldname format.
         Again, these should represent tables and fields defined on the DAL
         object.
-        
+
         It is also possible to specify both "fields" and the associated
         "colnames". In that case, "fields" can also include DAL Expression
         objects in addition to Field objects. For Field objects in "fields",
         the associated "colnames" must still be in tablename.fieldname format.
         For Expression objects in "fields", the associated "colnames" can
         be any arbitrary labels.
-        
+
         Note, the DAL Table objects referred to by "fields" or "colnames" can
         be dummy tables and do not have to represent any real tables in the
         database. Also, note that the "fields" and "colnames" must be in the
@@ -7390,7 +7422,7 @@ class Table(object):
         db.users.insert(name='me') # print db.users._insert(...) to see SQL
         db.users.drop()
     """
-    
+
     def __init__(
         self,
         db,
@@ -7472,13 +7504,15 @@ class Table(object):
         fields = list(fields)
 
         if db and db._adapter.uploads_in_blob==True:
+            uploadfields = [f.name for f in fields if f.type=='blob']
             for field in fields:
+                fn = field.uploadfield
                 if isinstance(field, Field) and field.type == 'upload'\
-                        and field.uploadfield is True:
-                    tmp = field.uploadfield = '%s_blob' % field.name
-        if isinstance(field.uploadfield,str) and \
-                not [f for f in fields if f.name==field.uploadfield]:
-            fields.append(Field(field.uploadfield,'blob',default=''))
+                        and fn is True:
+                    fn = field.uploadfield = '%s_blob' % field.name
+                if isinstance(fn,str) and not fn in uploadfields:
+                    fields.append(Field(fn,'blob',default='',
+                                        writable=False,readable=False))
 
         lower_fieldnames = set()
         reserved = dir(Table) + ['fields']
@@ -8073,6 +8107,10 @@ class Expression(object):
         db = self.db
         return Expression(db, db._adapter.EXTRACT, self, 'second', 'integer')
 
+    def epoch(self):
+        db = self.db
+        return Expression(db, db._adapter.EPOCH, self, None, 'integer')
+
     def __getslice__(self, start, stop):
         db = self.db
         if start < 0:
@@ -8213,7 +8251,7 @@ class Expression(object):
 
     def st_asgeojson(self, precision=15, options=0, version=1):
         return Expression(self.db, self.db._adapter.ST_ASGEOJSON, self,
-                          dict(precision=precision, options=options, 
+                          dict(precision=precision, options=options,
                                version=version), 'dict')
 
     def st_astext(self):
@@ -8462,7 +8500,7 @@ class Field(Expression):
         elif not filename:
             filename = file.name
         filename = os.path.basename(filename.replace('/', os.sep)\
-                                        .replace('\\', os.sep))        
+                                        .replace('\\', os.sep))
         m = REGEX_STORE_PATTERN.search(filename)
         extension = m and m.group('e') or 'txt'
         uuid_key = web2py_uuid().replace('-', '')[-16:]
@@ -8700,7 +8738,7 @@ class Set(object):
 
     def __call__(self, query, ignore_common_filters=False):
         if isinstance(query,Table):
-            query = query._id>0
+            query = query._id != None
         elif isinstance(query,str):
             query = Expression(self.db,query)
         elif isinstance(query,Field):
@@ -8717,7 +8755,12 @@ class Set(object):
 
     def _select(self, *fields, **attributes):
         adapter = self.db._adapter
-        fields = adapter.expand_all(fields, adapter.tables(self.query))
+        tablenames = adapter.tables(self.query,
+                                    attributes.get('join',None),
+                                    attributes.get('left',None),
+                                    attributes.get('orderby',None),
+                                    attributes.get('groupby',None))
+        fields = adapter.expand_all(fields, tablenames)
         return adapter._select(self.query,fields,attributes)
 
     def _delete(self):
@@ -8745,14 +8788,17 @@ class Set(object):
                 key,
                 (lambda self=self,distinct=distinct: \
                   db._adapter.count(self.query,distinct)),
-                time_expire)                
+                time_expire)
         return db._adapter.count(self.query,distinct)
 
     def select(self, *fields, **attributes):
-        if self.query is None:# and fields[0]._table._common_filter != None:
-            return self(fields[0]._table).select(*fields,**attributes)
         adapter = self.db._adapter
-        fields = adapter.expand_all(fields, adapter.tables(self.query))
+        tablenames = adapter.tables(self.query,
+                                    attributes.get('join',None),
+                                    attributes.get('left',None),
+                                    attributes.get('orderby',None),
+                                    attributes.get('groupby',None))
+        fields = adapter.expand_all(fields, tablenames)
         return adapter.select(self.query,fields,attributes)
 
     def nested_select(self,*fields,**attributes):
@@ -8860,7 +8906,7 @@ class RecordUpdater(object):
         self.colset, self.table, self.id = colset, table, id
 
     def __call__(self, **fields):
-        colset, table, id = self.colset, self.table, self.id 
+        colset, table, id = self.colset, self.table, self.id
         newfields = fields or dict(colset)
         for fieldname in newfields.keys():
             if not fieldname in table.fields or table[fieldname].type=='id':
@@ -9182,7 +9228,7 @@ class Rows(object):
     def xml(self,strict=False,row_name='row',rows_name='rows'):
         """
         serializes the table using sqlhtml.SQLTABLE (if present)
-        """        
+        """
         if strict:
             ncols = len(self.colnames)
             def f(row,field,indent='  '):

gluon/globals.py

@@ -51,6 +51,7 @@ except ImportError:
     have_minify = False
 
 regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$')
+regex_nopasswd = re.compile('(?<=\:)([^:@/]+)(?=@.+)')
 
 __all__ = ['Request', 'Response', 'Session']
 
@@ -230,7 +231,6 @@ class Response(Storage):
 
     def include_files(self):
 
-
         """
         Caching method for writing out files.
         By default, caches in ram for 5 minutes. To change,
@@ -244,8 +244,9 @@ class Response(Storage):
             if not item in files: files.append(item)
         if have_minify and (self.optimize_css or self.optimize_js):
             # cache for 5 minutes by default
+            key = hashlib.md5(repr(files)).hexdigest()
             cache = self.cache_includes or (current.cache.ram, 60*5)
-            def call_minify():
+            def call_minify(files=files):
                 return minify.minify(files,
                                      URL('static','temp'),
                                      current.request.folder,
@@ -253,7 +254,7 @@ class Response(Storage):
                                      self.optimize_js)
             if cache:
                 cache_model, time_expire = cache
-                files = cache_model('response.files.minified',
+                files = cache_model('response.files.minified/'+key,
                                     call_minify,
                                     time_expire)
             else:
@@ -278,7 +279,7 @@ class Response(Storage):
         chunk_size = DEFAULT_CHUNK_SIZE,
         request=None,
         attachment=False,
-        filename=None
+        filename=None,
         ):
         """
         if a controller function::
@@ -410,11 +411,13 @@ class Response(Storage):
             dbstats = [TABLE(*[TR(PRE(row[0]),'%.2fms' % (row[1]*1000)) \
                                    for row in i.db._timings]) \
                            for i in thread.instances]
-            dbtables = dict([(i.uri, {'defined': sorted(list(set(i.db.tables) - 
-                                                 set(i.db._LAZY_TABLES.keys()))) or
-                                                 '[no defined tables]',
-                                      'lazy': sorted(i.db._LAZY_TABLES.keys()) or
-                                              '[no lazy tables]'})
+            dbtables = dict([(regex_nopasswd.sub('******',i.uri), 
+                              {'defined': 
+                               sorted(list(set(i.db.tables) - 
+                                           set(i.db._LAZY_TABLES.keys()))) or
+                               '[no defined tables]',
+                               'lazy': sorted(i.db._LAZY_TABLES.keys()) or
+                               '[no lazy tables]'})
                              for i in thread.instances])
         else:
             dbstats = [] # if no db or on GAE

gluon/html.py

@@ -243,7 +243,8 @@ def URL(
         elif a and c and not f: (c,f,a)=(a,c,f)
         from globals import current
         if hasattr(current,'request'):
-            r = current.request
+            r = current.request    
+
     if r:
         application = r.application
         controller = r.controller
@@ -296,10 +297,10 @@ def URL(
     if other.endswith('/'):
         other += '/'    # add trailing slash to make last trailing empty arg explicit
 
-    if '_signature' in vars:
-        vars.pop('_signature')
     list_vars = []
     for (key, vals) in sorted(vars.items()):
+        if key == '_signature':
+            continue
         if not isinstance(vals, (list, tuple)):
             vals = [vals]
         for val in vals:
@@ -347,7 +348,8 @@ def URL(
 
     if regex_crlf.search(join([application, controller, function, other])):
         raise SyntaxError, 'CRLF Injection Detected'
-    url = url_out(r, env, application, controller, function,
+
+    url = url_out(r,env, application, controller, function,
                   args, other, scheme, host, port)
     return url
 
@@ -1743,7 +1745,7 @@ class INPUT(DIV):
         elif not t == 'submit':
             if value is None:
                 self['value'] = _value
-            else:
+            elif not isinstance(value,list):
                 self['_value'] = value
 
     def xml(self):
@@ -2109,8 +2111,10 @@ class FORM(DIV):
     REDIRECT_JS = "window.location='%s';return false"
 
     def add_button(self,value,url,_class=None):
-        self[0][-1][1].append(INPUT(_type="button",_value=value,_class=_class,
-                                    _onclick=self.REDIRECT_JS % url))
+        submit = self.element('input[type=submit]')
+        submit.parent.append(
+            INPUT(_type="button",_value=value,_class=_class,
+                  _onclick=self.REDIRECT_JS % url))
 
 
 
@@ -2227,6 +2231,7 @@ class MENU(DIV):
     def __init__(self, data, **args):
         self.data = data
         self.attributes = args
+        self.components = []
         if not '_class' in self.attributes:
             self['_class'] = 'web2py-menu web2py-menu-vertical'
         if not 'ul_class' in self.attributes:

gluon/http.py

@@ -89,21 +89,21 @@ class HTTP(BaseException):
             status = str(status)
             if not regex_status.match(status):
                 status = '500 %s' % (defined_status[500])
-        if not 'Content-Type' in headers:
-            headers['Content-Type'] = 'text/html; charset=UTF-8'
+        headers.setdefault('Content-Type','text/html; charset=UTF-8')
         body = self.body
         if status[:1] == '4':
             if not body:
                 body = status
             if isinstance(body, str):
-                if len(body)<512 and headers['Content-Type'].startswith('text/html'):
+                if len(body)<512 and \
+                        headers['Content-Type'].startswith('text/html'):
                     body += '<!-- %s //-->' % ('x'*512) ### trick IE
                 headers['Content-Length'] = len(body)
         rheaders = []
         for k, v in headers.iteritems():
             if isinstance(v, list):
                 rheaders += [(k, str(item)) for item in v]
-            else:
+            elif not v is None:
                 rheaders.append((k, str(v)))
         responder(status, rheaders)
         if env.get('request_method','')=='HEAD':

gluon/languages.py

@@ -29,15 +29,16 @@ from string import maketrans
 
 __all__ = ['translator', 'findT', 'update_all_languages']
 
-ospath = os.path
 ostat = os.stat
-osep = os.sep
+oslistdir = os.listdir
 pjoin = os.path.join
+pexists = os.path.exists
 pdirname = os.path.dirname
 isdir = os.path.isdir
 is_gae = settings.global_settings.web2py_runtime_gae
 
 DEFAULT_LANGUAGE = 'en'
+DEFAULT_LANGUAGE_NAME = 'English'
 
 # DEFAULT PLURAL-FORMS RULES:
 # language doesn't use plural forms
@@ -45,23 +46,7 @@ DEFAULT_NPLURALS = 1
 # only one singular/plural form is used
 DEFAULT_GET_PLURAL_ID = lambda n: 0
 # word is unchangeable
-DEFAULT_CONSTRUCTOR_PLURAL_FORM = lambda word, plural_id: word
-
-def safe_eval(text):
-    if text.strip():
-        try:
-            import ast
-            return ast.literal_eval(text)
-        except ImportError:
-            return eval(text,{},{})
-    return None
-
-# used as default filter in translator.M()
-def markmin_aux(m):
-    return '{%s}' % markmin_escape(m.group('s'))
-def markmin(s):
-    return render(regex_param.sub(markmin_aux,s),
-                  sep='br', autolinks=None, id_prefix='')
+DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word
 
 NUMBERS = (int,long,float)
 
@@ -75,14 +60,30 @@ regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL)
 regex_param=re.compile(r'{(?P<s>.+?)}')
 
 # pattern for a valid accept_language
-
 regex_language = \
-    re.compile('^([a-zA-Z]{2})(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$')
-regex_langfile = re.compile('^[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
+    re.compile('([a-z]{2}(?:\-[a-z]{2})?(?:\-[a-z]{2})?)(?:[,;]|$)')
+regex_langfile = re.compile('^[a-z]{2}(-[a-z]{2})?\.py$')
 regex_backslash = re.compile(r"\\([\\{}%])")
 regex_plural = re.compile('%({.+?})')
 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)\)}$')  # %%{word(varname or number)}
 regex_plural_tuple = re.compile('^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word}
+regex_plural_file = re.compile('^plural-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
+
+def safe_eval(text):
+    if text.strip():
+        try:
+            import ast
+            return ast.literal_eval(text)
+        except ImportError:
+            return eval(text,{},{})
+    return None
+
+# used as default filter in translator.M()
+def markmin(s):
+    def markmin_aux(m):
+        return '{%s}' % markmin_escape(m.group('s'))
+    return render(regex_param.sub(markmin_aux,s),
+                  sep='br', autolinks=None, id_prefix='')
 
 # UTF8 helper functions
 def upper_fun(s):
@@ -90,7 +91,7 @@ def upper_fun(s):
 def title_fun(s):
     return unicode(s,'utf-8').title().encode('utf-8')
 def cap_fun(s):
-    return lambda s: unicode(s,'utf-8').capitalize().encode('utf-8')
+    return unicode(s,'utf-8').capitalize().encode('utf-8')
 ttab_in  = maketrans("\\%{}", '\x1c\x1d\x1e\x1f')
 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
 
@@ -132,38 +133,6 @@ def clear_cache(filename):
     finally:
         lock.release()
 
-def lang_sampling(lang_tuple, langlist):
-    """
-    search *lang_tuple* in *langlist*
-
-    Args:
-        lang_tuple (tuple of strings): ('aa'[[,'-bb'],'-cc'])
-        langlist   (list of strings): [available languages]
-
-    Returns:
-        language from langlist or None
-    """
-    # step 1:
-    # compare "aa-bb-cc" | "aa-bb" | "aa" from lang_tuple
-    # with strings from langlist. Return appropriate string
-    # from langlist:
-    tries = range(len(lang_tuple),0,-1)
-    for i in tries:
-        language="".join(lang_tuple[:i])
-        if language in langlist:
-            return language
-    # step 2 (if not found in step 1):
-    # compare "aa-bb-cc" | "aa-bb" | "aa" from lang_tuple
-    # with left part of a string from langlist. Return
-    # appropriate string from langlist
-    for i in tries:
-        lang="".join(lang_tuple[:i])
-        for language in langlist:
-            if language.startswith(lang):
-                return language
-    return None
-
-
 def read_dict_aux(filename):
     lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
     clear_cache(filename)
@@ -175,57 +144,24 @@ def read_dict_aux(filename):
         return {'__corrupted__':status}
 
 def read_dict(filename):
-    """
-    return dictionary with translation messages
+    """ return dictionary with translation messages
     """
     return getcfs('lang:'+filename, filename,
                 lambda: read_dict_aux(filename))
 
 
-def get_lang_info(lang, langdir):
-    """
-    retrieve lang information from *langdir*/*lang*.py file.
-    Read few strings from lang.py file until keys !langname!,
-    !langcode! or keys greater then '!*' were found
-
-    args:
-        lang (str): lang-code or 'default'
-        langdir (str): path to 'languages' directory in web2py app dir
-
-    returns:
-        tuple(langcode, langname, langfile_mtime)
-        e.g.: ('en', 'English', 1338549043.0)
-    """
-    filename = ospath.join(langdir, lang+'.py')
-    d = read_dict(filename)    
-    langcode = d.get('!langcode!',DEFAULT_LANGUAGE)
-    langname = d.get('!langname!',langcode)
-    return (langcode, langname or langcode, ostat(filename).st_mtime)
-
-def read_possible_languages(appdir):
-    langs = {}
-    # scan languages directory for langfiles:
-    langdir = ospath.join(appdir,'languages')
-    for filename in os.listdir(langdir):
-        if regex_langfile.match(filename) or filename=='default.py':
-            lang = filename[:-3]
-            langs[lang] = get_lang_info(lang, langdir)
-    if not 'en' in langs:
-        # if default.py is not found, add default value:
-        langs['en'] = ('en', 'English', 0)
-    return langs
-
-def read_possible_plurals():
+def read_possible_plural_rules():
     """
     create list of all possible plural rules files
-    result is cached to increase speed
+    result is cached in PLURAL_RULES dictionary to increase speed
     """
-    plurals = {}
     try:
-        import contrib.plural_rules as package
+        import gluon.contrib.plural_rules as package
+        plurals = {}
         for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
             if len(modname)==2:
-                module = __import__(package.__name__+'.'+modname)
+                module = __import__(package.__name__+'.'+modname,
+                                    fromlist=[modname])
                 lang = modname
                 pname = modname+'.py'
                 nplurals = getattr(module,'nplurals', DEFAULT_NPLURALS)
@@ -234,19 +170,84 @@ def read_possible_plurals():
                     DEFAULT_GET_PLURAL_ID)
                 construct_plural_form = getattr(
                     module,'construct_plural_form',
-                    DEFAULT_CONSTRUCTOR_PLURAL_FORM)
+                    DEFAULT_CONSTRUCT_PLURAL_FORM)
                 plurals[lang] = (lang, nplurals, get_plural_id,
-                                 construct_plural_form, pname)
-    except ImportError:
-        logging.warn('Unable to import plural rules')
-    plurals['default'] = ('default',
-                          DEFAULT_NPLURALS,
-                          DEFAULT_GET_PLURAL_ID,
-                          DEFAULT_CONSTRUCTOR_PLURAL_FORM,
-                          None)
+                                 construct_plural_form)
+    except ImportError, e:
+        logging.warn('Unable to import plural rules: %s' % e)
     return plurals
 
-PLURAL_RULES = read_possible_plurals()
+PLURAL_RULES = read_possible_plural_rules()
+
+def read_possible_languages_aux(langdir):
+    def get_lang_struct(lang, langcode, langname, langfile_mtime):
+        if lang == 'default':
+            real_lang = langcode.lower()
+        else:
+            real_lang = lang
+        (prules_langcode,
+         nplurals,
+         get_plural_id,
+         construct_plural_form
+        ) = PLURAL_RULES.get(real_lang[:2],('default',
+                                            DEFAULT_NPLURALS,
+                                            DEFAULT_GET_PLURAL_ID,
+                                            DEFAULT_CONSTRUCT_PLURAL_FORM))
+        if prules_langcode != 'default':
+            (pluraldict_fname,
+             pluraldict_mtime) = plurals.get(real_lang,
+                                    plurals.get(real_lang[:2],
+                                        ('plural-%s.py'%real_lang,0)))
+        else:
+            pluraldict_fname = None
+            pluraldict_mtime = 0
+        return (langcode,        # language code from !langcode!
+                langname,        # language name in national spelling from !langname!
+                langfile_mtime,  # m_time of language file
+                pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist)
+                pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist
+                prules_langcode, # code of plural rules language or 'default'
+                nplurals,        # nplurals for current language
+                get_plural_id,   # get_plural_id() for current language
+                construct_plural_form) # construct_plural_form() for current language
+ 
+    plurals = {}
+    flist = oslistdir(langdir)
+    # scan languages directory for plural dict files:
+    for pname in flist:
+        if regex_plural_file.match(pname):
+            plurals[pname[7:-3]] = (pname,
+                ostat(pjoin(langdir,pname)).st_mtime)
+    langs = {}
+    # scan languages directory for langfiles:
+    for fname in flist:
+        if regex_langfile.match(fname) or fname == 'default.py':
+            fname_with_path = pjoin(langdir,fname)
+            d = read_dict(fname_with_path)    
+            lang = fname[:-3]
+            langcode = d.get('!langcode!', lang if lang != 'default'
+                                              else DEFAULT_LANGUAGE)
+            langname = d.get('!langname!',langcode)
+            langfile_mtime = ostat(fname_with_path).st_mtime
+            langs[lang] = get_lang_struct(lang, langcode,
+                                          langname, langfile_mtime)
+    if 'default' not in langs:
+        # if default.py is not found,
+        # add DEFAULT_LANGUAGE as default language:
+        langs['default'] = get_lang_struct('default', DEFAULT_LANGUAGE,
+                                           DEFAULT_LANGUAGE_NAME, 0)
+    deflang = langs['default']
+    deflangcode = deflang[0]
+    if deflangcode not in langs:
+        # create language from default.py:
+        langs[deflangcode] = deflang[:2]+(0,)+deflang[3:]
+                            
+    return langs
+
+def read_possible_languages(appdir):
+    langdir = pjoin(appdir,'languages')
+    return getcfs('langs:'+langdir, langdir,
+                   lambda: read_possible_languages_aux(langdir))
 
 def read_plural_dict_aux(filename):
     lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
@@ -401,42 +402,81 @@ class translator(object):
         T(\"Hello World\") # translates \"Hello World\" using the selected file
 
     notice 1: there is no need to force since, by default, T uses
-    http_accept_language to determine a translation file.
+       http_accept_language to determine a translation file.
     notice 2: 
        en and en-en are considered different languages!
     notice 3: 
        if language xx-yy is not found force() probes other similar
-    languages using such algorithm: 
+       languages using such algorithm: 
         xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py
     """
 
     def __init__(self, request):
         self.request = request
         self.folder = request.folder
-        self.langpath = ospath.join(self.folder,'languages')
-        self.filenames = set(os.listdir(self.langpath))
+        self.langpath = pjoin(self.folder, 'languages')
         self.http_accept_language = request.env.http_accept_language
-        # self.cache                        # filled in self.force()
-        # self.accepted_language = None     # filled in self.force()
-        # self.language_file = None         # filled in self.force()
-        # self.plural_language = None       # filled in self.force()
-        # self.nplurals = None              # filled in self.force()
-        # self.get_plural_id = None         # filled in self.force()
-        # self.construct_plural_form = None # filled in self.force()
-        # self.plural_rules_file = None     # filled in self.force()
-        # self.plural_file = None           # filled in self.force()
-        # self.plural_dict = None           # filled in self.force()
-        # self.plural_status = None         # filled in self.force()
-        self.requested_languages = \
-            self.force(self.http_accept_language)
+        # filled in self.force():
+        #------------------------
+        # self.cache
+        # self.accepted_language
+        # self.language_file
+        # self.plural_language
+        # self.nplurals
+        # self.get_plural_id
+        # self.construct_plural_form
+        # self.plural_file
+        # self.plural_dict
+        # self.requested_languages
+        #----------------------------------------
+        # filled in self.set_current_languages():
+        #----------------------------------------
+        # self.default_language_file
+        # self.default_t
+        # self.current_languages
+        self.set_current_languages()
         self.lazy = True
         self.otherTs = {}
         self.filter = markmin
         self.ftag = 'markmin'
 
+    def get_possible_languages_info(self, lang=None):
+        """
+        return info for selected language or dictionary with all
+            possible languages info from APP/languages/*.py
+        args:
+            *lang* (str): language
+        returns:
+            if *lang* is defined:
+               return tuple(langcode, langname, langfile_mtime,
+                            pluraldict_fname, pluraldict_mtime,
+                            prules_langcode, nplurals,
+                            get_plural_id, construct_plural_form)
+                            or None
+
+            if *lang* is NOT defined:
+               returns dictionary with all possible languages:
+            { langcode(from filename):
+                ( langcode,        # language code from !langcode!
+                  langname,        # language name in national spelling from !langname!
+                  langfile_mtime,  # m_time of language file
+                  pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist)
+                  pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist
+                  prules_langcode, # code of plural rules language or 'default'
+                  nplurals,        # nplurals for current language
+                  get_plural_id,   # get_plural_id() for current language
+                  construct_plural_form) # construct_plural_form() for current language
+            }
+        """
+        info = read_possible_languages(self.folder)
+        if lang: info = info.get(lang)
+        return info
+
     def get_possible_languages(self):
-        return [lang[:-3] for lang in self.filenames \
-                    if regex_langfile.match(lang)]
+        """ get list of all possible languages for current applications """
+        return list(set(self.current_languages +
+            [lang for lang in read_possible_languages(self.folder).iterkeys()
+                 if lang != 'default']))
 
     def set_current_languages(self, *languages):
         """
@@ -447,38 +487,30 @@ class translator(object):
         if len(languages) == 1 and isinstance(
             languages[0], (tuple, list)):
             languages = languages[0]
-        self.current_languages = languages
+        if not languages or languages[0] is None:
+            # set default language from default.py/DEFAULT_LANGUAGE 
+            pl_info = self.get_possible_languages_info('default')
+            if pl_info[2]==0: # langfile_mtime
+                # if languages/default.py is not found
+                self.default_language_file = self.langpath
+                self.default_t = {}
+                self.current_languages = [DEFAULT_LANGUAGE]
+            else:
+                self.default_language_file = pjoin(self.langpath,
+                                                   'default.py')
+                self.default_t = read_dict(self.default_language_file)
+                self.current_languages = [pl_info[0]] # !langcode!
+        else:
+            self.current_languages = list(languages)
         self.force(self.http_accept_language)
 
-    def set_plural(self, language):
-        """
-        initialize plural forms subsystem
-        invoked from self.force()
-        """
-        lang = language[:2] 
-        (self.plural_language,
-         self.nplurals,
-         self.get_plural_id,
-         self.construct_plural_form,
-         self.plural_filename
-         ) = PLURAL_RULES.get(language,PLURAL_RULES['default'])
-        for lang in (language, language[:5], language[:2]):
-            filename = 'plural-%s.py' % lang
-            if filename in self.filenames:
-                self.plural_file = ospath.join(self.langpath,filename)
-                self.plural_dict = read_plural_dict(self.plural_file)
-                break
-        else:
-            self.plural_file = None
-            self.plural_dict = {}
- 
+       
     def plural(self, word, n):
-        """
-        get plural form of word for number *n*
-            NOTE: *word* MUST be defined in current language
+        """ get plural form of word for number *n*
+            NOTE: *word" MUST be defined in current language
                   (T.accepted_language)
 
-            invoked from T()/M() in %%{} tag
+            invoked from T()/T.M() in %%{} tag
         args:
             word (str): word in singular
             n (numeric): number plural form created for
@@ -486,51 +518,34 @@ class translator(object):
         returns:
             (str): word in appropriate singular/plural form
         """
-        nplurals = self.nplurals
-        if int(n)==1:
+        if int(n) == 1:
             return word
         elif word:
             id = self.get_plural_id(abs(int(n)))
-            # id = 0 first plural form
-            # id = 1 second plural form
+            # id = 0 singular form
+            # id = 1 first plural form
+            # id = 2 second plural form
             # etc.
-            forms = self.plural_dict.get(word, [])
-            if len(forms)>=id:
-                # have this plural form
-                return forms[id-1]
-            else:
-                # guessing this plural form
-                forms += ['']*(nplurals-len(forms)-1)
-                form = self.construct_plural_form(word, id)
-                forms[id-1] = form
-                self.plural_dict[word] = forms
-                if self.plural_file and not is_gae:
-                    write_plural_dict(self.plural_file,
-                                      self.plural_dict)
-                return form
-
-    def get_possible_languages_info(self, lang=None):
-        """
-        return info for selected language or dictionary with all
-            possible languages info from APP/languages/*.py
-        args:
-            *lang* (str): language
-        returns:
-            if *lang* is defined:
-               return tuple(langcode, langname, langfile_mtime) or None
-
-            if *lang* is NOT defined:
-               returns dictionary with all possible languages:
-            { langcode(from filename): ( langcode(from !langcode! key),
-                                         langname(from !langname! key),
-                                         langfile_mtime ) }
-        """
-        info = read_possible_languages(self.folder)
-        if lang: info = info.get(lang)
-        return info
+            if id != 0:
+                forms = self.plural_dict.get(word, [])
+                if len(forms)>=id:
+                    # have this plural form:
+                    return forms[id-1]
+                else:
+                    # guessing this plural form
+                    forms += ['']*(self.nplurals-len(forms)-1)
+                    form = self.construct_plural_form(word, id)
+                    forms[id-1] = form
+                    self.plural_dict[word] = forms
+                    if self.plural_file and not is_gae:
+                        write_plural_dict(self.plural_file,
+                                          self.plural_dict)
+                    return form
+        return word
 
     def force(self, *languages):
         """
+
         select language(s) for translation
 
         if a list of languages is passed as a parameter,
@@ -541,41 +556,76 @@ class translator(object):
         default language will be selected if none
         of them matches possible_languages.
         """
+        pl_info = read_possible_languages(self.folder)
+        def set_plural(language):
+            """
+            initialize plural forms subsystem
+            """
+            lang_info = pl_info.get(language)
+            if lang_info:
+                (pname,
+                 pmtime,
+                 self.plural_language,
+                 self.nplurals,
+                 self.get_plural_id,
+                 self.construct_plural_form
+                ) = lang_info[3:]
+                pdict = {}
+                if pname:
+                    pname = pjoin(self.langpath, pname)
+                    if pmtime != 0:
+                        pdict = read_plural_dict(pname)
+                self.plural_file = pname
+                self.plural_dict = pdict
+            else:
+                self.plural_language = 'default'
+                self.nplurals = DEFAULT_NPLURALS
+                self.get_plural_id = DEFAULT_GET_PLURAL_ID
+                self.construct_plural_form = DEFAULT_CONSTRUCT_PLURAL_FORM
+                self.plural_file = None
+                self.plural_dict = {}
         language = ''
-        if isinstance(languages,str):
-            languages = regex_language.findall(languages.lower())
+        if len(languages)==1 and isinstance(languages[0],str):
+            languages = regex_language.findall(languages[0].lower())
         elif not languages or languages[0] is None:
             languages = []
-        for lang in languages:
-            if lang+'.py' in self.filenames:
-                language = lang
-                langfile = language+'.py'
-                break
-            elif len(lang)>5 and lang[:5]+'.py' in self.filenames:
-                language = lang[:5]
-                langfile = language+'.py'
-                break
-            elif len(lang)>2 and lang[:2]+'.py' in self.filenames:
-                language = lang[:2]
-                langfile = language+'.py'
-                break
-        else:
-            if 'default.py' in self.filenames:
-                language = DEFAULT_LANGUAGE
-                langfile = 'default.py'
-            else:
-                language = DEFAULT_LANGUAGE
-                langfile = None
-        self.accepted_language = language
-        if langfile:
-            self.language_file = ospath.join(self.langpath,langfile)
-            self.t = read_dict(self.language_file)
-        else:
-            self.language_file = None
-            self.t = {}
-        self.cache = global_language_cache.setdefault(
-            self.language_file,({},allocate_lock()))
-        self.set_plural(language)
+        self.requested_languages = languages = tuple(languages)
+        if languages:
+            all_languages = set(lang for lang in pl_info.iterkeys()
+                                if lang != 'default') \
+                                | set(self.current_languages)
+            for lang in languages:
+                # compare "aa-bb" | "aa" from *language* parameter
+                # with strings from langlist using such alghorythm:
+                # xx-yy.py -> xx.py -> xx*.py
+                lang5 = lang[:5]
+                if lang5 in all_languages:
+                    language = lang5
+                else:
+                    lang2 = lang[:2]
+                    if len(lang5)>2 and lang2 in all_languages:
+                        language = lang2
+                    else:
+                        for l in all_languages:
+                            if l[:2]==lang2:
+                                language = l
+                if language:
+                    if language in self.current_languages:
+                        break
+                    self.language_file = pjoin(self.langpath, language+'.py')
+                    self.t = read_dict(self.language_file)
+                    self.cache = global_language_cache.setdefault(
+                                         self.language_file,
+                                         ({},allocate_lock()))
+                    set_plural(language)
+                    self.accepted_language = language 
+                    return languages
+        self.accepted_language = language or self.current_languages[0]
+        self.language_file = self.default_language_file
+        self.cache = global_language_cache.setdefault(self.language_file,
+                                                      ({}, allocate_lock()))
+        self.t = self.default_t
+        set_plural(self.accepted_language)
         return languages
 
     def __call__(self, message, symbols={}, language=None, lazy=None):
@@ -659,22 +709,24 @@ class translator(object):
         the ## notation is ignored in multiline strings and strings that
         start with ##. this is to allow markmin syntax to be translated
         """
+        if isinstance(message, unicode): 
+            message = message.encode('utf8') 
+        if isinstance(prefix, unicode): 
+            prefix = prefix.encode('utf8') 
         key = prefix+message
         mt = self.t.get(key, None)
-        if mt is None:
-            # we did not find a translation
-            if message.find('##')>0 and not '\n' in message:
-                # remove comments
-                message = message.rsplit('##', 1)[0]
-            # guess translation same as original
-            self.t[key] = mt = message
-            # update language file for later translation
-            if self.language_file and not is_gae:
-                write_dict(self.language_file, self.t)
-            # fix backslash escaping
-            mt = regex_backslash.sub(
-                lambda m: m.group(1).translate(ttab_in), mt)
-        return mt
+        if mt is not None: return mt
+        # we did not find a translation
+        if message.find('##')>0 and not '\n' in message:
+            # remove comments
+            message = message.rsplit('##', 1)[0]
+        # guess translation same as original
+        self.t[key] = mt = self.default_t.get(key, message)
+        # update language file for latter translation
+        if self.language_file != self.default_language_file and not is_gae:
+            write_dict(self.language_file, self.t)
+        return regex_backslash.sub(
+            lambda m: m.group(1).translate(ttab_in), mt)
 
     def params_substitution(self, message, symbols):
         """
@@ -796,16 +848,16 @@ class translator(object):
             message = self.params_substitution(message, symbols)
         return message.translate(ttab_out)
 
-def findT(path, language='en'):
+def findT(path, language=DEFAULT_LANGUAGE):
     """
     must be run by the admin app
     """
-    lang_file = ospath.join(path, 'languages', language + '.py')
+    lang_file = pjoin(path, 'languages', language + '.py')
     sentences = read_dict(lang_file)
-    mp = ospath.join(path, 'models')
-    cp = ospath.join(path, 'controllers')
-    vp = ospath.join(path, 'views')
-    mop = ospath.join(path, 'modules')
+    mp = pjoin(path, 'models')
+    cp = pjoin(path, 'controllers')
+    vp = pjoin(path, 'views')
+    mop = pjoin(path, 'modules')
     for filename in \
             listdir(mp, '^.+\.py$', 0)+listdir(cp, '^.+\.py$', 0)\
             +listdir(vp, '^.+\.html$', 0)+listdir(mop, '^.+\.py$', 0):
@@ -827,10 +879,10 @@ def findT(path, language='en'):
                 sentences[message] = message
     if not '!langcode!' in sentences:
         sentences['!langcode!'] = (
-            'en' if language in ('default', 'en') else language)
+            DEFAULT_LANGUAGE if language in ('default', DEFAULT_LANGUAGE) else language)
     if not '!langname!' in sentences:
         sentences['!langname!'] = (
-            'English' if language in ('default', 'en')
+            DEFAULT_LANGUAGE_NAME if language in ('default', DEFAULT_LANGUAGE)
             else sentences['!langcode!'])
     write_dict(lang_file, sentences)
 
@@ -843,9 +895,10 @@ copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle)
 
 
 def update_all_languages(application_path):
-    path = ospath.join(application_path, 'languages/')
-    for language in listdir(path, regex_langfile):
-        findT(application_path, language[:-3])
+    path = pjoin(application_path, 'languages/')
+    for language in oslistdir(path):
+        if regex_langfile.match(language):
+            findT(application_path, language[:-3])
 
 
 if __name__ == '__main__':

gluon/main.py

@@ -226,6 +226,20 @@ def serve_controller(request, response, session):
     requests = ('requests' in globals()) and (requests+1) % 100 or 0
     if not requests: gc.collect()
     # end garbage collection logic
+
+    # ##################################################
+    # set default headers it not set
+    # ##################################################
+
+    default_headers = [
+        ('Content-Type', contenttype('.'+request.extension)),
+        ('Cache-Control','no-store, no-cache, must-revalidate, post-check=0, pre-check=0'),
+        ('Expires', time.strftime('%a, %d %b %Y %H:%M:%S GMT', 
+                                  time.gmtime())),
+        ('Pragma', 'no-cache')]
+    for key,value in default_headers:
+        response.headers.setdefault(key,value)
+
     raise HTTP(response.status, page, **response.headers)
 
 
@@ -488,19 +502,6 @@ def wsgibase(environ, responder):
                 if not env.web2py_disable_session:
                     session.connect(request, response)
 
-                # ##################################################
-                # set no-cache headers
-                # ##################################################
-                
-                headers = response.headers
-                headers['Content-Type'] = \
-                    contenttype('.'+request.extension)
-                headers['Cache-Control'] = \
-                    'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
-                headers['Expires'] = \
-                    time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime())
-                headers['Pragma'] = 'no-cache'
-
                 # ##################################################
                 # run controller
                 # ##################################################
@@ -509,14 +510,14 @@ def wsgibase(environ, responder):
                     import gluon.debug
                     # activate the debugger
                     gluon.debug.dbg.do_debug(mainpyfile=request.folder)
-
+                
                 serve_controller(request, response, session)
 
             except HTTP, http_response:
+
                 if static_file:
                     return http_response.to(responder,env=env)
                         
-
                 if request.body:
                     request.body.close()
 
@@ -544,29 +545,22 @@ def wsgibase(environ, responder):
                 # ##################################################
                     
                 session._try_store_on_disk(request, response)
+                
+                if request.cid:
+                    if response.flash:
+                        http_response.headers['web2py-component-flash'] = urllib2.quote(xmlescape(response.flash).replace('\n',''))
+                    if response.js:
+                        http_response.headers['web2py-component-command'] = response.js.replace('\n','')
 
                 # ##################################################
                 # store cookies in headers
                 # ##################################################
 
-                if request.cid:
-                    rheaders = http_response.headers
-                    if response.flash and \
-                            not 'web2py-component-flash' in rheaders:
-                        rheaders['web2py-component-flash'] = \
-                            urllib2.quote(xmlescape(response.flash)\
-                                              .replace('\n',''))
-                    if response.js and \
-                            not 'web2py-component-command' in rheaders:
-                        rheaders['web2py-component-command'] = \
-                            response.js.replace('\n','')
                 rcookies = response.cookies
-                if session._forget and \
-                        response.session_id_name in response.cookies:
+                if session._forget and response.session_id_name in rcookies:
                     del rcookies[response.session_id_name]
                 elif session._secure:
                     rcookies[response.session_id_name]['secure'] = True
-
                 http_response.cookies2headers(rcookies)
                 ticket=None
 

gluon/rewrite.py

@@ -162,10 +162,12 @@ def url_in(request, environ):
         return map_url_in(request, environ)
     return regex_url_in(request, environ)
 
-def url_out(request, env, application, controller, function, args, other, scheme, host, port):
+def url_out(request, env, application, controller, function, 
+            args, other, scheme, host, port):
     "assemble and rewrite outgoing URL"
     if routers:
-        acf = map_url_out(request, env, application, controller, function, args, other, scheme, host, port)
+        acf = map_url_out(request, env, application, controller,
+                          function, args, other, scheme, host, port)
         url = '%s%s' % (acf, other)
     else:
         url = '/%s/%s/%s%s' % (application, controller, function, other)
@@ -226,7 +228,8 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
                     # Rewrite routes_onerror path.
                     path_info = '/' + path_info.lstrip('/') # add leading '/' if missing
                     environ['PATH_INFO'] = path_info
-                    error_handling_path = url_in(request, environ)[1]['PATH_INFO']
+                    error_handling_path = \
+                        url_in(request, environ)[1]['PATH_INFO']
                     # Avoid infinite loop.
                     if error_handling_path != error_raising_path:
                         # wsgibase will be called recursively with the routes_onerror path.
@@ -895,7 +898,11 @@ class MapUrlIn(object):
         self.domain_controller = None
         self.domain_function = None
         arg0 = self.harg0
-        if (self.host, self.port) in base.domains:
+        if not base.exclusive_domain and base.applications and arg0 in base.applications:
+            self.application = arg0
+        elif not base.exclusive_domain and arg0 and not base.applications:
+            self.application = arg0
+        elif (self.host, self.port) in base.domains:
             (self.application, self.domain_controller, self.domain_function) = base.domains[(self.host, self.port)]
             self.env['domain_application'] = self.application
             self.env['domain_controller'] = self.domain_controller
@@ -1118,7 +1125,8 @@ class MapUrlIn(object):
 class MapUrlOut(object):
     "logic for mapping outgoing URLs"
 
-    def __init__(self, request, env, application, controller, function, args, other, scheme, host, port):
+    def __init__(self, request, env, application, controller,
+                 function, args, other, scheme, host, port):
         "initialize a map-out object"
         self.default_application = routers.BASE.default_application
         if application in routers:
@@ -1319,7 +1327,8 @@ def map_url_in(request, env, app=False):
     map.update_request()
     return (None, map.env)
 
-def map_url_out(request, env, application, controller, function, args, other, scheme, host, port):
+def map_url_out(request, env, application, controller,
+                function, args, other, scheme, host, port):
     '''
     supply /a/c/f (or /a/lang/c/f) portion of outgoing url
 
@@ -1345,7 +1354,8 @@ def map_url_out(request, env, application, controller, function, args, other, sc
 
     We assume that language names do not collide with a/c/f names.
     '''
-    map = MapUrlOut(request, env, application, controller, function, args, other, scheme, host, port)
+    map = MapUrlOut(request, env, application, controller,
+                    function, args, other, scheme, host, port)
     return map.acf()
 
 def get_effective_router(appname):

gluon/scheduler.py

@@ -88,7 +88,7 @@ except:
     from simplejson import loads, dumps
 
 
-from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB, IS_INT_IN_RANGE
+from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB, IS_INT_IN_RANGE, IS_DATETIME
 from gluon.utils import web2py_uuid
 
 
@@ -454,7 +454,8 @@ class Scheduler(MetaScheduler):
             Field('args','text',default='[]',requires=TYPE(list)),
             Field('vars','text',default='{}',requires=TYPE(dict)),
             Field('enabled','boolean',default=True),
-            Field('start_time','datetime',default=now, requires=IS_NOT_EMPTY()),
+            Field('start_time','datetime',default=now,
+                  requires = IS_DATETIME()),
             Field('next_run_time','datetime',default=now),
             Field('stop_time','datetime'),
             Field('repeats','integer',default=1,comment="0=unlimited",

gluon/sqlhtml.py

@@ -239,11 +239,14 @@ class ListWidget(StringWidget):
         _name = field.name
         if field.type=='list:integer': _class = 'integer'
         else: _class = 'string'
-        requires = field.requires if isinstance(field.requires, (IS_NOT_EMPTY, IS_LIST_OF)) else None
+        requires = field.requires if isinstance(
+            field.requires, (IS_NOT_EMPTY, IS_LIST_OF)) else None
         attributes['_style'] = 'list-style:none'
-        items=[LI(INPUT(_id=_id, _class=_class, _name=_name,
-                        value=v, hideerror=True, requires=requires),
-                  **attributes)  for v in value or ['']]
+        nvalue = value or ['']
+        items = [LI(INPUT(_id=_id, _class=_class, _name=_name,
+                          value=v, hideerror=k<len(nvalue)-1, 
+                          requires=requires),
+                    **attributes)  for (k,v) in enumerate(nvalue)]
         script=SCRIPT("""
 // from http://refactormycode.com/codes/694-expanding-input-list-using-jquery
 (function(){
@@ -1105,20 +1108,21 @@ class SQLFORM(FORM):
         self.components = [table]
 
     def createform(self, xfields):
-        if isinstance(self.formstyle, basestring):
-            if self.formstyle in SQLFORM.formstyles:
-                self.formstyle = SQLFORM.formstyles[self.formstyle]
+        formstyle = self.formstyle
+        if isinstance(formstyle, basestring):
+            if formstyle in SQLFORM.formstyles:
+                formstyle = SQLFORM.formstyles[formstyle]
             else:
                 raise RuntimeError, 'formstyle not found'
 
-        if callable(self.formstyle):
+        if callable(formstyle):
             # backward compatibility, 4 argument function is the old style
-            args, varargs, keywords, defaults = inspect.getargspec(self.formstyle)
+            args, varargs, keywords, defaults = inspect.getargspec(formstyle)
             if defaults and len(args) - len(defaults) == 4 or len(args) == 4:
                 table = TABLE()
                 for id,a,b,c in xfields:
                     raw_b = self.field_parent[id] = b
-                    newrows = self.formstyle(id,a,raw_b,c)
+                    newrows = formstyle(id,a,raw_b,c)
                     if type(newrows).__name__ != "tuple":
                         newrows = [newrows]
                     for newrow in newrows:
@@ -1126,7 +1130,7 @@ class SQLFORM(FORM):
             else:
                 for id,a,b,c in xfields:
                     self.field_parent[id] = b
-                table = self.formstyle(self, xfields)
+                table = formstyle(self, xfields)
         else:
             raise RuntimeError, 'formstyle not supported'
         return table
@@ -1409,7 +1413,7 @@ class SQLFORM(FORM):
         type(''): ('string', None),
         type(True): ('boolean', None),
         type(1): ('integer', IS_INT_IN_RANGE(-1e12,+1e12)),
-        type(1.0): ('double', IS_INT_IN_RANGE(-1e12,+1e12)),
+        type(1.0): ('double', IS_FLOAT_IN_RANGE()),
         type([]): ('list:string', None),
         type(datetime.date.today()): ('date', IS_DATE()),
         type(datetime.datetime.today()): ('datetime', IS_DATETIME())
@@ -1663,14 +1667,20 @@ class SQLFORM(FORM):
             return URL(**b)
 
         referrer = session.get('_web2py_grid_referrer_'+formname, url())
+        # if not user_signature every action is accessible
+        # else forbid access unless
+        # - url is based url
+        # - url has valid signature (vars are not signed, only path_info)
+        # = url does not contain 'create','delete','edit' (readonly)
         if user_signature:
-            if (args != request.args and user_signature and \
-                    not URL.verify(request,user_signature=user_signature)) or \
-                    (not (session.auth and session.auth.user) and \
-                         ('edit' in request.args or \
-                              'create' in request.args or \
-                              'delete' in request.args)):
-                session.flash = T('not authorized')
+            if not(
+                '/'.join(str(a) for a in args) == '/'.join(request.args) or
+                URL.verify(request,user_signature=user_signature,
+                           hash_vars=False) or not (
+                    'create' in request.args or
+                    'delete' in request.args or
+                    'edit' in request.args)):
+                session.flash = T('not authorized')                
                 redirect(referrer)
 
         def gridbutton(buttonclass='buttonadd', buttontext='Add',
@@ -1866,6 +1876,16 @@ class SQLFORM(FORM):
         session['_web2py_grid_referrer_'+formname] = url2(vars=request.vars)
         console = DIV(_class='web2py_console %(header)s %(cornertop)s' % ui)
         error = None
+        if create:
+            add = gridbutton(
+                    buttonclass='buttonadd',
+                    buttontext='Add',
+                    buttonurl=url(args=['new',tablename]))
+            if not searchable:
+                console.append(add)
+        else:
+            add = ''
+
         if searchable:
             sfields = reduce(lambda a,b:a+b,
                              [[f for f in t if f.readable] for t in tables])
@@ -1873,7 +1893,7 @@ class SQLFORM(FORM):
                 search_widget = search_widget[tablename]
             if search_widget=='default':
                 search_menu = SQLFORM.search_menu(sfields)
-                search_widget = lambda sfield, url: DIV(FORM(
+                search_widget = lambda sfield, url: CAT(add,FORM(
                     INPUT(_name='keywords',_value=request.vars.keywords,
                           _id='web2py_keywords',_onfocus="jQuery('#w2p_query_fields').change();jQuery('#w2p_query_panel').slideDown();"),
                     INPUT(_type='submit',_value=T('Search'),_class="btn"),
@@ -1893,11 +1913,6 @@ class SQLFORM(FORM):
                 error = T('Invalid query')
         else:
             subquery = None
-        if create:
-            console.append(gridbutton(
-                    buttonclass='buttonadd',
-                    buttontext='Add',
-                    buttonurl=url(args=['new',tablename])))
 
         if subquery:
             dbset = dbset(subquery)
@@ -2102,11 +2117,12 @@ class SQLFORM(FORM):
         else:
             export_menu = None
 
-        res = DIV(console,
-                  DIV(htmltable,_class="web2py_table"),
-                  DIV(paginator,_class=\
-                          "web2py_paginator %(header)s %(cornerbottom)s" % ui),
+        res = DIV(console,DIV(htmltable,_class="web2py_table"),
                   _class='%s %s' % (_class, ui.get('widget')))
+        if paginator.components:
+            res.append(
+                DIV(paginator,
+                    _class="web2py_paginator %(header)s %(cornerbottom)s"%ui))
         if export_menu: res.append(export_menu)
         res.create_form = create_form
         res.update_form = update_form
@@ -2353,6 +2369,7 @@ class SQLTABLE(TABLE):
         extracolumns=None,
         selectid=None,
         renderstyle=False,
+        cid=None,
         **attributes
         ):
 
@@ -2390,7 +2407,7 @@ class SQLTABLE(TABLE):
                     row.append(TH(coldict['label'],**attrcol))
                 elif orderby:
                     row.append(TH(A(headers.get(c, c),
-                                    _href=th_link+'?orderby=' + c)))
+                                    _href=th_link+'?orderby=' + c, cid=cid)))
                 else:
                     row.append(TH(headers.get(c, c)))
 
@@ -2415,7 +2432,7 @@ class SQLTABLE(TABLE):
                 _class = 'odd'
 
             if not selectid is None: #new implement
-                if record[self.id_field_name]==selectid:
+                if record.get('id') == selectid:
                     _class += ' rowselected'
 
             for colname in columns:

gluon/storage.py

@@ -38,29 +38,14 @@ class Storage(dict):
         >>> print o.a
         None
     """
-    __slots__=()
-    
+    __slots__=()    
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
     __getitem__ = dict.get
     __getattr__ = dict.get
     __repr__ = lambda self: '<Storage %s>' % dict.__repr__(self)
-    __getstate__ = dict
-    __setstate__ = dict.update
-    # def __getattr__(self, key):
-    #    return dict.get(self, key, None)
-    # def __setattr__(self, key, value):
-    #    self[key] = value
-    # def __getitem__(self, key):
-    #    return dict.get(self, key, None)
-    # def __delattr__(self, key):
-    #    del self[key]
-    # def __repr__(self):
-    #     return '<Storage %s>' % dict.__repr__(self)
-    # def __getstate__(self):
-    #     return dict(self)
-    # def __setstate__(self,values):
-    #     self.update(values)
+    # http://stackoverflow.com/questions/5247250/why-does-pickle-getstate-accept-as-a-return-value-the-very-instance-it-requi
+    __getstate__ = lambda self: None
 
     def getlist(self,key):
         """

gluon/streamer.py

@@ -60,8 +60,8 @@ def stream_file_or_304_or_206(
         fp.close()
     stat_file = os.stat(static_file)
     fsize = stat_file[stat.ST_SIZE]
-    mtime = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
-                          time.gmtime(stat_file[stat.ST_MTIME]))
+    modified = stat_file[stat.ST_MTIME]
+    mtime = time.strftime('%a, %d %b %Y %H:%M:%S GMT',time.gmtime(modified))
     headers.setdefault('Content-Type', contenttype(static_file))
     headers.setdefault('Last-Modified', mtime)
     headers.setdefault('Pragma', 'cache')
@@ -91,6 +91,14 @@ def stream_file_or_304_or_206(
         headers['Content-Length'] = '%i' % bytes
         status = 206
     else:
+        if 'gzip' in request.env.http_accept_encoding and\
+                not 'Content-Encoding' in headers:
+            gzipped = static_file + '.gz'
+            if os.path.isfile(gzipped) and os.path.getmtime(gzipped)>modified:
+                static_file = gzipped
+                fsize = os.path.getsize(gzipped)
+                headers['Content-Encoding'] = 'gzip'
+                headers['Vary'] = 'Accept-Encoding'
         try:
             stream = open(static_file, 'rb')
         except IOError, e:
@@ -106,10 +114,3 @@ def stream_file_or_304_or_206(
     else:
         wrapped = streamer(stream, chunk_size=chunk_size, bytes=bytes)
     raise HTTP(status, wrapped, **headers)
-
-
-
-
-
-
-

gluon/tests/test_dal.py

@@ -311,12 +311,13 @@ class TestDatetime(unittest.TestCase):
                          9, 30)), 3)
         self.assertEqual(len(db(db.t.a == datetime.datetime(1971, 12,
                          21, 11, 30)).select()), 1)
-        self.assertEqual(len(db(db.t.a.year() == 1971).select()), 2)
-        self.assertEqual(len(db(db.t.a.month() == 12).select()), 2)
-        self.assertEqual(len(db(db.t.a.day() == 21).select()), 3)
-        self.assertEqual(len(db(db.t.a.hour() == 11).select()), 1)
-        self.assertEqual(len(db(db.t.a.minutes() == 30).select()), 3)
-        self.assertEqual(len(db(db.t.a.seconds() == 0).select()), 3)
+        self.assertEqual(db(db.t.a.year() == 1971).count(), 2)
+        self.assertEqual(db(db.t.a.month() == 12).count(), 2)
+        self.assertEqual(db(db.t.a.day() == 21).count(), 3)
+        self.assertEqual(db(db.t.a.hour() == 11).count(), 1)
+        self.assertEqual(db(db.t.a.minutes() == 30).count(), 3)
+        self.assertEqual(db(db.t.a.seconds() == 0).count(), 3)
+        self.assertEqual(db(db.t.a.epoch()<365*24*3600).count(),1)
         db.t.drop()
 
 
@@ -329,7 +330,7 @@ class TestExpressions(unittest.TestCase):
         self.assertEqual(db.t.insert(a=2), 2)
         self.assertEqual(db.t.insert(a=3), 3)
         self.assertEqual(db(db.t.a == 3).update(a=db.t.a + 1), 1)
-        self.assertEqual(len(db(db.t.a == 4).select()), 1)
+        self.assertEqual(db(db.t.a == 4).count(), 1)
         db.t.drop()
 
 

gluon/tests/test_router.py

@@ -346,8 +346,13 @@ class TestRouter(unittest.TestCase):
         self.assertEqual(filter_url('http://domain1.com/abc', app=True), 'app1')
         self.assertEqual(filter_url('http://www.domain1.com/abc', app=True), 'app1')
         self.assertEqual(filter_url('http://domain2.com/abc', app=True), 'app2')
+        self.assertEqual(filter_url('http://domain2.com/admin', app=True), 'admin')
+
+        routers['BASE']['exclusive_domain'] = True
+        load(rdict=routers)
         self.assertEqual(filter_url('http://domain2.com/admin', app=True), 'app2')
 
+
         self.assertEqual(filter_url('http://domain.com/goodapp', app=True), 'goodapp')
         self.assertRaises(HTTP, filter_url, 'http://domain.com/bad!app', app=True)
         try:
@@ -428,7 +433,7 @@ class TestRouter(unittest.TestCase):
         self.assertEqual(filter_url('http://domain1.com/abc.html'), '/app1/c1/abc')
         self.assertEqual(filter_url('http://domain1.com/abc.css'), '/app1/c1/abc.css')
         self.assertEqual(filter_url('http://domain1.com/index/abc'), "/app1/c1/index ['abc']")
-        self.assertEqual(filter_url('http://domain2.com/app1'), "/app2a/c2a/app1")
+        self.assertEqual(filter_url('http://domain2.com/app1'), "/app1/c1/f1")
 
         self.assertEqual(filter_url('https://domain1.com/app1/ctr/fcn', domain=('app1',None), out=True), "/ctr/fcn")
         self.assertEqual(filter_url('https://www.domain1.com/app1/ctr/fcn', domain=('app1',None), out=True), "/ctr/fcn")
@@ -475,6 +480,98 @@ class TestRouter(unittest.TestCase):
             pass
         self.assertEqual(filter_url('http://domain1.com/app1/c1/f1', domain=('app2b',None), host='domain2.com', out=True), "/app1")
 
+
+    def test_router_domains_ed(self):
+        '''
+        Test URLs that map domains with exclusive_domain set
+        '''
+        routers = dict(
+            BASE = dict(
+                applications = ['app1', 'app2', 'app2A', 'app3', 'app4', 'app5', 'app6'],
+                exclusive_domain = True,
+                domains = {
+                    #  two domains to the same app
+                    "domain1.com"     : "app1",
+                    "www.domain1.com" : "app1",
+                    #  same domain, two ports, to two apps
+                    "domain2.com"      : "app2a",
+                    "domain2.com:8080" : "app2b",
+                    #  two domains, same app, two controllers
+                    "domain3a.com" : "app3/c3a",
+                    "domain3b.com" : "app3/c3b",
+                    #  two domains, same app & controller, two functions
+                    "domain4a.com" : "app4/c4/f4a",
+                    "domain4b.com" : "app4/c4/f4b",
+                    #  http vs https
+                    "domain6.com:80"  : "app6",
+                    "domain6.com:443" : "app6s",
+                },
+            ),
+            app1 =  dict( default_controller = 'c1',  default_function = 'f1',  controllers = ['c1'], exclusive_domain=True, ),
+            app2a = dict( default_controller = 'c2a', default_function = 'f2a', controllers = ['c2a'], ),
+            app2b = dict( default_controller = 'c2b', default_function = 'f2b', controllers = ['c2b'], ),
+            app3 =  dict( controllers = ['c3a', 'c3b'], ),
+            app4 =  dict( default_controller = 'c4', controllers = ['c4']),
+            app5 =  dict( default_controller = 'c5', controllers = ['c5'], domain = 'localhost' ),
+            app6 =  dict( default_controller = 'c6',  default_function = 'f6',  controllers = ['c6'], ),
+            app6s =  dict( default_controller = 'c6s',  default_function = 'f6s',  controllers = ['c6s'], ),
+        )
+
+        load(rdict=routers)
+        self.assertEqual(filter_url('http://domain1.com/abc'), '/app1/c1/abc')
+        self.assertEqual(filter_url('http://domain1.com/c1/abc'), '/app1/c1/abc')
+        self.assertEqual(filter_url('http://domain1.com/abc.html'), '/app1/c1/abc')
+        self.assertEqual(filter_url('http://domain1.com/abc.css'), '/app1/c1/abc.css')
+        self.assertEqual(filter_url('http://domain1.com/index/abc'), "/app1/c1/index ['abc']")
+        self.assertEqual(filter_url('http://domain2.com/app1'), "/app2a/c2a/app1")
+
+        self.assertEqual(filter_url('https://domain1.com/app1/ctr/fcn', domain=('app1',None), out=True), "/ctr/fcn")
+        self.assertEqual(filter_url('https://www.domain1.com/app1/ctr/fcn', domain=('app1',None), out=True), "/ctr/fcn")
+
+        self.assertEqual(filter_url('http://domain2.com/abc'), '/app2a/c2a/abc')
+        self.assertEqual(filter_url('http://domain2.com:8080/abc'), '/app2b/c2b/abc')
+
+        self.assertEqual(filter_url('http://domain2.com/app2a/ctr/fcn', domain=('app2a',None), out=True), "/ctr/fcn")
+        self.assertEqual(filter_url('http://domain2.com/app2a/ctr/f2a', domain=('app2a',None), out=True), "/ctr")
+        self.assertEqual(filter_url('http://domain2.com/app2a/c2a/f2a', domain=('app2a',None), out=True), "/")
+        self.assertEqual(filter_url('http://domain2.com/app2a/c2a/fcn', domain=('app2a',None), out=True), "/fcn")
+        
+        self.assertRaises(SyntaxError, filter_url, 'http://domain2.com/app2a/ctr/fcn', domain=('app2b',None), out=True)
+        self.assertRaises(SyntaxError, filter_url, 'http://domain2.com/app2a/ctr/f2a', domain=('app2b',None), out=True)
+        self.assertRaises(SyntaxError, filter_url, 'http://domain2.com/app2a/c2a/f2a', domain=('app2b',None), out=True)
+
+        self.assertEqual(filter_url('http://domain3a.com/'), '/app3/c3a/index')
+        self.assertEqual(filter_url('http://domain3a.com/abc'), '/app3/c3a/abc')
+        self.assertEqual(filter_url('http://domain3a.com/c3b'), '/app3/c3b/index')
+        self.assertEqual(filter_url('http://domain3b.com/abc'), '/app3/c3b/abc')
+
+        self.assertEqual(filter_url('http://domain3a.com/app3/c3a/fcn', domain=('app3','c3a'), out=True), "/fcn")
+        self.assertEqual(filter_url('http://domain3a.com/app3/c3a/fcn', domain=('app3','c3b'), out=True), "/c3a/fcn")
+        
+        self.assertRaises(SyntaxError, filter_url, 'http://domain3a.com/app3/c3a/fcn', domain=('app1',None), out=True)
+
+        self.assertEqual(filter_url('http://domain4a.com/abc'), '/app4/c4/abc')
+        self.assertEqual(filter_url('https://domain4a.com/app4/c4/fcn', domain=('app4',None), out=True), "/fcn")
+
+        self.assertEqual(filter_url('http://domain4a.com'), '/app4/c4/f4a')
+        self.assertEqual(filter_url('http://domain4b.com'), '/app4/c4/f4b')
+
+        self.assertEqual(filter_url('http://localhost/abc'), '/app5/c5/abc')
+        self.assertEqual(filter_url('http:///abc'), '/app5/c5/abc') # test null host => localhost
+        self.assertEqual(filter_url('https://localhost/app5/c5/fcn', domain=('app5',None), out=True), "/fcn")
+
+        self.assertEqual(filter_url('http://domain6.com'), '/app6/c6/f6')
+        self.assertEqual(filter_url('https://domain6.com'), '/app6s/c6s/f6s')
+
+        self.assertRaises(SyntaxError, filter_url, 'http://domain2.com/app3/c3a/f3', domain=('app2b',None), out=True)
+        self.assertRaises(SyntaxError, filter_url, 'http://domain1.com/app1/c1/f1', domain=('app2b',None), out=True)
+        try:
+            # 2.7+ only
+            self.assertRaisesRegexp(SyntaxError, 'cross-domain conflict', filter_url, 'http://domain1.com/app1/c1/f1', domain=('app2b',None), out=True)
+        except AttributeError:
+            pass
+        self.assertEqual(filter_url('http://domain1.com/app1/c1/f1', domain=('app2b',None), host='domain2.com', out=True), "/app1")
+
     def test_router_raise(self):
         '''
         Test URLs that raise exceptions

gluon/tools.py

@@ -794,6 +794,10 @@ def addrow(form, a, b, c, style, _id, position=-1):
                                     DIV(b, _class='w2p_fw'),
                                     DIV(c, _class='w2p_fc'),
                                     _id = _id))
+    elif style == "bootstrap":
+        form[0].insert(position, DIV(LABEL(a,_class='control-label'),
+                                    DIV(b,SPAN(c, _class='inline-help'),_class='controls'),
+                                    _class='control-group',_id = _id))
     else:
         form[0].insert(position, TR(TD(LABEL(a),_class='w2p_fl'),
                                     TD(b,_class='w2p_fw'),
@@ -1191,7 +1195,10 @@ class Auth(object):
                        'reset_password','request_reset_password',
                        'change_password','profile','groups',
                        'impersonate','not_authorized'):
-            return getattr(self,args[0])()
+            if len(request.args) >= 2:
+                return getattr(self,args[0])(request.args[1])
+            else:
+                return getattr(self,args[0])()
         elif args[0]=='cas' and not self.settings.cas_provider:
             if args(1) == self.settings.cas_actions['login']:
                 return self.cas_login(version=2)
@@ -1208,23 +1215,23 @@ class Auth(object):
 
     def navbar(self, prefix='Welcome', action=None,
                separators=(' [ ',' | ',' ] '), user_identifier=DEFAULT,
-               referrer_actions=DEFAULT):
+               referrer_actions=DEFAULT, mode='default'):
         referrer_actions = [] if not referrer_actions else referrer_actions
         request = current.request
+        asdropdown = (mode == 'dropdown')
         T = current.T
         if isinstance(prefix, str):
             prefix = T(prefix)
         if prefix:
             prefix = prefix.strip() + ' '
         if not action:
-            action=self.url(self.settings.function)
+            action = self.url(self.settings.function)
         s1,s2,s3 = separators
         if URL() == action:
             next = ''
         else:
             next = '?_next=' + urllib.quote(URL(args=request.args,
                                                 vars=request.get_vars))
-
         href = lambda function: '%s/%s%s' % (action, function,
             next if referrer_actions is DEFAULT or function in referrer_actions else '')
 
@@ -1244,11 +1251,19 @@ class Auth(object):
             profile = A(T('Profile'), _href=href('profile'))
             password = A(T('Password'), _href=href('change_password'))
             bar = SPAN(prefix, user_identifier, s1, logout, s3, _class='auth_navbar')
+
+            if asdropdown:
+                logout = LI(A(I(_class='icon-off'), ' '+T('Logout'), _href='%s/logout?_next=%s' %
+                      (action, urllib.quote(self.settings.logout_next)))) # the space before T('Logout') is intentional. It creates a gap between icon and text
+                profile = LI(A(I(_class='icon-user'), ' '+T('Profile'), _href=href('profile')))
+                password = LI(A(I(_class='icon-lock'), ' '+T('Password'), _href=href('change_password')))
+                bar = UL(logout,_class='dropdown-menu') # logout will be the last item in list
+
             if not 'profile' in self.settings.actions_disabled:
-                bar.insert(-1, s2)
+                if not asdropdown: bar.insert(-1, s2)
                 bar.insert(-1, profile)
             if not 'change_password' in self.settings.actions_disabled:
-                bar.insert(-1, s2)
+                if not asdropdown: bar.insert(-1, s2)
                 bar.insert(-1, password)
         else:
             login = A(T('Login'), _href=href('login'))
@@ -1259,17 +1274,33 @@ class Auth(object):
                 T('Lost password?'), _href=href('request_reset_password'))
             bar = SPAN(s1, login, s3, _class='auth_navbar')
 
+            if asdropdown:
+                login = LI(A(I(_class='icon-off'), ' '+T('Login'), _href=href('login'))) #the space before T('Login') is intentional. It creates a gap between icon and text
+                register = LI(A(I(_class='icon-user'), ' '+T('Register'), _href=href('register')))
+                retrieve_username = LI(A(I(_class='icon-edit'), ' '+T('Forgot username?'), _href=href('retrieve_username')))
+                lost_password = LI(A(I(_class='icon-lock'), ' '+T('Lost password?'), _href=href('request_reset_password')))
+                bar = UL(login,_class='dropdown-menu') # login will be the last item in list
+
             if not 'register' in self.settings.actions_disabled:
-                bar.insert(-1, s2)
+                if not asdropdown: bar.insert(-1, s2)
                 bar.insert(-1, register)
             if self.settings.use_username and not 'retrieve_username' \
                     in self.settings.actions_disabled:
-                bar.insert(-1, s2)
+                if not asdropdown: bar.insert(-1, s2)
                 bar.insert(-1, retrieve_username)
             if not 'request_reset_password' \
                     in self.settings.actions_disabled:
-                bar.insert(-1, s2)
+                if not asdropdown: bar.insert(-1, s2)
                 bar.insert(-1, lost_password)
+
+        if asdropdown:
+            bar.insert(-1, LI('',_class='divider'))
+            if self.user_id:               
+                bar = LI(A(prefix, user_identifier, _href='#'),
+                         bar,_class='dropdown')
+            else:
+                bar = LI(A(T('Login'), _href='#'),
+                         bar,_class='dropdown')
         return bar
 
     def __get_migrate(self, tablename, migrate=True):
@@ -1877,20 +1908,33 @@ class Auth(object):
 
             if self.settings.remember_me_form:
                 ## adds a new input checkbox "remember me for longer"
-                addrow(form,XML(" "),
-                       DIV(XML(" "),
-                           INPUT(_type='checkbox',
-                                 _class='checkbox',
-                                 _id="auth_user_remember",
-                                 _name="remember",
-                                 ),
-                           XML("  "),
+                if self.settings.formstyle != 'bootstrap':
+                    addrow(form,XML(" "),
+                               DIV(XML(" "),
+                                   INPUT(_type='checkbox',
+                                         _class='checkbox',
+                                         _id="auth_user_remember",
+                                         _name="remember",
+                                        ),
+                                    XML("  "),
+                                    LABEL(
+                                          self.messages.label_remember_me,
+                                          _for="auth_user_remember",
+                                        )),"",
+                               self.settings.formstyle,
+                               'auth_user_remember__row')
+                elif self.settings.formstyle == 'bootstrap':
+                    addrow(form,
+                           "",
                            LABEL(
-                            self.messages.label_remember_me,
-                            _for="auth_user_remember",
-                            )),"",
-                       self.settings.formstyle,
-                       'auth_user_remember__row')
+                                 INPUT(_type='checkbox',
+                                       _id="auth_user_remember",
+                                       _name="remember"),
+                                 self.messages.label_remember_me,
+                                 _class="checkbox"),
+                            "",
+                            self.settings.formstyle,
+                            'auth_user_remember__row')
 
             captcha = self.settings.login_captcha or \
                 (self.settings.login_captcha!=False and self.settings.captcha)
@@ -2107,6 +2151,9 @@ class Auth(object):
                                 repr(request.vars.get(passfield, None)),
                             error_message=self.messages.mismatched_password))
 
+                    if formstyle == 'bootstrap' :
+                        form.custom.widget.password_two['_class'] = 'input-xlarge'
+
                     addrow(form, self.messages.verify_password + self.settings.label_separator,
                            form.custom.widget.password_two,
                            self.messages.verify_password_comment,
@@ -2699,7 +2746,7 @@ class Auth(object):
             self.user = session.auth.user
         if requested_id is DEFAULT and not request.post_vars:
             return SQLFORM.factory(Field('user_id', 'integer'))
-        return self.user
+        return SQLFORM(table_user, user.id, readonly=True)
 
     def update_groups(self):
         if not self.user:
@@ -3155,14 +3202,17 @@ class Auth(object):
     def wiki(self,
              slug=None,
              env=None,
-             render=None,
+             render='markmin',
              manage_permissions=False,
-             force_prefix='', 
+             force_prefix='',
+             restrict_search=False,
              resolve=True):
         if not hasattr(self,'_wiki'):
             self._wiki = Wiki(self,render=render,
                               manage_permissions=manage_permissions,
-                              force_prefix=force_prefix,env=env)
+                              force_prefix=force_prefix,
+                              restrict_search=restrict_search,
+                              env=env)
         else:
             self._wiki.env.update(env or {})
         # if resolve is set to True, process request as wiki call
@@ -4506,7 +4556,8 @@ class Wiki(object):
         controller, function, args = items[0], items[1], items[2:]
         return LOAD(controller, function, args=args, ajax=True).xml()
     def __init__(self,auth,env=None,render='markmin',
-                 manage_permissions=False,force_prefix=''):
+                 manage_permissions=False,force_prefix='',
+                 restrict_search=False):
         self.env = env or {}
         self.env['component'] = Wiki.component
         if render == 'markmin': render=self.markmin_render
@@ -4518,46 +4569,47 @@ class Wiki(object):
             self.force_prefix = force_prefix
         self.host = current.request.env.http_host
         perms = self.manage_permissions = manage_permissions
+        self.restrict_search = restrict_search
         db = auth.db
-        table_definitions = {
-            'wiki_page':{
-                'args':[
-                    Field('slug',
-                          requires=[IS_SLUG(),
-                                    IS_NOT_IN_DB(db,'wiki_page.slug')],
-                          readable=False,writable=False),
-                    Field('title',unique=True),
-                    Field('body','text',notnull=True),
-                    Field('tags','list:string'),
-                    Field('can_read','list:string',
-                          writable=perms,
-                          readable=perms,
-                          default=[Wiki.everybody]),
-                    Field('can_edit', 'list:string',
-                          writable=perms,readable=perms,
-                          default=[Wiki.everybody]),
+        table_definitions = [
+            ('wiki_page',{
+                    'args':[
+                        Field('slug',
+                              requires=[IS_SLUG(),
+                                        IS_NOT_IN_DB(db,'wiki_page.slug')],
+                              readable=False,writable=False),
+                        Field('title',unique=True),
+                        Field('body','text',notnull=True),
+                        Field('tags','list:string'),
+                        Field('can_read','list:string',
+                              writable=perms,
+                              readable=perms,
+                              default=[Wiki.everybody]),
+                        Field('can_edit', 'list:string',
+                              writable=perms,readable=perms,
+                              default=[Wiki.everybody]),
                     Field('changelog'),
-                    Field('html','text',compute=render,
-                          readable=False, writable=False),
-                    auth.signature],
-                'vars':{'format':'%(title)s'}},
-             'wiki_tag':{
-                'args':[
-                    Field('name'),
-                    Field('wiki_page','reference wiki_page'),
-                    auth.signature],
-                'vars':{'format':'%(name)s'}},
-           'wiki_media':{
-                'args':[
-                    Field('wiki_page','reference wiki_page'),
-                    Field('title',required=True),
-                    Field('file','upload',required=True),
-                    auth.signature],
-                'vars':{'format':'%(title)s'}}
-            }
+                        Field('html','text',compute=render,
+                              readable=False, writable=False),
+                        auth.signature],
+              'vars':{'format':'%(title)s'}}),
+            ('wiki_tag',{
+                    'args':[
+                        Field('name'),
+                        Field('wiki_page','reference wiki_page'),
+                        auth.signature],
+                    'vars':{'format':'%(name)s'}}),
+            ('wiki_media',{
+                    'args':[
+                        Field('wiki_page','reference wiki_page'),
+                        Field('title',required=True),
+                        Field('filename','upload',required=True),
+                        auth.signature],
+                    'vars':{'format':'%(title)s'}})
+            ]
 
         # define only non-existent tables
-        for key, value in table_definitions.iteritems():
+        for key, value in table_definitions:
             if not key in db.tables():
                 db.define_table(key, *value['args'], **value['vars'])
 
@@ -4722,10 +4774,11 @@ class Wiki(object):
         db = auth.db
         page = db.wiki_page(slug=slug)
         if not (page and self.can_edit(page)): return self.not_authorized(page)
-        self.auth.db.wiki_media.id.represent = lambda id,row:\
+        self.auth.db.wiki_media.id.represent = lambda id,row: \
+            id if not row.filename else \
             SPAN('@////%i/%s.%s' % \
                      (id,IS_SLUG.urlify(row.title.split('.')[0]),
-                      row.file.split('.')[-1]))
+                      row.filename.split('.')[-1]))
         self.auth.db.wiki_media.wiki_page.default = page.id
         self.auth.db.wiki_media.wiki_page.writable = False
         content = SQLFORM.grid(
@@ -4770,7 +4823,7 @@ class Wiki(object):
             if self.manage_permissions:
                 page = db.wiki_page(media.wiki_page)
                 if not self.can_read(page): return self.not_authorized(page)
-            request.args = [media.file]
+            request.args = [media.filename]
             return current.response.download(request,db)
         else:
             raise HTTP(404)
@@ -4826,9 +4879,8 @@ class Wiki(object):
                             URL(controller,function,args=('_pages'))))
             submenu.append((current.T('Edit Menu'),None,
                             URL(controller,function,args=('_edit','wiki-menu'))))
-        # if self.can_search():
-            submenu.append((current.T('Search Pages'),None,
-                            URL(controller,function,args=('_search'))))
+        submenu.append((current.T('Search Pages'),None,
+                        URL(controller,function,args=('_search'))))
         return menu
 
     def search(self,tags=None,query=None,cloud=True,preview=True,
@@ -4857,6 +4909,8 @@ class Wiki(object):
                 query = (db.wiki_page.id==db.wiki_tag.wiki_page)&\
                     (db.wiki_tag.name.belongs(tags))
                 query = query|db.wiki_page.title.startswith(request.vars.q)
+            if self.restrict_search and not self.manage():
+                query = query&(db.wiki_page.created_by==self.auth.user_id)
             pages = db(query).select(
                 *fields,**dict(orderby=orderby or ~count,
                                groupby=db.wiki_page.id,

gluon/utils.py

@@ -169,7 +169,7 @@ def is_valid_ip_address(address):
                 return False
         else: # try validate using Regex
             match = REGEX_IPv4.match(address)
-            if match and all(0<=int(math.group(i))<256 for i in (1,2,3,4)):
+            if match and all(0<=int(match.group(i))<256 for i in (1,2,3,4)):
                 return True
             return False
     elif hasattr(socket,'inet_pton'): # assume IPv6, try using the OS

gluon/validators.py

@@ -462,12 +462,13 @@ class IS_IN_DB(Validator):
             groupby = self.groupby
             distinct = self.distinct
             dd = dict(orderby=orderby, groupby=groupby,
-                      distinct=distinct, cache=self.cache)
+                      distinct=distinct, cache=self.cache,
+                      cacheable=True)
             records = self.dbset(table).select(*fields, **dd)
         else:
             orderby = self.orderby or \
                 reduce(lambda a,b:a|b,(f for f in fields if not f.name=='id'))
-            dd = dict(orderby=orderby, cache=self.cache)
+            dd = dict(orderby=orderby, cache=self.cache, cacheable=True)
             records = self.dbset(table).select(table.ALL, **dd)
         self.theset = [str(r[self.kfield]) for r in records]
         if isinstance(self.label,str):
@@ -488,6 +489,8 @@ class IS_IN_DB(Validator):
         table = self.dbset.db[self.ktable]
         field = table[self.kfield]
         if self.multiple:
+            if self._and:
+                raise NotImplementedError
             if isinstance(value,list):
                 values=value
             elif value:
@@ -497,8 +500,20 @@ class IS_IN_DB(Validator):
             if isinstance(self.multiple,(tuple,list)) and \
                     not self.multiple[0]<=len(values)<self.multiple[1]:
                 return (values, translate(self.error_message))
-            if self.dbset(field.belongs(values)).count()==len(values):
-                return (values, None)
+            if self.theset:
+                if not [v for v in values if not v in self.theset]:
+                    return (values, None)
+            else:
+                from dal import GoogleDatastoreAdapter
+                def count(values, s=self.dbset, f=field):
+                    return s(f.belongs(map(int,values))).count()
+                if isinstance(self.dbset.db._adapter, GoogleDatastoreAdapter):
+                    range_ids = range(0,len(ids),30)
+                    total = sum(count(values[i:i+30]) for i in range_ids)
+                    if total == len(values):
+                        return (values, None)
+                elif count(values) == len(values):
+                    return (values, None)
         elif self.theset:
             if str(value) in self.theset:
                 if self._and:
@@ -2329,11 +2344,12 @@ class IS_LIST_OF(Validator):
         new_value = []
         if self.other:
             for item in ivalue:
-                (v, e) = self.other(item)
-                if e:
-                    return (value, e)
-                else:
-                    new_value.append(v)
+                if item.strip():
+                    (v, e) = self.other(item)
+                    if e:
+                        return (ivalue, e)
+                    else:
+                        new_value.append(v)
             ivalue = new_value
         return (ivalue, None)
 
@@ -2592,7 +2608,9 @@ class LazyCrypt(object):
                 key = self.crypt.key
         else:
             key = ''
-        if stored_password.count('$')==2:
+        if stored_password is None:
+            return False
+        elif stored_password.count('$')==2:
             (digest_alg, salt, hash) = stored_password.split('$')
             h = simple_hash(self.password, key, salt, digest_alg)
             temp_pass = '%s$%s$%s' % (digest_alg, salt, h)

gluon/winservice.py

@@ -154,6 +154,8 @@ class Web2pyService(Service):
 def web2py_windows_service_handler(argv=None, opt_file='options'):
     path = os.path.dirname(__file__)
     web2py_path = up(path)
+    if web2py_path.endswith('.zip'): # in case bianry distro 'library.zip'
+        web2py_path = os.path.dirname(web2py_path)
     os.chdir(web2py_path)
     classstring = os.path.normpath(
         os.path.join(web2py_path,'gluon.winservice.Web2pyService'))