commit merge

gluon/cache.py

@@ -298,7 +298,7 @@ class CacheOnDisk(CacheAbstract):
             try:
                 storage = self._open_shelf_with_lock()
                 try:
-                    if not storage.has_key(CacheAbstract.cache_stats_name):
+                    if not CacheAbstract.cache_stats_name in storage:
                         storage[CacheAbstract.cache_stats_name] = {
                             'hit_total': 0,
                             'misses': 0,

gluon/compileapp.py

@@ -50,6 +50,8 @@ is_pypy = settings.global_settings.is_pypy
 is_gae  = settings.global_settings.web2py_runtime_gae
 is_jython = settings.global_settings.is_jython
 
+pjoin = os.path.join
+
 TEST_CODE = \
     r"""
 def _TEST():
@@ -319,7 +321,7 @@ def local_import_aux(name, reload_force=False, app='welcome'):
 """
 OLD IMPLEMENTATION:
     items = name.replace('/','.').split('.')
-    filename, modulepath = items[-1], os.path.join(apath,'modules',*items[:-1])
+    filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1])
     imp.acquire_lock()
     try:
         file=None
@@ -348,11 +350,9 @@ def build_environment(request, response, session, store_current=True):
     """
     Build the environment dictionary into which web2py files are executed.
     """
-
-    _validators = validators
-    _html = html
-    environment = dict(map(lambda key: (key, getattr(_html, key)), _html.__all__))
-    environment.update(map(lambda key: (key, getattr(_validators, key)), _validators.__all__))
+    h,v = html,validators
+    environment = dict((k,getattr(h,k)) for k  in h.__all__)
+    environment.update((k,getattr(v, k)) for k in v.__all__)
     if not request.env:
         request.env = Storage()
 
@@ -389,7 +389,7 @@ def build_environment(request, response, session, store_current=True):
     environment['local_import'] = \
         lambda name, reload=False, app=request.application:\
         local_import_aux(name,reload,app)
-    BaseAdapter.set_folder(os.path.join(request.folder, 'databases'))
+    BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
     response._view_environment = copy.copy(environment)
     return environment
 
@@ -419,11 +419,11 @@ def compile_views(folder):
     Compiles all the views in the application specified by `folder`
     """
 
-    path = os.path.join(folder, 'views')
+    path = pjoin(folder, 'views')
     for file in listdir(path, '^[\w/\-]+(\.\w+)+$'):
         data = parse_template(file, path)
         filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')
-        filename = os.path.join(folder, 'compiled', filename)
+        filename = pjoin(folder, 'compiled', filename)
         write_file(filename, data)
         save_pyc(filename)
         os.unlink(filename)
@@ -434,10 +434,10 @@ def compile_models(folder):
     Compiles all the models in the application specified by `folder`
     """
 
-    path = os.path.join(folder, 'models')
+    path = pjoin(folder, 'models')
     for file in listdir(path, '.+\.py$'):
-        data = read_file(os.path.join(path, file))
-        filename = os.path.join(folder, 'compiled','models',file)
+        data = read_file(pjoin(path, file))
+        filename = pjoin(folder, 'compiled','models',file)
         mktree(filename)
         write_file(filename, data)
         save_pyc(filename)
@@ -449,15 +449,15 @@ def compile_controllers(folder):
     Compiles all the controllers in the application specified by `folder`
     """
 
-    path = os.path.join(folder, 'controllers')
+    path = pjoin(folder, 'controllers')
     for file in listdir(path, '.+\.py$'):
-        ### why is this here? save_pyc(os.path.join(path, file))
-        data = read_file(os.path.join(path,file))
+        ### why is this here? save_pyc(pjoin(path, file))
+        data = read_file(pjoin(path,file))
         exposed = regex_expose.findall(data)
         for function in exposed:
             command = data + "\nresponse._vars=response._caller(%s)\n" % \
                 function
-            filename = os.path.join(folder, 'compiled', ('controllers/'
+            filename = pjoin(folder, 'compiled', ('controllers/'
                                      + file[:-3]).replace('/', '_')
                                      + '_' + function + '.py')
             write_file(filename, command)
@@ -474,18 +474,18 @@ def run_models_in(environment):
     folder = environment['request'].folder
     c = environment['request'].controller
     f = environment['request'].function
-    cpath = os.path.join(folder, 'compiled')
+    cpath = pjoin(folder, 'compiled')
     if os.path.exists(cpath):
         for model in listdir(cpath, '^models_\w+\.pyc$', 0):
             restricted(read_pyc(model), environment, layer=model)
-        path = os.path.join(cpath, 'models')
+        path = pjoin(cpath, 'models')
         models = listdir(path, '^\w+\.pyc$',0,sort=False)
         compiled=True
     else:
-        path = os.path.join(folder, 'models')
+        path = pjoin(folder, 'models')
         models = listdir(path, '^\w+\.py$',0,sort=False)
         compiled=False
-    paths = (path, os.path.join(path,c), os.path.join(path,c,f))
+    paths = (path, pjoin(path,c), pjoin(path,c,f))
     for model in models:
         if not os.path.split(model)[0] in paths and c!='appadmin':
             continue
@@ -509,11 +509,11 @@ def run_controller_in(controller, function, environment):
     # if compiled should run compiled!
 
     folder = environment['request'].folder
-    path = os.path.join(folder, 'compiled')
+    path = pjoin(folder, 'compiled')
     badc = 'invalid controller (%s/%s)' % (controller, function)
     badf = 'invalid function (%s/%s)' % (controller, function)
     if os.path.exists(path):
-        filename = os.path.join(path, 'controllers_%s_%s.pyc'
+        filename = pjoin(path, 'controllers_%s_%s.pyc'
                                  % (controller, function))
         if not os.path.exists(filename):
             raise HTTP(404,
@@ -528,7 +528,7 @@ def run_controller_in(controller, function, environment):
         [add_path_first(path) for path in paths]
         # TESTING END
 
-        filename = os.path.join(folder, 'controllers/%s.py'
+        filename = pjoin(folder, 'controllers/%s.py'
                                  % controller)
         if not os.path.exists(filename):
             raise HTTP(404,
@@ -539,7 +539,7 @@ def run_controller_in(controller, function, environment):
         code += TEST_CODE
         restricted(code, environment, layer=filename)
     else:
-        filename = os.path.join(folder, 'controllers/%s.py'
+        filename = pjoin(folder, 'controllers/%s.py'
                                  % controller)
         if not os.path.exists(filename):
             raise HTTP(404,
@@ -576,19 +576,20 @@ def run_view_in(environment):
 
     request = environment['request']
     response = environment['response']
+    view = response.view
     folder = request.folder
-    path = os.path.join(folder, 'compiled')
-    badv = 'invalid view (%s)' % response.view
+    path = pjoin(folder, 'compiled')
+    badv = 'invalid view (%s)' % view
     patterns = response.generic_patterns or []
     regex = re.compile('|'.join(map(fnmatch.translate, patterns)))
     short_action =  '%(controller)s/%(function)s.%(extension)s' % request
     allow_generic = patterns and regex.search(short_action)
-    if not isinstance(response.view, str):
-        ccode = parse_template(response.view, os.path.join(folder, 'views'),
+    if not isinstance(view, str):
+        ccode = parse_template(view, pjoin(folder, 'views'),
                                context=environment)
         restricted(ccode, environment, 'file stream')
     elif os.path.exists(path):
-        x = response.view.replace('/', '_')
+        x = view.replace('/', '_')
         files = ['views_%s.pyc' % x]
         if allow_generic:
             files.append('views_generic.%s.pyc' % request.extension)
@@ -599,7 +600,7 @@ def run_view_in(environment):
                 files.append('views_generic.pyc')
         # end backward compatibility code
         for f in files:
-            filename = os.path.join(path,f)
+            filename = pjoin(path,f)
             if os.path.exists(filename):
                 code = read_pyc(filename)
                 restricted(code, environment, layer=filename)
@@ -608,10 +609,10 @@ def run_view_in(environment):
                    rewrite.thread.routes.error_message % badv,
                    web2py_error=badv)
     else:
-        filename = os.path.join(folder, 'views', response.view)
+        filename = pjoin(folder, 'views', view)
         if not os.path.exists(filename) and allow_generic:
-            response.view = 'generic.' + request.extension
-            filename = os.path.join(folder, 'views', response.view)
+            view = 'generic.' + request.extension
+            filename = pjoin(folder, 'views', view)
         if not os.path.exists(filename):
             raise HTTP(404,
                        rewrite.thread.routes.error_message % badv,
@@ -619,12 +620,12 @@ def run_view_in(environment):
         layer = filename
         if is_gae:
             ccode = getcfs(layer, filename,
-                           lambda: compile2(parse_template(response.view,
-                                            os.path.join(folder, 'views'),
+                           lambda: compile2(parse_template(view,
+                                            pjoin(folder, 'views'),
                                             context=environment),layer))
         else:
-            ccode = parse_template(response.view,
-                                   os.path.join(folder, 'views'),
+            ccode = parse_template(view,
+                                   pjoin(folder, 'views'),
                                    context=environment)
         restricted(ccode, environment, layer)
 
@@ -633,8 +634,8 @@ def remove_compiled_application(folder):
     Deletes the folder `compiled` containing the compiled application.
     """
     try:
-        shutil.rmtree(os.path.join(folder, 'compiled'))
-        path = os.path.join(folder, 'controllers')
+        shutil.rmtree(pjoin(folder, 'compiled'))
+        path = pjoin(folder, 'controllers')
         for file in listdir(path,'.*\.pyc$',drop=False):
             os.unlink(file)
     except OSError:
@@ -646,7 +647,7 @@ def compile_application(folder):
     Compiles all models, views, controller for the application in `folder`.
     """
     remove_compiled_application(folder)
-    os.mkdir(os.path.join(folder, 'compiled'))
+    os.mkdir(pjoin(folder, 'compiled'))
     compile_models(folder)
     compile_controllers(folder)
     compile_views(folder)

gluon/globals.py

@@ -117,7 +117,8 @@ class Request(Storage):
             user_agent_parser.detect(self.env.http_user_agent)
         user_agent = Storage(user_agent)
         for key,value in user_agent.items():
-            if isinstance(value,dict): user_agent[key] = Storage(value)
+            if isinstance(value,dict):
+                user_agent[key] = Storage(value)
         return user_agent
 
     def requires_https(self):
@@ -222,9 +223,9 @@ class Response(Storage):
         return page
 
     def include_meta(self):
-        s = '\n'
-        for key,value in (self.meta or {}).items():
-            s += '<meta name="%s" content="%s" />\n' % (key,xmlescape(value))
+        s = '\n'.join(
+            '<meta name="%s" content="%s" />\n' % (k,xmlescape(v))
+            for k,v in (self.meta or {}).iteritems())
         self.write(s,escape=False)
 
     def include_files(self):
@@ -297,14 +298,15 @@ class Response(Storage):
         default to the last request argument otherwise)
         """
 
+        headers = self.headers
         # for attachment settings and backward compatibility
-        keys = [item.lower() for item in self.headers]
+        keys = [item.lower() for item in headers]
         if attachment:
             if filename is None:
                 attname = ""
             else:
                 attname = filename
-            self.headers["Content-Disposition"] = \
+            headers["Content-Disposition"] = \
                 "attachment;filename=%s" % attname
 
         if not request:
@@ -313,30 +315,31 @@ class Response(Storage):
             stream_file_or_304_or_206(stream,
                                       chunk_size=chunk_size,
                                       request=request,
-                                      headers=self.headers)
+                                      headers=headers)
 
         # ## the following is for backward compatibility
         if hasattr(stream, 'name'):
             filename = stream.name
 
         if filename and not 'content-type' in keys:
-            self.headers['Content-Type'] = contenttype(filename)
+            headers['Content-Type'] = contenttype(filename)
         if filename and not 'content-length' in keys:
             try:
-                self.headers['Content-Length'] = \
+                headers['Content-Length'] = \
                     os.path.getsize(filename)
             except OSError:
                 pass
-
+        
+        env = request.env
         # Internet Explorer < 9.0 will not allow downloads over SSL unless caching is enabled
-        if request.is_https and isinstance(request.env.http_user_agent,str) and \
-                not re.search(r'Opera', request.env.http_user_agent) and \
-                re.search(r'MSIE [5-8][^0-9]', request.env.http_user_agent):
-            self.headers['Pragma'] = 'cache'
-            self.headers['Cache-Control'] = 'private'
+        if request.is_https and isinstance(env.http_user_agent,str) and \
+                not re.search(r'Opera', env.http_user_agent) and \
+                re.search(r'MSIE [5-8][^0-9]', env.http_user_agent):
+            headers['Pragma'] = 'cache'
+            headers['Cache-Control'] = 'private'
 
-        if request and request.env.web2py_use_wsgi_file_wrapper:
-            wrapped = request.env.wsgi_file_wrapper(stream, chunk_size)
+        if request and env.web2py_use_wsgi_file_wrapper:
+            wrapped = env.wsgi_file_wrapper(stream, chunk_size)
         else:
             wrapped = streamer(stream, chunk_size=chunk_size)
         return wrapped
@@ -364,11 +367,13 @@ class Response(Storage):
             (filename, stream) = field.retrieve(name)
         except IOError:
             raise HTTP(404)
-        self.headers['Content-Type'] = contenttype(name)
+        headers = self.headers
+        headers['Content-Type'] = contenttype(name)
         if attachment:
-            self.headers['Content-Disposition'] = \
-                "attachment; filename=%s" % filename
-        return self.stream(stream, chunk_size = chunk_size, request=request)
+            headers['Content-Disposition'] = \
+                'attachment; filename=%s' % filename
+        return self.stream(stream, chunk_size=chunk_size, request=request)
+                           
 
     def json(self, data, default=None):
         return json(data, default = default or custom_json)
@@ -405,8 +410,15 @@ class Response(Storage):
             dbstats = [TABLE(*[TR(PRE(row[0]),'%.2fms' % (row[1]*1000)) \
                                    for row in i.db._timings]) \
                            for i in thread.instances]
+            dbtables = dict([(i.uri, {'defined': sorted(list(set(i.db.tables) - 
+                                                 set(i.db._LAZY_TABLES.keys()))) or
+                                                 '[no defined tables]',
+                                      'lazy': sorted(i.db._LAZY_TABLES.keys()) or
+                                              '[no lazy tables]'})
+                             for i in thread.instances])
         else:
             dbstats = [] # if no db or on GAE
+            dbtables = {}
         u = web2py_uuid()
         return DIV(
             BUTTON('design',_onclick="document.location='%s'" % admin),
@@ -416,6 +428,8 @@ class Response(Storage):
             DIV(BEAUTIFY(current.session),_class="hidden",_id="session-%s"%u),
             BUTTON('response',_onclick="jQuery('#response-%s').slideToggle()"%u),
             DIV(BEAUTIFY(current.response),_class="hidden",_id="response-%s"%u),
+            BUTTON('db tables',_onclick="jQuery('#db-tables-%s').slideToggle()"%u),
+            DIV(BEAUTIFY(dbtables),_class="hidden",_id="db-tables-%s"%u),
             BUTTON('db stats',_onclick="jQuery('#db-stats-%s').slideToggle()"%u),
             DIV(BEAUTIFY(dbstats),_class="hidden",_id="db-stats-%s"%u),
             SCRIPT("jQuery('.hidden').hide()")
@@ -452,14 +466,15 @@ class Session(Storage):
         response.session_id_name = 'session_id_%s' % masterapp.lower()
 
         # Load session data from cookie
-
+        cookies = request.cookies
+            
         if cookie_key:
             response.session_cookie_key = cookie_key
             response.session_cookie_key2 = hashlib.md5(cookie_key).digest()
-            cookie_name = request.application.lower()+'_session_data'
+            cookie_name = masterapp.lower()+'_session_data'
             response.session_cookie_name = cookie_name
-            if cookie_data in request.cookies:
-                cookie_value = request.cookies[cookie_name].value
+            if cookie_name in cookies:
+                cookie_value = cookies[cookie_name].value
                 cookie_parts = cookie_value.split(":")
                 enc = cookie_parts[2]
                 cipher = AES.new(cookie_key)
@@ -477,9 +492,9 @@ class Session(Storage):
                 return
             response.session_new = False
             client = request.client and request.client.replace(':', '.')
-            if response.session_id_name in request.cookies:
+            if response.session_id_name in cookies:
                 response.session_id = \
-                    request.cookies[response.session_id_name].value
+                    cookies[response.session_id_name].value
                 if regex_session_id.match(response.session_id):
                     response.session_filename = \
                         os.path.join(up(request.folder), masterapp,
@@ -532,38 +547,35 @@ class Session(Storage):
                 table_migrate = False
             tname = tablename + '_' + masterapp
             table = db.get(tname, None)
+            Field = db.Field
             if table is None:
-                table = db.define_table(
+                db.define_table(
                     tname,
-                    db.Field('locked', 'boolean', default=False),
-                    db.Field('client_ip', length=64),
-                    db.Field('created_datetime', 'datetime',
+                    Field('locked', 'boolean', default=False),
+                    Field('client_ip', length=64),
+                    Field('created_datetime', 'datetime',
                              default=request.now),
-                    db.Field('modified_datetime', 'datetime'),
-                    db.Field('unique_key', length=64),
-                    db.Field('session_data', 'blob'),
+                    Field('modified_datetime', 'datetime'),
+                    Field('unique_key', length=64),
+                    Field('session_data', 'blob'),
                     migrate=table_migrate,
                     )
+                table = db[tname] # to allow for lazy table
             try:
 
-                        # Get session data out of the database
-
-                        # Key comes from the cookie
-                key = request.cookies[response.session_id_name].value
+                # Get session data out of the database
+                # Key comes from the cookie
+                key = cookies[response.session_id_name].value
                 (record_id, unique_key) = key.split(':')
                 if record_id == '0':
                     raise Exception, 'record_id == 0'
-                        # Select from database.
+                        # Select from database
                 rows = db(table.id == record_id).select()
-
-                        # Make sure the session data exists in the database
+                # Make sure the session data exists in the database
                 if len(rows) == 0 or rows[0].unique_key != unique_key:
                     raise Exception, 'No record'
-
-                 # rows[0].update_record(locked=True)
-
-
-                    # Unpickle the data
+                # rows[0].update_record(locked=True)
+                # Unpickle the data
                 session_data = cPickle.loads(rows[0].session_data)
                 self.update(session_data)
             except Exception:
@@ -573,8 +585,9 @@ class Session(Storage):
             response._dbtable_and_field = \
                 (response.session_id_name, table, record_id, unique_key)
             response.session_id = '%s:%s' % (record_id, unique_key)
-        response.cookies[response.session_id_name] = response.session_id
-        response.cookies[response.session_id_name]['path'] = '/'
+        rcookies = response.cookies
+        rcookies[response.session_id_name] = response.session_id
+        rcookies[response.session_id_name]['path'] = '/'
         self.__hash = hashlib.md5(str(self)).digest()
         if self.flash:
             (response.flash, self.flash) = (self.flash, None)

gluon/html.py

@@ -296,7 +296,8 @@ def URL(
     if other.endswith('/'):
         other += '/'    # add trailing slash to make last trailing empty arg explicit
 
-    if vars.has_key('_signature'): vars.pop('_signature')
+    if '_signature' in vars:
+        vars.pop('_signature')
     list_vars = []
     for (key, vals) in sorted(vars.items()):
         if not isinstance(vals, (list, tuple)):
@@ -387,7 +388,7 @@ def verifyURL(request, hmac_key=None, hash_vars=True, salt=None, user_signature=
 
     """
 
-    if not request.get_vars.has_key('_signature'):
+    if not '_signature' in request.get_vars:
         return False # no signature in the request URL
 
     # check if user_signature requires
@@ -484,15 +485,17 @@ class XmlComponent(object):
             components += [other]
         return CAT(*components)
 
-    def add_class(self, name):
+    def add_class(self, name):        
         """ add a class to _class attribute """
-        classes = set(self['_class'].split())|set(name.split())
+        c = self['_class']
+        classes = (set(c.split()) if c else set())|set(name.split())
         self['_class'] = ' '.join(classes) if classes else None
         return self
 
     def remove_class(self, name):
         """ remove a class from _class attribute """
-        classes = set(self['_class'].split())-set(name.split())
+        c = self['_class']
+        classes = (set(c.split()) if c else set())-set(name.split())
         self['_class'] = ' '.join(classes) if classes else None
         return self
 
@@ -656,17 +659,17 @@ class DIV(XmlComponent):
         self.attributes = attributes
         self._fixup()
         # converts special attributes in components attributes
-        self._postprocessing()
         self.parent = None
         for c in self.components:
             self._setnode(c)
+        self._postprocessing()
 
     def update(self, **kargs):
         """
         dictionary like updating of the tag attributes
         """
 
-        for (key, value) in kargs.items():
+        for (key, value) in kargs.iteritems():
             self[key] = value
         return self
 
@@ -811,7 +814,9 @@ class DIV(XmlComponent):
                 c.latest = self.latest
                 c.session = self.session
                 c.formname = self.formname
-                if hideerror: c['hideerror'] = hideerror
+                if hideerror and not \
+                        self.attributes.get('hideerror',False):
+                    c['hideerror'] = hideerror
                 newstatus = c._traverse(status,hideerror) and newstatus
 
         # for input, textarea, select, option
@@ -1038,7 +1043,7 @@ class DIV(XmlComponent):
         tag = getattr(self,'tag').replace('/', '')
         if args and tag not in args:
             check = False
-        for (key, value) in kargs.items():
+        for (key, value) in kargs.iteritems():
             if key not in ['first_only', 'replace', 'find_text']:
                 if isinstance(value, (str, int)):
                     if self[key] != str(value):
@@ -1109,16 +1114,15 @@ class DIV(XmlComponent):
         sibs = [s for s in self.parent.components if not s == self]
         matches = []
         first_only = False
-        if kargs.has_key("first_only"):
-            first_only = kargs["first_only"]
-            del kargs["first_only"]
+        if 'first_only' in kargs:
+            first_only = kargs.pop('first_only')
         for c in sibs:
             try:
                 check = True
                 tag = getattr(c,'tag').replace("/","")
                 if args and tag not in args:
                         check = False
-                for (key, value) in kargs.items():
+                for (key, value) in kargs.iteritems():
                     if c[key] != value:
                             check = False
                 if check:
@@ -1682,14 +1686,14 @@ class INPUT(DIV):
         if name is None or name == '':
             return True
         name = str(name)
-
+        request_vars_get = self.request_vars.get
         if self['_type'] != 'checkbox':
             self['old_value'] = self['value'] or self['_value'] or ''
-            value = self.request_vars.get(name, '')
+            value = request_vars_get(name, '')
             self['value'] = value
         else:
             self['old_value'] = self['value'] or False
-            value = self.request_vars.get(name)
+            value = request_vars_get(name)
             if isinstance(value, (tuple, list)):
                 self['value'] = self['_value'] in value
             else:
@@ -1932,14 +1936,15 @@ class FORM(DIV):
         # check formname and formkey
 
         status = True
-        if self.session:
-            formkey = self.session.get('_formkey[%s]' % self.formname, None)
+        request_vars = self.request_vars
+        if session:
+            formkey = session.get('_formkey[%s]' % formname, None)
             # check if user tampering with form and void CSRF
-            if formkey != self.request_vars._formkey:
+            if formkey != request_vars._formkey:
                 status = False
-        if self.formname != self.request_vars._formname:
+        if formname != request_vars._formname:
             status = False
-        if status and self.session:
+        if status and session:
             # check if editing a record that has been modified by the server
             if hasattr(self,'record_hash') and self.record_hash != formkey:
                 status = False
@@ -1983,10 +1988,10 @@ class FORM(DIV):
 
     def hidden_fields(self):
         c = []
+        attr = self.attributes.get('hidden',{})
         if 'hidden' in self.attributes:
-            for (key, value) in self.attributes.get('hidden',{}).items():
-                c.append(INPUT(_type='hidden', _name=key, _value=value))
-
+            c = [INPUT(_type='hidden', _name=key, _value=value)
+                 for (key, value) in attr.iteritems()]
         if hasattr(self, 'formkey') and self.formkey:
             c.append(INPUT(_type='hidden', _name='_formkey',
                      _value=self.formkey))
@@ -2055,7 +2060,7 @@ class FORM(DIV):
                 onsuccess(self)
             if next:
                 if self.vars:
-                    for key,value in self.vars.items():
+                    for key,value in self.vars.iteritems():
                         next = next.replace('[%s]' % key,
                                             urllib.quote(str(value)))
                     if not next.startswith('/'):
@@ -2116,11 +2121,11 @@ class FORM(DIV):
         inputs = [INPUT(_type='button',
                         _value=name,
                         _onclick=FORM.REDIRECT_JS % link) \
-                      for name,link in buttons.items()]
+                      for name,link in buttons.iteritems()]
         inputs += [INPUT(_type='hidden',
                          _name=name,
                          _value=value)
-                   for name,value in hidden.items()]
+                   for name,value in hidden.iteritems()]
         form = FORM(INPUT(_type='submit',_value=text),*inputs)
         form.process()
         return form
@@ -2268,10 +2273,11 @@ class MENU(DIV):
             select = SELECT(**self.attributes)
         for item in data:
             if len(item) <= 4 or item[4] == True:
-                if item[2]:
-                    select.append(OPTION(CAT(prefix, item[0]), _value=item[2], _selected=item[1]))
-                    if len(item)>3 and len(item[3]):
-                        self.serialize_mobile(item[3], select, prefix = CAT(prefix, item[0], '/'))
+                select.append(OPTION(CAT(prefix, item[0]), 
+                                     _value=item[2], _selected=item[1]))
+                if len(item)>3 and len(item[3]):
+                    self.serialize_mobile(
+                        item[3], select, prefix = CAT(prefix, item[0], '/'))
         select['_onchange'] = 'window.location=this.value'
         return select
 
@@ -2323,7 +2329,7 @@ def test():
     >>> print form.accepts({'myvar':'34'}, formname=None)
     False
     >>> print form.xml()
-    <form action="" enctype="multipart/form-data" method="post"><input class="invalidinput" name="myvar" type="text" value="34" /><div class="error" id="myvar__error">invalid expression</div></form>
+    <form action="" enctype="multipart/form-data" method="post"><input class="invalidinput" name="myvar" type="text" value="34" /><div class="error_wrapper"><div class="error" id="myvar__error">invalid expression</div></div></form>
     >>> print form.accepts({'myvar':'4'}, formname=None, keepvalues=True)
     True
     >>> print form.xml()
@@ -2337,7 +2343,7 @@ def test():
     >>> print form.accepts({'myvar':'as df'}, formname=None)
     False
     >>> print form.xml()
-    <form action=\"\" enctype=\"multipart/form-data\" method=\"post\"><input class=\"invalidinput\" name=\"myvar\" type=\"text\" value=\"as df\" /><div class=\"error\" id=\"myvar__error\">only alphanumeric!</div></form>
+    <form action="" enctype="multipart/form-data" method="post"><input class="invalidinput" name="myvar" type="text" value="as df" /><div class="error_wrapper"><div class="error" id="myvar__error">only alphanumeric!</div></div></form>
     >>> session={}
     >>> form=FORM(INPUT(value=\"Hello World\", _name=\"var\", requires=IS_MATCH('^\w+$')))
     >>> if form.accepts({}, session,formname=None): print 'passed'

gluon/http.py

@@ -77,41 +77,45 @@ class HTTP(BaseException):
                 str(cookie)[11:] for cookie in cookies.values()]
 
     def to(self, responder):
-        if self.status in defined_status:
-            status = '%d %s' % (self.status, defined_status[self.status])
+        status = self.status
+        headers = self.headers
+        if status in defined_status:
+            status = '%d %s' % (status, defined_status[status])
         else:
-            status = str(self.status) + ' '
-        if not 'Content-Type' in self.headers:
-            self.headers['Content-Type'] = 'text/html; charset=UTF-8'
+            status = str(status) + ' '
+        if not 'Content-Type' in headers:
+            headers['Content-Type'] = 'text/html; charset=UTF-8'
         body = self.body
         if status[:1] == '4':
             if not body:
                 body = status
             if isinstance(body, str):
-                if len(body)<512 and self.headers['Content-Type'].startswith('text/html'):
+                if len(body)<512 and headers['Content-Type'].startswith('text/html'):
                     body += '<!-- %s //-->' % ('x'*512) ### trick IE
-                self.headers['Content-Length'] = len(body)
-        headers = []
-        for (k, v) in self.headers.items():
+                headers['Content-Length'] = len(body)
+        rheaders = []
+        for k, v in headers.iteritems():
             if isinstance(v, list):
-                for item in v:
-                    headers.append((k, str(item)))
+                rheaders += [(k, str(item)) for item in v]
             else:
-                headers.append((k, str(v)))
-        responder(status, headers)
-        if hasattr(body, '__iter__') and not isinstance(self.body, str):
+                rheaders.append((k, str(v)))
+        responder(status, rheaders)
+        if isinstance(body,str):
+            return [body]
+        elif hasattr(body, '__iter__'):
             return body
-        return [str(body)]
+        else:
+            return [str(body)]
 
     @property
     def message(self):
-        '''
+        """
         compose a message describing this exception
 
-        "status defined_status [web2py_error]"
+            "status defined_status [web2py_error]"
 
         message elements that are not defined are omitted
-        '''
+        """
         msg = '%(status)d'
         if self.status in defined_status:
             msg = '%(status)d %(defined_status)s'

gluon/languages.py

@@ -10,7 +10,7 @@ Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine)
                                <dbdevelop@gmail.com>
 """
 
-from os import path as ospath, stat as ostat, sep as osep
+import os
 import re
 from utf8 import Utf8
 from cgi import escape
@@ -20,7 +20,7 @@ import marshal
 import copy_reg
 from fileutils import abspath, listdir
 import settings
-from cfs import getcfs, cfs
+from cfs import getcfs
 from thread import allocate_lock
 from html import XML, xmlescape
 from contrib.markmin.markmin2html import render, markmin_escape
@@ -28,10 +28,37 @@ from string import maketrans
 
 __all__ = ['translator', 'findT', 'update_all_languages']
 
+ospath = os.path
+ostat = os.stat
+osep = os.sep
+isdir = os.path.isdir
+is_gae = settings.global_settings.web2py_runtime_gae
+
+DEFAULT_LANGUAGE = 'en'
+
+# DEFAULT PLURAL-FORMS RULES:
+# language doesn't use plural forms
+DEFAULT_NPLURALS = 1
+# only one singular/plural form is used
+DEFAULT_GET_PLURAL_ID = lambda n: 0
+# word is unchangeable
+DEFAULT_CONSTRUCTOR_PLURAL_FORM = lambda word, plural_id: word
+
+def safe_eval(text):
+    if text.strip():
+        try:
+            import ast
+            return ast.literal_eval(text)
+        except ImportError:
+            return eval(text,{},{})
+    return None
+
 # used as default filter in translator.M()
-markmin = lambda s: render( regex_param.sub(
-                       lambda m: '{' + markmin_escape(m.group('s')) + '}',
-                          s ), sep='br', autolinks=None, id_prefix='' )
+def markmin_aux(m):
+    return '{%s}' % markmin_escape(m.group('s'))
+def markmin(s):
+    return render(regex_param.sub(markmin_aux,s),
+                  sep='br', autolinks=None, id_prefix='')
 
 NUMBERS = (int,long,float)
 
@@ -49,28 +76,24 @@ regex_param=re.compile(r'{(?P<s>.+?)}')
 regex_language = \
     re.compile('^([a-zA-Z]{2})(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$')
 regex_langfile = re.compile('^[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
-regex_langinfo = re.compile("^[^'\"]*['\"]([^'\"]*)['\"]\s*:\s*['\"]([^'\"]*)['\"].*$")
 regex_backslash = re.compile(r"\\([\\{}%])")
 regex_plural = re.compile('%({.+?})')
 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)\)}$')  # %%{word(varname or number)}
 regex_plural_tuple = re.compile('^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word}
-regex_plural_q = re.compile('^asdf$') # %%{?word?cnt}, %%{??cnt} or %%{?cnt}
 regex_plural_rules = re.compile('^plural_rules-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
 
-upper_fun = lambda s: unicode(s,'utf-8').upper().encode('utf-8')
-title_fun = lambda s: unicode(s,'utf-8').title().encode('utf-8')
-cap_fun   = lambda s: unicode(s,'utf-8').capitalize().encode('utf-8')
-
-# DEFAULT PLURAL-FORMS RULES:
-default_nplurals = 1 # language doesn't use plural forms
-default_get_plural_id = lambda n: 0 # only one singular/plural form is used
-default_construct_plural_form = lambda word, plural_id: word # word is unchangeable
-
+# UTF8 helper functions
+def upper_fun(s):
+    return unicode(s,'utf-8').upper().encode('utf-8')
+def title_fun(s):
+    return unicode(s,'utf-8').title().encode('utf-8')
+def cap_fun(s):
+    return lambda s: unicode(s,'utf-8').capitalize().encode('utf-8')
 ttab_in  = maketrans("\\%{}", '\x1c\x1d\x1e\x1f')
 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
 
 # cache of translated messages:
-# of structure:
+# global_language_cache:
 # { 'languages/xx.py':
 #     ( {"def-message": "xx-message",
 #        ...
@@ -78,35 +101,38 @@ ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
 #  'languages/yy.py': ( {dict}, lock_object )
 #  ...
 # }
-tcache={}
+
+global_language_cache={}
 
 def get_from_cache(cache, val, fun):
-    lock=cache[1]
+    lang_dict, lock = cache
     lock.acquire()
     try:
-        result=cache[0].get(val);
+        result = lang_dict.get(val);
     finally:
         lock.release()
     if result:
         return result
     lock.acquire()
     try:
-        result=cache[0].setdefault(val, fun())
+        result = lang_dict.setdefault(val, fun())
     finally:
         lock.release()
     return result
 
-def clear_cache(cache):
-    lock=cache[1]
+def clear_cache(filename):
+    cache = global_language_cache.setdefault(
+        filename, ({}, allocate_lock()))
+    lang_dict, lock = cache
     lock.acquire()
     try:
-        cache[0].clear();
+        lang_dict.clear();
     finally:
         lock.release()
 
-
 def lang_sampling(lang_tuple, langlist):
-    """ search *lang_tuple* in *langlist*
+    """
+    search *lang_tuple* in *langlist*
 
     Args:
         lang_tuple (tuple of strings): ('aa'[[,'-bb'],'-cc'])
@@ -137,32 +163,28 @@ def lang_sampling(lang_tuple, langlist):
 
 
 def read_dict_aux(filename):
-    fp = portalocker.LockedFile(filename, 'r')
-    lang_text = fp.read().replace('\r\n', '\n')
-    fp.close()
-    # clear cache of processed messages:
-    clear_cache(tcache.setdefault(filename, ({}, allocate_lock())))
-    if not lang_text.strip():
-        return {}
+    lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
+    clear_cache(filename)
     try:
-        return eval(lang_text)
+        return safe_eval(lang_text) or {}
     except Exception, e:
-        status='Syntax error in %s (%s)' % (filename, e)
+        status = 'Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
         return {'__corrupted__':status}
 
-
 def read_dict(filename):
-    """ return dictionary with translation messages
+    """
+    return dictionary with translation messages
     """
     return getcfs('lang:'+filename, filename,
                 lambda: read_dict_aux(filename))
 
 
 def get_lang_info(lang, langdir):
-    """retrieve lang information from *langdir*/*lang*.py file.
-       Read few strings from lang.py file until keys !langname!,
-       !langcode! or keys greater then '!*' were found
+    """
+    retrieve lang information from *langdir*/*lang*.py file.
+    Read few strings from lang.py file until keys !langname!,
+    !langcode! or keys greater then '!*' were found
 
     args:
         lang (str): lang-code or 'default'
@@ -173,159 +195,78 @@ def get_lang_info(lang, langdir):
         e.g.: ('en', 'English', 1338549043.0)
     """
     filename = ospath.join(langdir, lang+'.py')
-    langcode=langname=''
-    f = portalocker.LockedFile(filename, 'r')
-    try:
-        while not (langcode and langname):
-            line = f.readline()
-            if not line:
-               break
-            match=regex_langinfo.match(line)
-            if match:
-                k = match.group(1)
-                if k == '!langname!':
-                    langname = match.group(2)
-                elif k == '!langcode!':
-                    langcode = match.group(2)
-                elif k[0:1] > '!':
-                    break
-    finally:
-        f.close()
-    if not langcode:
-        langcode = lang if lang != 'default' else 'en'
-    return langcode, langname or langcode, ostat(filename).st_mtime
+    d = read_dict(filename)    
+    langcode = d.get('!langcode!',DEFAULT_LANGUAGE)
+    langname = d.get('!langname!',langcode)
+    return (langcode, langname or langcode, ostat(filename).st_mtime)
 
-def read_possible_languages_aux(langdir):
+def read_possible_languages(appdir):
     langs = {}
     # scan languages directory for langfiles:
-    for langfile in [f for f in
-                      listdir(langdir, regex_langfile) +
-                      listdir(langdir, '^default\.py$')
-                      if osep not in f]:
-        lang=langfile[:-3]
-        langs[lang]=get_lang_info(lang, langdir)
-    if 'default' not in langs:
+    langdir = ospath.join(appdir,'languages')
+    for filename in os.listdir(langdir):
+        if regex_langfile.match(filename) or filename=='default.py':
+            lang = filename[:-3]
+            langs[lang] = get_lang_info(lang, langdir)
+    if not 'en' in langs:
         # if default.py is not found, add default value:
-        langs['default'] = ('en', 'English', 0)
-    deflang=langs['default']
-    if deflang[0] not in langs:
-        # create language from default.py:
-        langs[deflang[0]] = (deflang[0], deflang[1], 0)
+        langs['en'] = ('en', 'English', 0)
     return langs
 
-def read_possible_languages(path):
-    lang_path = ospath.join(path, 'languages')
-    return getcfs('langs:'+lang_path, lang_path,
-                   lambda: read_possible_languages_aux(lang_path))
-
-
-def read_plural_rules_aux(filename):
-    """retrieve plural rules from rules/*plural_rules-lang*.py file.
+def read_global_plural_rules(filename):
+    """
+    retrieve plural rules from rules/*plural_rules-lang*.py file.
 
     args:
         filename (str): plural_rules filename
 
     returns:
-        tuple(nplurals, get_plural_id, construct_plural_form)
-        e.g.: (3, <function>, <function>)
+        (nplurals, get_plural_id, construct_plural_form, status)
+        e.g.: (3, <function>, <function>, ok)
     """
-    f = portalocker.LockedFile(filename, 'r')
-    plural_py=f.read().replace('\r\n','\n')
-    f.close()
+    env = {}
+    data = portalocker.read_locked(filename)
     try:
-        exec(plural_py)
-        nplurals=locals().get('nplurals', default_nplurals)
-        get_plural_id=locals().get('get_plural_id', default_get_plural_id)
-        construct_plural_form=locals().get('construct_plural_form',
-                               default_construct_plural_form)
+        exec(data) in env
         status='ok'
     except Exception, e:
-        nplurals=default_nplurals
-        get_plural_id=default_get_plural_id
-        construct_plural_form=default_construct_plural_form
         status='Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
+    nplurals = env.get('nplurals', DEFAULT_NPLURALS)
+    get_plural_id = env.get('get_plural_id', DEFAULT_GET_PLURAL_ID)
+    construct_plural_form = env.get('construct_plural_form',
+                                    DEFAULT_CONSTRUCTOR_PLURAL_FORM)
     return (nplurals, get_plural_id, construct_plural_form, status)
 
-def read_plural_rules(lang):
-    filename = abspath('gluon','contrib','rules', 'plural_rules-%s.py' % lang)
-    return getcfs('plural_rules-'+lang, filename,
-               lambda: read_plural_rules_aux(filename))
 
-pcache={}
 def read_possible_plurals():
-    """ create list of all possible plural rules files
-        result is cached to increase speed
     """
-    global pcache
+    create list of all possible plural rules files
+    result is cached to increase speed
+    """
     pdir = abspath('gluon','contrib','rules')
     plurals = {}
     # scan rules directory for plural_rules-*.py files:
-    for pname in [f for f in listdir(pdir, regex_plural_rules)
-                  if osep not in f]:
-
-        lang=pname[13:-3]
-        fname=ospath.join(pdir, pname)
-        mtime=ostat(fname).st_mtime
-        if lang in pcache and pcache[lang][2] == mtime:
-            # if plural_file's mtime wasn't changed - use previous value:
-            plurals[lang]=pcache[lang]
-        else:
-            # otherwise, reread plural_rules-file:
-            if 'plural_rules-'+lang in cfs:
-               n,f1,f2,status=read_plural_rules(lang)
-            else:
-               n,f1,f2,status=read_plural_rules_aux(fname)
-            plurals[lang]=(n, pname, mtime, status)
-    pcache=plurals
-    return pcache
-
-def get_plural_rules(languages):
-    """get plural-forms rules for language *lang*
-       if rules not found - default rules will be return and lang=='unknown'
-
-    args:
-        lang (str): the languages, for one of which the plural-forms is return
-
-    returns:
-        tuples(lang, plural_rules-filename, nplurals,
-                        get_plural_id(), construct_plural_form(), status)
-    """
-    if isinstance(languages, str):
-       languages = [languages]
-
-    all_plurals=read_possible_plurals()
-    for lang in languages:
-        match_language = regex_language.match(lang.strip().lower())
-        if match_language:
-            match_language = tuple(part
-                                   for part in match_language.groups()
-                                   if part)
-            lang = lang_sampling(match_language, all_plurals.keys())
-            if lang:
-               ( nplurals,
-                  get_plural_id,
-                  construct_plural_form,
-                  status
-               ) = read_plural_rules(lang)
-               return (lang, all_plurals[lang][1], nplurals,
-                                                   get_plural_id,
-                                                   construct_plural_form,
-                                                   status)
-    return ('unknown', None, default_nplurals,
-                             default_get_plural_id,
-                             default_construct_plural_form,
-                             'ok')
+    for pname in os.listdir(pdir):
+        if not isdir(pname) and regex_plural_rules.match(pname):
+            lang = pname[13:-3]
+            fname = ospath.join(pdir, pname)
+            n, f1, f2, status = read_global_plural_rules(fname)
+            if status == 'ok':
+                plurals[lang] = (lang, n, f1, f2, pname)
+    plurals['default'] = ('default',
+                          DEFAULT_NPLURALS,
+                          DEFAULT_GET_PLURAL_ID,
+                          DEFAULT_CONSTRUCTOR_PLURAL_FORM,
+                          None)
+    return plurals
 
+PLURAL_RULES = read_possible_plurals()
 
 def read_plural_dict_aux(filename):
-    fp = portalocker.LockedFile(filename, 'r')
-    lang_text = fp.read().replace('\r\n', '\n')
-    fp.close()
-    if not lang_text.strip():
-        return {}
+    lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
     try:
-        return eval(lang_text)
+        return eval(lang_text) or {}
     except Exception, e:
         status='Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
@@ -335,7 +276,6 @@ def read_plural_dict(filename):
     return getcfs('plurals:'+filename, filename,
                       lambda: read_plural_dict_aux(filename))
 
-
 def write_plural_dict(filename, contents):
     if '__corrupted__' in contents:
         return
@@ -466,13 +406,10 @@ class lazyT(object):
         return lazyT(self.m, symbols, self.T, self.f, self.t, self.M)
 
 class translator(object):
-
     """
     this class is instantiated by gluon.compileapp.build_environment
     as the T object
-
     ::
-
         T.force(None) # turns off translation
         T.force('fr, it') # forces web2py to translate using fr.py or it.py
 
@@ -480,27 +417,21 @@ class translator(object):
 
     notice 1: there is no need to force since, by default, T uses
     http_accept_language to determine a translation file.
-
-    notice 2: en and en-en are considered different languages!
-
-    notice 3: if language xx-yy is not found force() probes other similar
-    languages using such algorithm: xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py
+    notice 2: 
+       en and en-en are considered different languages!
+    notice 3: 
+       if language xx-yy is not found force() probes other similar
+    languages using such algorithm: 
+        xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py
     """
 
     def __init__(self, request):
-        global tcache
         self.request = request
         self.folder = request.folder
-        dfile = ospath.join(self.folder,'languages','default.py')
-        if ospath.exists(dfile):
-            self.default_language_file = dfile
-            self.default_t = read_dict(dfile)
-        else: # languages/default.py is not found
-            self.default_language_file = ospath.join(self.folder, 'languages','')
-            self.default_t = {}
-        self.cache = tcache.setdefault(self.default_language_file, ({}, allocate_lock()))
-        self.current_languages = [self.get_possible_languages_info('default')[0]]
+        self.langpath = ospath.join(self.folder,'languages')
+        self.filenames = set(os.listdir(self.langpath))        
         self.http_accept_language = request.env.http_accept_language
+        # self.cache                        # filled in self.force()
         # self.accepted_language = None     # filled in self.force()
         # self.language_file = None         # filled in self.force()
         # self.plural_language = None       # filled in self.force()
@@ -511,12 +442,89 @@ class translator(object):
         # self.plural_file = None           # filled in self.force()
         # self.plural_dict = None           # filled in self.force()
         # self.plural_status = None         # filled in self.force()
-        self.requested_languages = self.force(self.http_accept_language)
+
+        self.requested_languages = \
+            self.force(self.http_accept_language)
         self.lazy = True
         self.otherTs = {}
         self.filter = markmin
         self.ftag = 'markmin'
 
+    def get_possible_languages(self):
+        return [lang[:-3] for lang in self.filenames \
+                    if regex_langfile.match(lang)]
+
+    def set_current_languages(self, *languages):
+        """
+        set current AKA "default" languages
+        setting one of this languages makes force() function
+        turn translation off to use default language
+        """
+        if len(languages) == 1 and isinstance(
+            languages[0], (tuple, list)):
+            languages = languages[0]
+        self.current_languages = languages
+        self.force(self.http_accept_language)
+
+    def set_plural(self, language):
+        """
+        initialize plural forms subsystem
+        invoked from self.force()
+        """
+        lang = language[:2] 
+        (self.plural_language,
+         self.nplurals,
+         self.get_plural_id,
+         self.construct_plural_form,
+         self.plural_filename
+         ) = PLURAL_RULES.get(language,PLURAL_RULES['default'])
+        for lang in (language, language[:5], language[:2]):
+            filename = 'plural-%s.py' % lang
+            if filename in self.filenames:
+                self.plural_file = ospath.join(self.langpath,filename)
+                self.plural_dict = read_plural_dict(self.plural_file)
+                break
+        else:
+            self.plural_file = None
+            self.plural_dict = {}
+ 
+    def plural(self, word, n):
+        """
+        get plural form of word for number *n*
+            NOTE: *word* MUST be defined in current language
+                  (T.accepted_language)
+
+            invoked from T()/M() in %%{} tag
+        args:
+            word (str): word in singular
+            n (numeric): number plural form created for
+
+        returns:
+            (str): word in appropriate singular/plural form
+        """
+        nplurals = self.nplurals
+        if int(n)==1:
+            return word
+        elif word:
+            id = min(int(n)-1,1) # self.get_plural_id(abs(int(n)))
+            # id = 0 first plural form
+            # id = 1 second plural form
+            # etc.
+            forms = self.plural_dict.get(word, [])
+            if len(forms)>=id:
+                # have this plural form
+                return forms[id-1]
+            else:
+                # guessing this plural form
+                forms += ['']*(nplurals-len(forms)-1)
+                form = self.construct_plural_form(word, id)
+                forms[id-1] = form
+                self.plural_dict[word] = forms
+                if self.plural_file and not is_gae:
+                    write_plural_dict(self.plural_file,
+                                      self.plural_dict)
+                return form
+
     def get_possible_languages_info(self, lang=None):
         """
         return info for selected language or dictionary with all
@@ -533,90 +541,12 @@ class translator(object):
                                          langname(from !langname! key),
                                          langfile_mtime ) }
         """
-        if lang:
-            return read_possible_languages(self.folder).get(lang)
-        return read_possible_languages(self.folder)
-
-    def get_possible_languages(self):
-        """ get list of all possible languages for current applications """
-        return sorted( set(lang for lang in
-                           read_possible_languages(self.folder).iterkeys()
-                           if lang != 'default')
-                     | set(self.current_languages))
-
-    def set_current_languages(self, *languages):
-        """
-        set current AKA "default" languages
-        setting one of this languages makes force() function
-        turn translation off to use default language
-        """
-        if len(languages) == 1 and isinstance(languages[0], (tuple, list)):
-            languages = languages[0]
-        self.current_languages = languages
-        self.force(self.http_accept_language)
-
-    def set_plural(self, languages):
-        """ initialize plural forms subsystem
-            invoked from self.force()
-        """
-        ( self.plural_language,
-          self.plural_rules_file,
-          self.nplurals,
-          self.get_plural_id,
-          self.construct_plural_form,
-          self.plural_status
-        ) = get_plural_rules(languages)
-
-        if self.plural_language == 'unknown':
-            self.plural_file = None
-            self.plural_dict = {}
-        else:
-            self.plural_file = ospath.join(self.folder,
-                                           'languages',
-                                           'plural-%s.py' % self.plural_language)
-            if ospath.exists(self.plural_file):
-                self.plural_dict = read_plural_dict(self.plural_file)
-            else:
-                self.plural_dict = {}
-
-    def plural(self, word, n):
-        """ get plural form of word for number *n*
-            NOTE: *word" MUST be defined in current language
-                  (T.accepted_language)
-
-            invoked from T()/M() in %%{} tag
-        args:
-            word (str): word in singular
-            n (numeric): number plural form created for
-
-        returns:
-            (str): word in appropriate singular/plural form
-        """
-        nplurals = self.nplurals
-        if word:
-            id = self.get_plural_id(abs(int(n)))
-            if id > 0:
-                forms = self.plural_dict.get(word, [])
-                if forms:
-                    try:
-                        form = forms[id-1]
-                    except:
-                        form = None
-                    if form: return form
-                form = self.construct_plural_form(word, id)
-                if len(forms) < nplurals-1:
-                    forms.extend('' for i in xrange(nplurals-len(forms)-1))
-                forms[id-1] = form
-                self.plural_dict[word] = forms
-                if (self.plural_file and
-                       not settings.global_settings.web2py_runtime_gae):
-                    write_plural_dict(self.plural_file, self.plural_dict)
-                return form
-        return word
+        info = read_possible_languages(self.folder)
+        if lang: info = info.get(lang)
+        return info
 
     def force(self, *languages):
         """
-
         select language(s) for translation
 
         if a list of languages is passed as a parameter,
@@ -627,49 +557,41 @@ class translator(object):
         default language will be selected if none
         of them matches possible_languages.
         """
-        global tcache
         language = ''
-
-        if not languages or languages[0] is None:
+        if isinstance(languages,str):
+            languages = regex_language.findall(languages.lower())
+        elif not languages or languages[0] is None:
             languages = []
-        if len(languages) == 1 and isinstance(languages[0], (str, unicode)):
-            languages = languages[0]
-
-        if languages:
-            if isinstance(languages, (str, unicode)):
-                parts = languages.split(';')
-                languages = []
-                for al in parts:
-                    languages.extend(al.split(','))
-
-            possible_languages = self.get_possible_languages()
-            for lang in languages:
-                match_language = regex_language.match(lang.strip().lower())
-                if match_language:
-                    match_language = tuple(part
-                                           for part in match_language.groups()
-                                           if part)
-                    language = lang_sampling(match_language,
-                                             self.current_languages)
-                    if language:
-                        break
-                    language = lang_sampling(match_language, possible_languages)
-                    if language:
-                        self.language_file = ospath.join(self.folder,
-                                                         'languages',
-                                                         language + '.py')
-                        if ospath.exists(self.language_file):
-                            self.t = read_dict(self.language_file)
-                            self.cache = tcache.setdefault(self.language_file,
-                                                           ({},allocate_lock()))
-                            self.set_plural(language)
-                            self.accepted_language = language
-                            return languages
-        self.accepted_language = language or self.current_languages[0]
-        self.language_file = self.default_language_file
-        self.cache = tcache[self.language_file]
-        self.t = self.default_t
-        self.set_plural(language or self.current_languages)
+        for lang in languages:
+            if lang+'.py' in self.filenames:
+                language = lang
+                langfile = language+'.py'
+                break
+            elif len(lang)>5 and lang[:5]+'.py' in self.filenames:
+                language = lang[:5]
+                langfile = language+'.py'
+                break
+            elif len(lang)>2 and lang[:2]+'.py' in self.filenames:
+                language = lang[:2]
+                langfile = language+'.py'
+                break
+        else:
+            if 'default.py' in self.filenames:
+                language = DEFAULT_LANGUAGE
+                langfile = 'default.py'
+            else:
+                language = DEFAULT_LANGUAGE
+                langfile = None
+        self.accepted_language = language
+        if langfile:
+            self.language_file = ospath.join(self.langpath,langfile)
+            self.t = read_dict(self.language_file)
+        else:
+            self.language_file = None
+            self.t = {}
+        self.cache = global_language_cache.setdefault(
+            self.language_file,({},allocate_lock()))
+        self.set_plural(language)
         return languages
 
     def __call__(self, message, symbols={}, language=None, lazy=None):
@@ -700,26 +622,30 @@ class translator(object):
             prefix = '@'+(ftag or 'userdef')+'\x01'
         else:
             prefix = '@'+self.ftag+'\x01'
-        message = get_from_cache(self.cache, prefix+message,
-                                 lambda: get_tr(message, prefix, filter))
+        message = get_from_cache(
+            self.cache, prefix+message,
+            lambda: get_tr(message, prefix, filter))
         if symbols or symbols == 0 or symbols == "":
             if isinstance(symbols, dict):
-                symbols.update( (key, xmlescape(value).translate(ttab_in))
-                                for key, value in symbols.iteritems()
-                                 if not isinstance(value, NUMBERS) )
+                symbols.update(
+                    (key, xmlescape(value).translate(ttab_in))
+                    for key, value in symbols.iteritems()
+                    if not isinstance(value, NUMBERS) )
             else:
                 if not isinstance(symbols, tuple):
                     symbols = (symbols,)
-                symbols = tuple(value if isinstance(value, NUMBERS)
-                                    else xmlescape(value).translate(ttab_in)
-                                     for value in symbols)
+                symbols = tuple(
+                    value if isinstance(value, NUMBERS)
+                    else xmlescape(value).translate(ttab_in)
+                    for value in symbols)
             message = self.params_substitution(message, symbols)
         return XML(message.translate(ttab_out))
 
-    def M(self, message, symbols={}, language=None, lazy=None, filter=None, ftag=None):
-        """ get cached translated markmin-message with inserted parametes
-
-            if lazy==True lazyT object is returned
+    def M(self, message, symbols={}, language=None, 
+          lazy=None, filter=None, ftag=None):
+        """
+        get cached translated markmin-message with inserted parametes
+        if lazy==True lazyT object is returned
         """
         if lazy is None:
             lazy = self.lazy
@@ -751,18 +677,20 @@ class translator(object):
         """
         key = prefix+message
         mt = self.t.get(key, None)
-        if mt is not None:
-            return mt
-        if not message.startswith('##') and not '\n' in message:
-            tokens = message.rsplit('##', 1)
-        else:
-            # this allows markmin syntax in translations
-            tokens = [message]
-        self.t[key] = mt = self.default_t.get(key, tokens[0])
-        if (self.language_file != self.default_language_file and
-             not settings.global_settings.web2py_runtime_gae):
-            write_dict(self.language_file, self.t)
-        return  regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), mt)
+        if mt is None:
+            # we did not find a translation
+            if message.find('##')>0 and not '\n' in message:
+                # remove comments
+                message = message.rsplit('##', 1)[0]
+            # guess translation same as original
+            self.t[key] = mt = message
+            # update language file for later translation
+            if self.language_file and not is_gae:
+                write_dict(self.language_file, self.t)
+            # fix backslash escaping
+            mt = regex_backslash.sub(
+                lambda m: m.group(1).translate(ttab_in), mt)
+        return mt
 
     def params_substitution(self, message, symbols):
         """
@@ -866,19 +794,21 @@ class translator(object):
         """
         get cached translated message with inserted parameters(symbols)
         """
-        message = get_from_cache(self.cache, message, lambda: self.get_t(message))
+        message = get_from_cache(self.cache, message, 
+                                 lambda: self.get_t(message))
         if symbols or symbols == 0 or symbols == "":
             if isinstance(symbols, dict):
-                symbols.update( (key, str(value).translate(ttab_in))
-                                for key, value in symbols.iteritems()
-                                 if not isinstance(value, NUMBERS) )
+                symbols.update(
+                    (key, str(value).translate(ttab_in))
+                    for key, value in symbols.iteritems()
+                    if not isinstance(value, NUMBERS) )
             else:
                 if not isinstance(symbols, tuple):
                     symbols = (symbols,)
-                symbols = tuple(value if isinstance(value, NUMBERS)
-                                    else str(value).translate(ttab_in)
-                                     for value in symbols)
-
+                symbols = tuple(
+                    value if isinstance(value, NUMBERS)
+                    else str(value).translate(ttab_in)
+                    for value in symbols)
             message = self.params_substitution(message, symbols)
         return message.translate(ttab_out)
 
@@ -892,26 +822,25 @@ def findT(path, language='en'):
     cp = ospath.join(path, 'controllers')
     vp = ospath.join(path, 'views')
     mop = ospath.join(path, 'modules')
-    for file in listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\
-         + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0):
-        fp = portalocker.LockedFile(file, 'r')
-        data = fp.read()
-        fp.close()
+    for filename in \
+            listdir(mp, '^.+\.py$', 0)+listdir(cp, '^.+\.py$', 0)\
+            +listdir(vp, '^.+\.html$', 0)+listdir(mop, '^.+\.py$', 0):
+        data = portalocker.read_locked(filename)
         items = regex_translate.findall(data)
         for item in items:
             try:
-                message = eval(item)
-                if not message.startswith('#') and not '\n' in message:
-                    tokens = message.rsplit('##', 1)
-                else:
-                    # this allows markmin syntax in translations
-                    tokens = [message]
-                if len(tokens) == 2:
-                    message = tokens[0].strip() + '##' + tokens[1].strip()
-                if message and not message in sentences:
-                    sentences[message] = message
+                message = safe_eval(item)
             except:
-                pass
+                continue # silently ignore inproperly formatted strings
+            if not message.startswith('#') and not '\n' in message:
+                tokens = message.rsplit('##', 1)
+            else:
+                # this allows markmin syntax in translations
+                tokens = [message]
+            if len(tokens) == 2:
+                message = tokens[0].strip()+'##'+tokens[1].strip()
+            if message and not message in sentences:
+                sentences[message] = message
     if not '!langcode!' in sentences:
         sentences['!langcode!'] = (
             'en' if language in ('default', 'en') else language)
@@ -937,6 +866,3 @@ def update_all_languages(application_path):
 if __name__ == '__main__':
     import doctest
     doctest.testmod()
-
-
-

gluon/main.py

@@ -21,7 +21,6 @@ import re
 import copy
 import sys
 import time
-import thread
 import datetime
 import signal
 import socket
@@ -29,6 +28,7 @@ import tempfile
 import random
 import string
 import urllib2
+from thread import allocate_lock
 
 from fileutils import abspath, write_file, parse_version
 from settings import global_settings
@@ -69,8 +69,11 @@ import logging.config
 import gluon.messageboxhandler
 logging.gluon = gluon
 
+exists = os.path.exists
+pjoin = os.path.join
+
 logpath = abspath("logging.conf")
-if os.path.exists(logpath):
+if exists(logpath):
     logging.config.fileConfig(abspath("logging.conf"))
 else:
     logging.basicConfig()
@@ -87,10 +90,10 @@ from dal import BaseAdapter
 from settings import global_settings
 from validators import CRYPT
 from cache import Cache
-from html import URL as Url, xmlescape
+from html import URL, xmlescape
 from utils import is_valid_ip_address
+from rewrite import load, url_in, thread as rwthread, try_rewrite_on_error
 import newcron
-import rewrite
 
 __all__ = ['wsgibase', 'save_password', 'appfactory', 'HttpServer']
 
@@ -103,7 +106,7 @@ requests = 0    # gc timer
 regex_client = re.compile('[\w\-:]+(\.[\w\-]+)*\.?')  # ## to account for IPV6
 
 try:
-    version_info = open(os.path.join(global_settings.gluon_parent, 'VERSION'), 'r')
+    version_info = open(pjoin(global_settings.gluon_parent, 'VERSION'), 'r')
     raw_version_string = version_info.read().strip()
     version_info.close()
     global_settings.web2py_version = parse_version(raw_version_string)
@@ -118,7 +121,7 @@ except:
     if not global_settings.web2py_runtime_gae:
         logger.warn('unable to import Rocket')
 
-rewrite.load()
+load()
 
 def get_client(env):
     """
@@ -129,11 +132,16 @@ def get_client(env):
     """
     g = regex_client.search(env.get('http_x_forwarded_for', ''))
     if g:
-        return g.group()
-    g = regex_client.search(env.get('remote_addr', ''))
-    if g:
-        return g.group()
-    return '127.0.0.1'
+        client = g.group()
+    else:
+        g = regex_client.search(env.get('remote_addr', ''))
+        if g:
+            client = g.group()
+        else:
+            client = '127.0.0.1'
+    if not is_valid_ip_address(client):
+        raise HTTP(400,"Bad Request (request.client=%s)" % client)
+    return client
 
 def copystream_progress(request, chunk_size= 10**5):
     """
@@ -253,7 +261,8 @@ def middleware_aux(request, response, *middleware_apps):
         for item in middleware_apps:
             app=item(app)
         def caller(app):
-            return app(request.wsgi.environ,request.wsgi.start_response)
+            wsgi = request.wsgi
+            return app(wsgi.environ, wsgi.start_response)
         return lambda caller=caller, app=app: caller(app)
     return middleware
 
@@ -279,14 +288,14 @@ def parse_get_post_vars(request, environ):
 
     # parse POST variables on POST, PUT, BOTH only in post_vars
     try:
-        request.body = copystream_progress(request) ### stores request body
+        request.body = body = copystream_progress(request)
     except IOError:
         raise HTTP(400,"Bad Request - HTTP body is incomplete")
-    if (request.body and request.env.request_method in ('POST', 'PUT', 'BOTH')):
-        dpost = cgi.FieldStorage(fp=request.body,environ=environ,keep_blank_values=1)
+    if (body and request.env.request_method in ('POST', 'PUT', 'BOTH')):
+        dpost = cgi.FieldStorage(fp=body,environ=environ,keep_blank_values=1)
         # The same detection used by FieldStorage to detect multipart POSTs
         is_multipart = dpost.type[:10] == 'multipart/'
-        request.body.seek(0)
+        body.seek(0)
         isle25 = sys.version_info[1] <= 5
 
         def listify(a):
@@ -357,9 +366,10 @@ def wsgibase(environ, responder):
     request = Request()
     response = Response()
     session = Session()
-    request.env.web2py_path = global_settings.applications_parent
-    request.env.web2py_version = web2py_version
-    request.env.update(global_settings)
+    env = request.env
+    env.web2py_path = global_settings.applications_parent
+    env.web2py_version = web2py_version
+    env.update(global_settings)
     static_file = False
     try:
         try:
@@ -373,79 +383,80 @@ def wsgibase(environ, responder):
                 # serve file if static
                 # ##################################################
 
-                if not environ.get('PATH_INFO',None) and \
-                        environ.get('REQUEST_URI',None):
-                    # for fcgi, get path_info and query_string from request_uri
+                eget = environ.get
+                if not eget('PATH_INFO',None) and eget('REQUEST_URI',None):
+                    # for fcgi, get path_info and 
+                    # query_string from request_uri
                     items = environ['REQUEST_URI'].split('?')
                     environ['PATH_INFO'] = items[0]
                     if len(items) > 1:
                         environ['QUERY_STRING'] = items[1]
                     else:
                         environ['QUERY_STRING'] = ''
-                if not environ.get('HTTP_HOST',None):
-                    environ['HTTP_HOST'] = '%s:%s' % (environ.get('SERVER_NAME'),
-                                                      environ.get('SERVER_PORT'))
+                if not eget('HTTP_HOST',None):
+                    environ['HTTP_HOST'] = \
+                        eget('SERVER_NAME')+':'+eget('SERVER_PORT')
+                    
+
+                (static_file, environ) = url_in(request, environ)
 
-                (static_file, environ) = rewrite.url_in(request, environ)
                 if static_file:
-                    if environ.get('QUERY_STRING', '')[:10] == 'attachment':
-                        response.headers['Content-Disposition'] = 'attachment'
+                    if environ.get('QUERY_STRING','').startswith(
+                        'attachment'):
+                        response.headers['Content-Disposition'] \
+                            = 'attachment'
                     response.stream(static_file, request=request)
 
                 # ##################################################
                 # fill in request items
                 # ##################################################
-
-                http_host = request.env.http_host.split(':',1)[0]
-
-                local_hosts = [http_host,'::1','127.0.0.1','::ffff:127.0.0.1']
+                app = request.application ## must go after url_in!
+                                
+                http_host = env.http_host.split(':',1)[0]
+                local_hosts = [http_host,'::1','127.0.0.1',
+                               '::ffff:127.0.0.1']
                 if not global_settings.web2py_runtime_gae:
                     local_hosts.append(socket.gethostname())
-                    try: local_hosts.append(socket.gethostbyname(http_host))
-                    except socket.gaierror: pass
-                request.client = get_client(request.env)
-                if not is_valid_ip_address(request.client):
-                    raise HTTP(400,"Bad Request (request.client=%s)" % \
-                                   request.client)
-                request.folder = abspath('applications',
-                                         request.application) + os.sep
-                x_req_with = str(request.env.http_x_requested_with).lower()
-                request.ajax = x_req_with == 'xmlhttprequest'
-                request.cid = request.env.http_web2py_component_element
-                request.is_local = request.env.remote_addr in local_hosts
-                request.is_https = request.env.wsgi_url_scheme \
-                    in ['https', 'HTTPS'] or request.env.https == 'on'
-
-                # ##################################################
-                # compute a request.uuid to be used for tickets and toolbar
-                # ##################################################
-
-                response.uuid = request.compute_uuid()
+                    try:
+                        local_hosts.append(
+                        socket.gethostbyname(http_host))
+                    except socket.gaierror:
+                        pass
+                client = get_client(env)                
+                x_req_with = str(env.http_x_requested_with).lower()
+                
+                request.update(dict(
+                        client = client,
+                        folder = abspath('applications',app) + os.sep,
+                        ajax = x_req_with == 'xmlhttprequest',
+                        cid = env.http_web2py_component_element,
+                        is_local = env.remote_addr in local_hosts,
+                        is_https = env.wsgi_url_scheme \
+                            in ['https', 'HTTPS'] or env.https=='on'))
+                request.uuid = request.compute_uuid() # requires client
 
                 # ##################################################
                 # access the requested application
                 # ##################################################
 
-                if not os.path.exists(request.folder):
-                    if request.application == \
-                            rewrite.thread.routes.default_application \
-                            and request.application != 'welcome':
-                        request.application = 'welcome'
-                        redirect(Url(r=request))
-                    elif rewrite.thread.routes.error_handler:
-                        _handler = rewrite.thread.routes.error_handler
-                        redirect(Url(_handler['application'],
+                if not exists(request.folder):
+                    if app == rwthread.routes.default_application \
+                            and app != 'welcome':
+                        redirect(URL('welcome','default','index'))
+                    elif rwthread.routes.error_handler:
+                        _handler = rwthread.routes.error_handler
+                        redirect(URL(_handler['application'],
                                      _handler['controller'],
                                      _handler['function'],
-                                     args=request.application))
+                                     args=app))
                     else:
-                        raise HTTP(404, rewrite.thread.routes.error_message \
+                        raise HTTP(404, rwthread.routes.error_message \
                                        % 'invalid request',
                                    web2py_error='invalid application')
                 elif not request.is_local and \
-                        os.path.exists(os.path.join(request.folder,'DISABLED')):
+                        exists(pjoin(request.folder,'DISABLED')):
                     raise HTTP(503, "<html><body><h1>Temporarily down for maintenance</h1></body></html>")
-                request.url = Url(r=request,
+                request.url = URL(r=request,
                                   args=request.args,
                                   extension=request.raw_extension)
 
@@ -492,22 +503,23 @@ def wsgibase(environ, responder):
                 # ##################################################
                 # set no-cache headers
                 # ##################################################
-
-                response.headers['Content-Type'] = \
+                
+                headers = response.headers
+                headers['Content-Type'] = \
                     contenttype('.'+request.extension)
-                response.headers['Cache-Control'] = \
+                headers['Cache-Control'] = \
                     'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
-                response.headers['Expires'] = \
+                headers['Expires'] = \
                     time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime())
-                response.headers['Pragma'] = 'no-cache'
+                headers['Pragma'] = 'no-cache'
 
                 # ##################################################
                 # run controller
                 # ##################################################
 
-                if global_settings.debugging and request.application != "admin":
+                if global_settings.debugging and app != "admin":
                     import gluon.debug
-                    # activate the debugger and wait to reach application code
+                    # activate the debugger
                     gluon.debug.dbg.do_debug(mainpyfile=request.folder)
 
                 serve_controller(request, response, session)
@@ -549,22 +561,24 @@ def wsgibase(environ, responder):
                 # ##################################################
 
                 if request.cid:
-                    if response.flash and not 'web2py-component-flash' \
-                            in http_response.headers:
-                        http_response.headers['web2py-component-flash'] = \
+                    rheaders
+                    if response.flash and \
+                            not 'web2py-component-flash' in rheaders:
+                        rheaders['web2py-component-flash'] = \
                             urllib2.quote(xmlescape(response.flash)\
                                               .replace('\n',''))
-                    if response.js and not 'web2py-component-command' \
-                            in http_response.headers:
-                        http_response.headers['web2py-component-command'] = \
+                    if response.js and \
+                            not 'web2py-component-command' in readers:
+                        readers['web2py-component-command'] = \
                             response.js.replace('\n','')
+                rcookies = response.cookies
                 if session._forget and \
                         response.session_id_name in response.cookies:
-                    del response.cookies[response.session_id_name]
+                    del rcookies[response.session_id_name]
                 elif session._secure:
-                    response.cookies[response.session_id_name]['secure'] = True
+                    rcookies[response.session_id_name]['secure'] = True
 
-                http_response.cookies2headers(response.cookies)
+                http_response.cookies2headers(rcookies)
                 ticket=None
 
             except RestrictedError, e:
@@ -583,7 +597,7 @@ def wsgibase(environ, responder):
                     BaseAdapter.close_all_instances('rollback')
 
                 http_response = \
-                    HTTP(500, rewrite.thread.routes.error_message_ticket % \
+                    HTTP(500, rwthread.routes.error_message_ticket % \
                              dict(ticket=ticket),
                          web2py_error='ticket %s' % ticket)
 
@@ -606,7 +620,7 @@ def wsgibase(environ, responder):
             e = RestrictedError('Framework', '', '', locals())
             ticket = e.log(request) or 'unrecoverable'
             http_response = \
-                HTTP(500, rewrite.thread.routes.error_message_ticket \
+                HTTP(500, rwthread.routes.error_message_ticket \
                          % dict(ticket=ticket),
                      web2py_error='ticket %s' % ticket)
 
@@ -616,7 +630,7 @@ def wsgibase(environ, responder):
             response.session_file.close()
 
     session._unlock(response)
-    http_response, new_environ = rewrite.try_rewrite_on_error(
+    http_response, new_environ = try_rewrite_on_error(
         http_response, request, environ, ticket)
     if not http_response:
         return wsgibase(new_environ,responder)
@@ -641,7 +655,7 @@ def save_password(password, port):
         print '*********************************************************'
     elif password == '<recycle>':
         # reuse the current password if any
-        if os.path.exists(password_file):
+        if exists(password_file):
             return
         else:
             password = ''
@@ -672,9 +686,9 @@ def appfactory(wsgiapp=wsgibase,
             [, profilerfilename='profiler.log']]])
 
     """
-    if profilerfilename and os.path.exists(profilerfilename):
+    if profilerfilename and exists(profilerfilename):
         os.unlink(profilerfilename)
-    locker = thread.allocate_lock()
+    locker = allocate_lock()
 
     def app_with_logging(environ, responder):
         """
@@ -785,7 +799,7 @@ class HttpServer(object):
             os.chdir(path)
             [add_path_first(p) for p in (path, abspath('site-packages'), "")]
             custom_import_install(web2py_path)
-            if os.path.exists("logging.conf"):
+            if exists("logging.conf"):
                 logging.config.fileConfig("logging.conf")
 
         save_password(password, port)
@@ -800,9 +814,9 @@ class HttpServer(object):
             logger.info('SSL is off')
         elif not rocket.ssl:
             logger.warning('Python "ssl" module unavailable. SSL is OFF')
-        elif not os.path.exists(ssl_certificate):
+        elif not exists(ssl_certificate):
             logger.warning('unable to open SSL certificate. SSL is OFF')
-        elif not os.path.exists(ssl_private_key):
+        elif not exists(ssl_private_key):
             logger.warning('unable to open SSL private key. SSL is OFF')
         else:
             sock_list.extend([ssl_private_key, ssl_certificate])
@@ -848,9 +862,3 @@ class HttpServer(object):
         except:
             pass
 
-
-
-
-
-
-

gluon/portalocker.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# portalocker.py - Cross-platform (posix/nt) API for flock-style file locking.
+# portalocker.py 
+# Cross-platform (posix/nt) API for flock-style file locking.
 #                  Requires python 1.5.2 or better.
 
 """
@@ -141,6 +142,17 @@ class LockedFile(object):
     def __del__(self):
         self.close()
 
+def read_locked(filename):
+    fp = LockedFile(filename, 'r')
+    data = fp.read()
+    fp.close()
+    return data
+
+def write_locked(filename,data):
+    fp = LockedFile(filename, 'w')
+    data = fp.write(data)
+    fp.close()
+
 if __name__=='__main__':
     f = LockedFile('test.txt',mode='wb')
     f.write('test ok')

gluon/restricted.py

@@ -88,16 +88,19 @@ class TicketStorage(Storage):
         ticket_id,
         ):
         if not self.db:
-            ef = self._error_file(request, ticket_id, 'rb', app)
+            try:
+                ef = self._error_file(request, ticket_id, 'rb', app)
+            except IOError:
+                return {}
             try:
                 return cPickle.load(ef)
             finally:
                 ef.close()
-        table = self._get_table(self.db, self.tablename, app)
-        rows = self.db(table.ticket_id == ticket_id).select()
-        if rows:
-            return cPickle.loads(rows[0].ticket_data)
-        return None
+        else:
+            table = self._get_table(self.db, self.tablename, app)
+            rows = self.db(table.ticket_id == ticket_id).select()
+            return cPickle.loads(rows[0].ticket_data) if rows else {}
+            
 
 
 class RestrictedError(Exception):
@@ -164,10 +167,10 @@ class RestrictedError(Exception):
         ticket_storage = TicketStorage(db=request.tickets_db)
         d = ticket_storage.load(request, app, ticket_id)
 
-        self.layer = d['layer']
-        self.code = d['code']
-        self.output = d['output']
-        self.traceback = d['traceback']
+        self.layer = d.get('layer')
+        self.code = d.get('code')
+        self.output = d.get('output')
+        self.traceback = d.get('traceback')
         self.snapshot = d.get('snapshot')
 
     def __str__(self):

gluon/rewrite.py

@@ -27,9 +27,17 @@ from http import HTTP
 from fileutils import abspath, read_file
 from settings import global_settings
 
-logger = logging.getLogger('web2py.rewrite')
+isdir = os.path.isdir
+isfile = os.path.isfile
+exists = os.path.exists
+pjoin = os.path.join
 
-thread = threading.local()  # thread-local storage for routing parameters
+logger = logging.getLogger('web2py.rewrite')
+thread = threading.local()  # thread-local storage for routing params
+
+regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
+regex_anything = re.compile(r'(?<!\\)\$anything')
+regex_redirect = re.compile(r'(\d+)->(.*)')
 
 def _router_default():
     "return new copy of default base router"
@@ -75,7 +83,7 @@ def _params_default(app=None):
 
 params_apps = dict()
 params = _params_default(app=None)  # regex rewrite parameters
-thread.routes = params              # default to base regex rewrite parameters
+thread.routes = params  # default to base regex rewrite parameters
 routers = None
 
 def log_rewrite(string):
@@ -97,13 +105,18 @@ def log_rewrite(string):
     else:
         logger.debug(string)
 
-ROUTER_KEYS = set(('default_application', 'applications', 'default_controller', 'controllers',
-    'default_function', 'functions', 'default_language', 'languages',
-    'domain', 'domains', 'root_static', 'path_prefix',
-    'exclusive_domain', 'map_hyphen', 'map_static',
-    'acfe_match', 'file_match', 'args_match'))
+ROUTER_KEYS = set(
+    ('default_application', 'applications', 
+     'default_controller', 'controllers',
+     'default_function', 'functions', 
+     'default_language', 'languages',
+     'domain', 'domains', 'root_static', 'path_prefix',
+     'exclusive_domain', 'map_hyphen', 'map_static',
+     'acfe_match', 'file_match', 'args_match'))
 
-ROUTER_BASE_KEYS = set(('applications', 'default_application', 'domains', 'path_prefix'))
+ROUTER_BASE_KEYS = set(
+    ('applications', 'default_application', 
+     'domains', 'path_prefix'))
 
 #  The external interface to rewrite consists of:
 #
@@ -223,9 +236,7 @@ def try_redirect_on_error(http_object, request, ticket=None):
                         (redir,status,ticket,
                          urllib.quote_plus(request.env.request_uri),
                          request.url)
-                return HTTP(303,
-                            'You are being redirected <a href="%s">here</a>' % url,
-                            Location=url)
+                return HTTP(303,'You are being redirected <a href="%s">here</a>' % url,Location=url)
     return http_object
 
 
@@ -258,7 +269,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
                 path = abspath(routes)
             else:
                 path = abspath('applications', app, routes)
-            if not os.path.exists(path):
+            if not exists(path):
                 return
             data = read_file(path).replace('\r\n','\n')
 
@@ -309,9 +320,11 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
         #    parse the app-specific routes.py if present
         #
         all_apps = []
-        for appname in [app for app in os.listdir(abspath('applications')) if not app.startswith('.')]:
-            if os.path.isdir(abspath('applications', appname)) and \
-               os.path.isdir(abspath('applications', appname, 'controllers')):
+        apppath = abspath('applications')
+        for appname in os.listdir(apppath):
+            if not appname.startswith('.') and \
+                    isdir(abspath(apppath,appname)) and \
+                    isdir(abspath(apppath,appname,'controllers')):
                 all_apps.append(appname)
                 if routers:
                     router = Storage(routers.BASE)   # new copy
@@ -321,7 +334,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
                                 raise SyntaxError, "BASE-only key '%s' in router '%s'" % (key, appname)
                         router.update(routers[appname])
                     routers[appname] = router
-                if os.path.exists(abspath('applications', appname, routes)):
+                if exists(abspath('applications', appname, routes)):
                     load(routes, appname)
 
         if routers:
@@ -336,14 +349,9 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
     log_rewrite('URL rewrite is on. configuration in %s' % path)
 
 
-regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
-regex_anything = re.compile(r'(?<!\\)\$anything')
-regex_redirect = re.compile(r'(\d+)->(.*)')
-
 def compile_regex(k, v):
     """
     Preprocess and compile the regular expressions in routes_app/in/out
-
     The resulting regex will match a pattern of the form:
 
         [remote address]:[protocol]://[host]:[method] [path]
@@ -380,8 +388,9 @@ def compile_regex(k, v):
 def load_routers(all_apps):
     "load-time post-processing of routers"
 
-    for app in routers.keys():
-        # initialize apps with routers that aren't present, on behalf of unit tests
+    for app in routers:
+        # initialize apps with routers that aren't present,
+        # on behalf of unit tests
         if app not in all_apps:
             all_apps.append(app)
             router = Storage(routers.BASE)   # new copy
@@ -420,10 +429,10 @@ def load_routers(all_apps):
                 routers.BASE.domains[router.domain] = app
             if isinstance(router.controllers, str) and router.controllers == 'DEFAULT':
                 router.controllers = set()
-                if os.path.isdir(abspath('applications', app)):
+                if isdir(abspath('applications', app)):
                     cpath = abspath('applications', app, 'controllers')
                     for cname in os.listdir(cpath):
-                        if os.path.isfile(abspath(cpath, cname)) and cname.endswith('.py'):
+                        if isfile(abspath(cpath, cname)) and cname.endswith('.py'):
                             router.controllers.add(cname[:-3])
             if router.controllers:
                 router.controllers.add('static')
@@ -458,16 +467,20 @@ def load_routers(all_apps):
     #
     domains = dict()
     if routers.BASE.domains:
-        for (domain, app) in [(d.strip(':'), a.strip('/')) for (d, a) in routers.BASE.domains.items()]:
-            port = None
+        for (d, a) in routers.BASE.domains.iteritems():
+            (domain, app) = (d.strip(':'), a.strip('/'))
             if ':' in domain:
                 (domain, port) = domain.split(':')
-            ctlr = None
-            fcn = None
+            else:
+                port = None
             if '/' in app:
                 (app, ctlr) = app.split('/', 1)
+            else:
+                ctlr = None
             if ctlr and '/' in ctlr:
                 (ctlr, fcn) = ctlr.split('/')
+            else:
+                fcn = None
             if app not in all_apps and app not in routers:
                 raise SyntaxError, "unknown app '%s' in domains" % app
             domains[(domain, port)] = (app, ctlr, fcn)
@@ -514,7 +527,8 @@ def regex_filter_in(e):
     query = e.get('QUERY_STRING', None)
     e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
     if thread.routes.routes_in:
-        path = regex_uri(e, thread.routes.routes_in, "routes_in", e['PATH_INFO'])
+        path = regex_uri(e, thread.routes.routes_in, 
+                         "routes_in", e['PATH_INFO'])
         rmatch = regex_redirect.match(path)
         if rmatch:
             raise HTTP(int(rmatch.group(1)),location=rmatch.group(2))
@@ -581,6 +595,9 @@ regex_args = re.compile(r'''
      /?$)    # trailing slash
      ''', re.X)
 
+def sluggify(key):
+    return key.lower().replace('.','_')
+
 def regex_url_in(request, environ):
     "rewrite and parse incoming URL"
 
@@ -594,9 +611,8 @@ def regex_url_in(request, environ):
 
     if thread.routes.routes_in:
         environ = regex_filter_in(environ)
-
-    for (key, value) in environ.items():
-        request.env[key.lower().replace('.', '_')] = value
+    
+    request.env.update((sluggify(k),v) for k,v in environ.iteritems())
 
     path = request.env.path_info.replace('\\', '/')
 
@@ -606,7 +622,7 @@ def regex_url_in(request, environ):
 
     match = regex_static.match(regex_space.sub('_', path))
     if match and match.group('x'):
-        static_file = os.path.join(request.env.applications_parent,
+        static_file = pjoin(request.env.applications_parent,
                                    'applications', match.group('b'),
                                    'static', match.group('x'))
         return (static_file, environ)
@@ -908,7 +924,7 @@ class MapUrlIn(object):
         '''
         if len(self.args) == 1 and self.arg0 in self.router.root_static:
             self.controller = self.request.controller = 'static'
-            root_static_file = os.path.join(self.request.env.applications_parent,
+            root_static_file = pjoin(self.request.env.applications_parent,
                                    'applications', self.application,
                                    self.controller, self.arg0)
             log_rewrite("route: root static=%s" % root_static_file)
@@ -962,7 +978,8 @@ class MapUrlIn(object):
                 bad_static = bad_static or name in ('', '.', '..') or not self.router._file_match.match(name)
         if bad_static:
             log_rewrite('bad static path=%s' % file)
-            raise HTTP(400, thread.routes.error_message % 'invalid request',
+            raise HTTP(400,
+                       thread.routes.error_message % 'invalid request',
                        web2py_error='invalid static file')
         #
         #  support language-specific static subdirectories,
@@ -970,13 +987,13 @@ class MapUrlIn(object):
         #  if language-specific file doesn't exist, try same file in static
         #
         if self.language:
-            static_file = os.path.join(self.request.env.applications_parent,
-                                   'applications', self.application,
-                                   'static', self.language, file)
-        if not self.language or not os.path.isfile(static_file):
-            static_file = os.path.join(self.request.env.applications_parent,
-                                   'applications', self.application,
-                                   'static', file)
+            static_file = pjoin(self.request.env.applications_parent,
+                                'applications', self.application,
+                                'static', self.language, file)
+        if not self.language or not isfile(static_file):
+            static_file = pjoin(self.request.env.applications_parent,
+                                'applications', self.application,
+                                'static', file)
         log_rewrite("route: static=%s" % static_file)
         return static_file
 
@@ -1040,12 +1057,14 @@ class MapUrlIn(object):
             uri += '.' + self.extension
         if self.language:
             uri = '/%s%s' % (self.language, uri)
-        uri = '/%s%s' % (app, uri)
-        uri += self.args and urllib.quote('/' + '/'.join([str(x) for x in self.args])) or ''
-        uri += (self.query and ('?' + self.query) or '')
+        uri = '/%s%s%s%s' % (
+            app, 
+            uri,
+            urllib.quote('/'+'/'.join(str(x) for x in self.args)) if self.args else '',
+            ('?' + self.query) if self.query else '')
         self.env['REQUEST_URI'] = uri
-        for (key, value) in self.env.items():
-            self.request.env[key.lower().replace('.', '_')] = value
+        self.request.env.update(
+            (sluggify(k),v) for k,v in self.env.iteritems())
 
     @property
     def arg0(self):

gluon/rocket.py

@@ -1407,7 +1407,7 @@ class Worker(Thread):
             raise BadRequest
 
         req = match.groupdict()
-        for k,v in req.items():
+        for k,v in req.iteritems():
             if not v:
                 req[k] = ""
             if k == 'path':
@@ -1694,7 +1694,8 @@ class FileSystemWorker(Worker):
 
         try:
             # Get our file path
-            headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()])
+            reader = self.read_headers(sock_file)
+            headers = dict((k.lower(),v) for k,v in reader.iteritems())
             rpath = request.get('path', '').lstrip('/')
             filepath = os.path.join(self.root, rpath)
             filepath = os.path.abspath(filepath)

gluon/settings.py

@@ -18,7 +18,8 @@ if not hasattr(os, 'mkdir'):
 if global_settings.db_sessions is not True:
     global_settings.db_sessions = set()
 
-global_settings.gluon_parent = os.environ.get('web2py_path', os.getcwd())
+global_settings.gluon_parent = \
+    os.environ.get('web2py_path', os.getcwd())
 
 global_settings.applications_parent = global_settings.gluon_parent
 
@@ -26,12 +27,14 @@ global_settings.app_folders = set()
 
 global_settings.debugging = False
 
-global_settings.is_pypy = hasattr(platform,'python_implementation') and \
-                          platform.python_implementation() == 'PyPy'
+global_settings.is_pypy = \
+    hasattr(platform,'python_implementation') and \
+    platform.python_implementation() == 'PyPy'
 
-global_settings.is_jython = 'java' in sys.platform.lower() or \
-                            hasattr(sys, 'JYTHON_JAR') or \
-                            str(sys.copyright).find('Jython') > 0
+global_settings.is_jython = \
+    'java' in sys.platform.lower() or \
+    hasattr(sys, 'JYTHON_JAR') or \
+    str(sys.copyright).find('Jython') > 0
 
 
 

gluon/sqlhtml.py

@@ -36,6 +36,8 @@ import re
 import cStringIO
 from gluon import current, redirect, A, URL, DIV, H3, UL, LI, SPAN, INPUT
 import inspect
+import settings
+is_gae = settings.global_settings.web2py_runtime_gae
 
 table_field = re.compile('[\w_]+\.[\w_]+')
 widget_class = re.compile('^\w*')
@@ -581,7 +583,7 @@ class AutocompleteWidget(object):
     def callback(self):
         if self.keyword in self.request.vars:
             field = self.fields[0]
-            if settings.global_settings.web2py_runtime_gae:
+            if is_gae:
                 rows = self.db(field.__ge__(self.request.vars[self.keyword])&field.__lt__(self.request.vars[self.keyword]+ u'\ufffd')).select(orderby=self.orderby,limitby=self.limitby,*self.fields)
             else:
                 rows = self.db(field.like(self.request.vars[self.keyword]+'%')).select(orderby=self.orderby,limitby=self.limitby,distinct=self.distinct,*self.fields)
@@ -1834,7 +1836,7 @@ class SQLFORM(FORM):
             else:
                 rows = dbset.select(left=left,orderby=orderby,*columns)
 
-            if exportManager.has_key(export_type):
+            if export_type in exportManager:
                 value = exportManager[export_type]
                 clazz = value[0] if hasattr(value, '__getitem__') else value
                 oExp = clazz(rows)

gluon/template.py

@@ -15,10 +15,10 @@ Contributors:
 """
 
 import os
-import re
 import cgi
 import cStringIO
 import logging
+from re import compile, sub, escape, DOTALL
 try:
     # have web2py
     from restricted import RestrictedError
@@ -57,6 +57,18 @@ class SuperNode(Node):
     def __repr__(self):
         return "%s->%s" % (self.name, self.value)
 
+def output_aux(node,blocks):
+    # If we have a block level
+    #   If we can override this block.
+    #     Override block from vars.
+    #   Else we take the default
+    # Else its just a string
+    return (blocks[node.name].output(blocks)
+            if node.name in blocks else
+            node.output(blocks)) \
+            if isinstance(node, BlockNode) \
+            else str(node)
+
 class BlockNode(Node):
     """
     Block Container.
@@ -81,8 +93,7 @@ class BlockNode(Node):
 
     def __repr__(self):
         lines = ['%sblock %s%s' % (self.left,self.name,self.right)]
-        for node in self.nodes:
-            lines.append(str(node))
+        lines += [str(node) for node in self.nodes]
         lines.append('%send%s' % (self.left, self.right))
         return ''.join(lines)
 
@@ -90,11 +101,8 @@ class BlockNode(Node):
         """
         Get this BlockNodes content, not including child Nodes
         """
-        lines = []
-        for node in self.nodes:
-            if not isinstance(node, BlockNode):
-                lines.append(str(node))
-        return ''.join(lines)
+        return ''.join(str(node) for node in self.nodes \
+                           if not isinstance(node, BlockNode))
 
     def append(self, node):
         """
@@ -122,30 +130,14 @@ class BlockNode(Node):
         else:
             raise TypeError("Invalid type; must be instance of ``BlockNode``. %s" % other)
 
+
     def output(self, blocks):
         """
         Merges all nodes into a single string.
-
         blocks -- Dictionary of blocks that are extending
         from this template.
         """
-        lines = []
-        # Get each of our nodes
-        for node in self.nodes:
-            # If we have a block level node.
-            if isinstance(node, BlockNode):
-                # If we can override this block.
-                if node.name in blocks:
-                    # Override block from vars.
-                    lines.append(blocks[node.name].output(blocks))
-                # Else we take the default
-                else:
-                    lines.append(node.output(blocks))
-            # Else its just a string
-            else:
-                lines.append(str(node))
-        # Now combine all of our lines together.
-        return ''.join(lines)
+        return ''.join(output_aux(node,blocks) for node in self.nodes)
 
 class Content(BlockNode):
     """
@@ -165,29 +157,13 @@ class Content(BlockNode):
         self.pre_extend = pre_extend
 
     def __str__(self):
-        lines = []
-        # For each of our nodes
-        for node in self.nodes:
-            # If it is a block node.
-            if isinstance(node, BlockNode):
-                # And the node has a name that corresponds with a block in us
-                if node.name in self.blocks:
-                    # Use the overriding output.
-                    lines.append(self.blocks[node.name].output(self.blocks))
-                else:
-                    # Otherwise we just use the nodes output.
-                    lines.append(node.output(self.blocks))
-            else:
-                # It is just a string, so include it.
-                lines.append(str(node))
-        # Merge our list together.
-        return ''.join(lines)
+        return ''.join(output_aux(node,self.blocks) for node in self.nodes)
 
     def _insert(self, other, index = 0):
         """
         Inserts object at index.
         """
-        if isinstance(other, str) or isinstance(other, Node):
+        if isinstance(other, (str, Node)):
             self.nodes.insert(index, other)
         else:
             raise TypeError("Invalid type, must be instance of ``str`` or ``Node``.")
@@ -201,8 +177,7 @@ class Content(BlockNode):
         if isinstance(other, (list, tuple)):
             # Must reverse so the order stays the same.
             other.reverse()
-            for item in other:
-                self._insert(item, index)
+            (self._insert(item, index) for item in other)
         else:
             self._insert(other, index)
 
@@ -210,7 +185,7 @@ class Content(BlockNode):
         """
         Adds a node to list. If it is a BlockNode then we assign a block for it.
         """
-        if isinstance(node, str) or isinstance(node, Node):
+        if isinstance(node, (str, Node)):
             self.nodes.append(node)
             if isinstance(node, BlockNode):
                 self.blocks[node.name] = node
@@ -233,18 +208,18 @@ class Content(BlockNode):
 class TemplateParser(object):
 
     default_delimiters = ('{{','}}')
-    r_tag = re.compile(r'(\{\{.*?\}\})', re.DOTALL)
+    r_tag = compile(r'(\{\{.*?\}\})', DOTALL)
 
-    r_multiline = re.compile(r'(""".*?""")|(\'\'\'.*?\'\'\')', re.DOTALL)
+    r_multiline = compile(r'(""".*?""")|(\'\'\'.*?\'\'\')', DOTALL)
 
     # These are used for re-indentation.
     # Indent + 1
-    re_block = re.compile('^(elif |else:|except:|except |finally:).*$',
-                      re.DOTALL)
+    re_block = compile('^(elif |else:|except:|except |finally:).*$',DOTALL)
+                      
     # Indent - 1
-    re_unblock = re.compile('^(return|continue|break|raise)( .*)?$', re.DOTALL)
+    re_unblock = compile('^(return|continue|break|raise)( .*)?$', DOTALL)
     # Indent - 1
-    re_pass = re.compile('^pass( .*)?$', re.DOTALL)
+    re_pass = compile('^pass( .*)?$', DOTALL)
 
     def __init__(self, text,
                  name    = "ParserContainer",
@@ -291,13 +266,16 @@ class TemplateParser(object):
         # allow optional alternative delimiters
         self.delimiters = delimiters
         if delimiters != self.default_delimiters:
-            escaped_delimiters = (re.escape(delimiters[0]),re.escape(delimiters[1]))
-            self.r_tag = re.compile(r'(%s.*?%s)' % escaped_delimiters, re.DOTALL)
-        elif context.has_key('response') and hasattr(context['response'],'delimiters'):
+            escaped_delimiters = (escape(delimiters[0]),
+                                  escape(delimiters[1]))
+            self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters, DOTALL)
+        elif hasattr(context.get('response',None),'delimiters'):
             if context['response'].delimiters != self.default_delimiters:
-                escaped_delimiters = (re.escape(context['response'].delimiters[0]),
-                                      re.escape(context['response'].delimiters[1]))
-                self.r_tag = re.compile(r'(%s.*?%s)' % escaped_delimiters,re.DOTALL)
+                escaped_delimiters = (
+                    escape(context['response'].delimiters[0]),
+                    escape(context['response'].delimiters[1]))
+                self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters,
+                                     DOTALL)
 
         # Create a root level Content that everything will go into.
         self.content = Content(name=name)
@@ -524,17 +502,19 @@ class TemplateParser(object):
         # the parent nodes.
         self.content.nodes = []
 
+        t_content = t.content
+        
         # Set our include, unique by filename
-        t.content.blocks['__include__' + filename] = buf
+        t_content.blocks['__include__' + filename] = buf
 
         # Make sure our pre_extended nodes go first
-        t.content.insert(pre)
+        t_content.insert(pre)
 
         # Then we extend our blocks
-        t.content.extend(self.content)
+        t_content.extend(self.content)
 
         # Work off the parent node.
-        self.content = t.content
+        self.content = t_content
 
     def parse(self, text):
 
@@ -553,69 +533,20 @@ class TemplateParser(object):
         ij = self.r_tag.split(text)
         # j = current index
         # i = current item
+        stack = self.stack
         for j in range(len(ij)):
             i = ij[j]
 
             if i:
-                if len(self.stack) == 0:
+                if not stack:
                     self._raise_error('The "end" tag is unmatched, please check if you have a starting "block" tag')
 
                 # Our current element in the stack.
-                top = self.stack[-1]
+                top = stack[-1]
 
                 if in_tag:
                     line = i
 
-                    # If we are missing any strings!!!!
-                    # This usually happens with the following example
-                    # template code
-                    #
-                    # {{a = '}}'}}
-                    # or
-                    # {{a = '}}blahblah{{'}}
-                    #
-                    # This will fix these
-                    # This is commented out because the current template
-                    # system has this same limitation. Since this has a
-                    # performance hit on larger templates, I do not recommend
-                    # using this code on production systems. This is still here
-                    # for "i told you it *can* be fixed" purposes.
-                    #
-                    #
-#                    if line.count("'") % 2 != 0 or line.count('"') % 2 != 0:
-#
-#                        # Look ahead
-#                        la = 1
-#                        nextline = ij[j+la]
-#
-#                        # As long as we have not found our ending
-#                        # brackets keep going
-#                        while '}}' not in nextline:
-#                            la += 1
-#                            nextline += ij[j+la]
-#                            # clear this line, so we
-#                            # don't attempt to parse it
-#                            # this is why there is an "if i"
-#                            # around line 530
-#                            ij[j+la] = ''
-#
-#                        # retrieve our index.
-#                        index = nextline.index('}}')
-#
-#                        # Everything before the new brackets
-#                        before = nextline[:index+2]
-#
-#                        # Everything after
-#                        after = nextline[index+2:]
-#
-#                        # Make the next line everything after
-#                        # so it parses correctly, this *should* be
-#                        # all html
-#                        ij[j+1] = after
-#
-#                        # Add everything before to the current line
-#                        line += before
-
                     # Get rid of '{{' and '}}'
                     line = line[2:-2].strip()
 
@@ -633,9 +564,9 @@ class TemplateParser(object):
                     # Perform block comment escaping.
                     # This performs escaping ON anything
                     # in between """ and """
-                    line = re.sub(TemplateParser.r_multiline,
-                                remove_newline,
-                                line)
+                    line = sub(TemplateParser.r_multiline,
+                               remove_newline,
+                               line)
 
                     if line.startswith('='):
                         # IE: {{=response.title}}
@@ -672,7 +603,7 @@ class TemplateParser(object):
                         self.lexers[name](parser    = self,
                                           value     = value,
                                           top       = top,
-                                          stack     = self.stack,)
+                                          stack     = stack)
 
                     elif name == '=':
                         # So we have a variable to insert into
@@ -693,7 +624,7 @@ class TemplateParser(object):
                         # so anything after this gets added
                         # to this node. This allows us to
                         # "nest" nodes.
-                        self.stack.append(node)
+                        stack.append(node)
 
                     elif name == 'end' and not value.startswith('='):
                         # We are done with this node.
@@ -702,7 +633,7 @@ class TemplateParser(object):
                         self.blocks[top.name] = top
 
                         # Pop it.
-                        self.stack.pop()
+                        stack.pop()
 
                     elif name == 'super' and not value.startswith('='):
                         # Get our correct target name
@@ -757,17 +688,17 @@ class TemplateParser(object):
                             # So we can properly put a response.write() in place.
                             continuation = False
                             len_parsed = 0
-                            for k in range(len(tokens)):
+                            for k, token in enumerate(tokens):
 
-                                tokens[k] = tokens[k].strip()
-                                len_parsed += len(tokens[k])
+                                token = tokens[k] = token.strip()
+                                len_parsed += len(token)
 
-                                if tokens[k].startswith('='):
-                                    if tokens[k].endswith('\\'):
+                                if token.startswith('='):
+                                    if token.endswith('\\'):
                                         continuation = True
-                                        tokens[k] = "\n%s(%s" % (self.writer, tokens[k][1:].strip())
+                                        tokens[k] = "\n%s(%s" % (self.writer, token[1:].strip())
                                     else:
-                                        tokens[k] = "\n%s(%s)" % (self.writer, tokens[k][1:].strip())
+                                        tokens[k] = "\n%s(%s)" % (self.writer, token[1:].strip())
                                 elif continuation:
                                     tokens[k] += ')'
                                     continuation = False
@@ -912,7 +843,7 @@ def render(content = "hello world",
 
         # Add it to the context so we can use it.
         if not 'NOESCAPE' in context:
-            context['NOESCAPE'] = XML
+            context['NOESCAPE'] = NOESCAPE
 
     # save current response class
     if context and 'response' in context:

gluon/tests/test_dal.py

@@ -129,6 +129,26 @@ class TestFields(unittest.TestCase):
             )
         self.assertEqual(db.t.insert(a=t0), 1)
         self.assertEqual(db().select(db.t.a)[0].a, t0)
+
+        ## Row APIs
+        row = db().select(db.t.a)[0]
+        self.assertEqual(db.t[1].a,t0)
+        self.assertEqual(db.t['a'],db.t.a)
+        self.assertEqual(db.t(1).a,t0)
+        self.assertTrue(db.t(1,a=None)==None)
+        self.assertFalse(db.t(1,a=t0)==None)
+        self.assertEqual(row.a,t0)
+        self.assertEqual(row['a'],t0)
+        self.assertEqual(row['t.a'],t0)
+        self.assertEqual(row('t.a'),t0)
+
+        ## Lazy and Virtual fields
+        db.t.b = Field.Virtual(lambda row: row.t.a)
+        db.t.c = Field.Lazy(lambda row: row.t.a)
+        row = db().select(db.t.a)[0]
+        self.assertEqual(row.b,t0)
+        self.assertEqual(row.c(),t0)
+
         db.t.drop()
         db.define_table('t', Field('a', 'time', default='11:30'))
         t0 = datetime.time(10, 30, 55)
@@ -369,6 +389,17 @@ class TestJoin(unittest.TestCase):
         db.t1.drop()
         db.t2.drop()
 
+        db.define_table('person',Field('name'))
+        id = db.person.insert(name="max")
+        self.assertEqual(id.name,'max')
+        db.define_table('dog',Field('name'),Field('owner','reference person'))
+        db.dog.insert(name='skipper',owner=1)
+        row = db(db.person.id==db.dog.owner).select().first()
+        self.assertEqual(row[db.person.name],'max')
+        self.assertEqual(row['person.name'],'max')
+        db.dog.drop()
+        self.assertEqual(len(db.person._referenced_by),0)
+        db.person.drop()
 
 class TestMinMaxSum(unittest.TestCase):
 

gluon/tests/test_languages.py

@@ -17,6 +17,7 @@ import languages
 import tempfile
 import threading
 import logging
+from storage import Storage
 
 try:
     import multiprocessing
@@ -53,6 +54,39 @@ try:
             for result in results:
                 self.assertTrue(result)
 
+
+    class TestTranslations(unittest.TestCase):
+    
+        def setUp(self):            
+            self.request = Storage()
+            self.request.folder = 'applications/welcome'
+            self.request.env = Storage()
+            self.request.env.http_accept_language = 'en'
+            
+            
+        def tearDown(self):
+            pass
+    
+        def test_plain(self):
+            T = languages.translator(self.request)
+            self.assertEqual(str(T('Hello World')),
+                             'Hello World')
+            self.assertEqual(str(T('Hello World## comment')),
+                             'Hello World')
+            self.assertEqual(str(T('%s %%{shop}', 1)),
+                             '1 shop')
+            self.assertEqual(str(T('%s %%{shop}', 2)),
+                             '2 shops')
+            self.assertEqual(str(T('%s %%{shop[0]}', 1)),
+                             '1 shop')
+            self.assertEqual(str(T('%s %%{shop[0]}', 2)),
+                             '2 shops')
+            self.assertEqual(str(T.M('**Hello World**')),
+                             '<strong>Hello World</strong>')
+            T.force('it')
+            self.assertEqual(str(T('Hello World')),
+                             'Salve Mondo')
+
 except ImportError:
     logging.warning("Skipped test case, no multiprocessing module.")
         

gluon/tools.py

@@ -31,6 +31,7 @@ from utils import web2py_uuid
 from fileutils import read_file, check_credentials
 from gluon import *
 from gluon.contrib.autolinks import expand_one
+from gluon.dal import Row
 
 import serializers
 
@@ -798,6 +799,141 @@ def addrow(form, a, b, c, style, _id, position=-1):
 
 
 class Auth(object):
+
+    default_settings = {
+        'hideerror': False,
+        'password_min_length': 4,
+        'cas_maps': None,
+        'reset_password_requires_verification': False,
+        'registration_requires_verification': False,
+        'registration_requires_approval': False,
+        'login_after_registration': False,
+        'login_after_password_change': True,
+        'alternate_requires_registration': False,
+        'create_user_groups': "user_%(id)s",
+        'everybody_group_id': None,
+        'login_captcha': None,
+        'register_captcha': None,
+        'retrieve_username_captcha': None,
+        'retrieve_password_captcha': None,
+        'captcha': None,
+        'expiration': 3600,            # one hour
+        'long_expiration': 3600*30*24, # one month
+        'remember_me_form': True,
+        'allow_basic_login': False,
+        'allow_basic_login_only': False,
+        'on_failed_authentication': lambda x: redirect(x),
+        'formstyle': 'table3cols',
+        'label_separator': ': ',
+        'password_field': 'password',
+        'table_user_name': 'auth_user',
+        'table_group_name': 'auth_group',
+        'table_membership_name': 'auth_membership',
+        'table_permission_name': 'auth_permission',
+        'table_event_name': 'auth_event',
+        'table_cas_name': 'auth_cas',
+        'table_user': None,
+        'table_group': None,
+        'table_membership': None,
+        'table_permission': None,
+        'table_event': None,
+        'table_cas': None,
+        'showid': False,
+        'login_email_validate': True,
+        'login_userfield': None,
+        'logout_onlogout': None,
+        'register_fields': None,
+        'register_verify_password': True,
+        'profile_fields': None,
+        'email_case_sensitive': True,
+        'username_case_sensitive': True,
+        }
+        # ## these are messages that can be customized
+    default_messages = {
+        'login_button': 'Login',
+        'register_button': 'Register',
+        'password_reset_button': 'Request reset password',
+        'password_change_button': 'Change password',
+        'profile_save_button': 'Save profile',
+        'submit_button': 'Submit',
+        'verify_password': 'Verify Password',
+        'delete_label': 'Check to delete',
+        'function_disabled': 'Function disabled',
+        'access_denied': 'Insufficient privileges',
+        'registration_verifying': 'Registration needs verification',
+        'registration_pending': 'Registration is pending approval',
+        'login_disabled': 'Login disabled by administrator',
+        'logged_in': 'Logged in',
+        'email_sent': 'Email sent',
+        'unable_to_send_email': 'Unable to send email',
+        'email_verified': 'Email verified',
+        'logged_out': 'Logged out',
+        'registration_successful': 'Registration successful',
+        'invalid_email': 'Invalid email',
+        'unable_send_email': 'Unable to send email',
+        'invalid_login': 'Invalid login',
+        'invalid_user': 'Invalid user',
+        'invalid_password': 'Invalid password',
+        'is_empty': "Cannot be empty",
+        'mismatched_password': "Password fields don't match",
+        'verify_email': 'Click on the link %(link)s to verify your email',
+        'verify_email_subject': 'Email verification',
+        'username_sent': 'Your username was emailed to you',
+        'new_password_sent': 'A new password was emailed to you',
+        'password_changed': 'Password changed',
+        'retrieve_username': 'Your username is: %(username)s',
+        'retrieve_username_subject': 'Username retrieve',
+        'retrieve_password': 'Your password is: %(password)s',
+        'retrieve_password_subject': 'Password retrieve',
+        'reset_password': \
+            'Click on the link %(link)s to reset your password',
+        'reset_password_subject': 'Password reset',
+        'invalid_reset_password': 'Invalid reset password',
+        'profile_updated': 'Profile updated',
+        'new_password': 'New password',
+        'old_password': 'Old password',
+        'group_description': 'Group uniquely assigned to user %(id)s',
+        'register_log': 'User %(id)s Registered',
+        'login_log': 'User %(id)s Logged-in',
+        'login_failed_log': None,
+        'logout_log': 'User %(id)s Logged-out',
+        'profile_log': 'User %(id)s Profile updated',
+        'verify_email_log': 'User %(id)s Verification email sent',
+        'retrieve_username_log': 'User %(id)s Username retrieved',
+        'retrieve_password_log': 'User %(id)s Password retrieved',
+        'reset_password_log': 'User %(id)s Password reset',
+        'change_password_log': 'User %(id)s Password changed',
+        'add_group_log': 'Group %(group_id)s created',
+        'del_group_log': 'Group %(group_id)s deleted',
+        'add_membership_log': None,
+        'del_membership_log': None,
+        'has_membership_log': None,
+        'add_permission_log': None,
+        'del_permission_log': None,
+        'has_permission_log': None,
+        'impersonate_log': 'User %(id)s is impersonating %(other_id)s',
+        'label_first_name': 'First name',
+        'label_last_name': 'Last name',
+        'label_username': 'Username',
+        'label_email': 'E-mail',
+        'label_password': 'Password',
+        'label_registration_key': 'Registration key',
+        'label_reset_password_key': 'Reset Password key',
+        'label_registration_id': 'Registration identifier',
+        'label_role': 'Role',
+        'label_description': 'Description',
+        'label_user_id': 'User ID',
+        'label_group_id': 'Group ID',
+        'label_name': 'Name',
+        'label_table_name': 'Object or table name',
+        'label_record_id': 'Record ID',
+        'label_time_stamp': 'Timestamp',
+        'label_client_ip': 'Client IP',
+        'label_origin': 'Origin',
+        'label_remember_me': "Remember me (for 30 days)",
+        'verify_password_comment': 'please input your password again',
+        }
+
     """
     Class for authentication, authorization, role based access control.
 
@@ -894,10 +1030,11 @@ class Auth(object):
             open(filename,'w').write(key)
         return key
 
-    def url(self, f=None, args=None, vars=None):
+    def url(self, f=None, args=None, vars=None, scheme=False):
         if args is None: args=[]
         if vars is None: vars={}
-        return URL(c=self.settings.controller, f=f, args=args, vars=vars)
+        return URL(c=self.settings.controller, 
+                   f=f, args=args, vars=vars,scheme=scheme)
 
     def here(self):
         return URL(args=current.request.args,vars=current.request.vars)
@@ -933,221 +1070,67 @@ class Auth(object):
         else:
             self.user = None
             session.auth = None
-        settings = self.settings = Settings()
-
         # ## what happens after login?
 
         self.next = current.request.vars._next
         if isinstance(self.next,(list,tuple)):
             self.next = self.next[0]
-
+        url_index = URL(controller,'index')
+        url_login = URL(controller,function,args='login')
         # ## what happens after registration?
-
-        settings.hideerror = False
-        settings.password_min_length = 4
-        settings.cas_domains = [request.env.http_host]
-        settings.cas_provider = cas_provider
-        settings.cas_actions = {'login':'login',
-                                'validate':'validate',
-                                'servicevalidate':'serviceValidate',
-                                'proxyvalidate':'proxyValidate',
-                                'logout':'logout'}
-        settings.cas_maps = None
-        settings.extra_fields = {}
-        settings.actions_disabled = []
-        settings.reset_password_requires_verification = False
-        settings.registration_requires_verification = False
-        settings.registration_requires_approval = False
-        settings.login_after_registration = False
-        settings.alternate_requires_registration = False
-        settings.create_user_groups = "user_%(id)s"
-        settings.everybody_group_id = None
-
-        settings.controller = controller
-        settings.function = function
-        settings.login_url = self.url(function, args='login')
-        settings.logged_url = self.url(function, args='profile')
-        settings.download_url = self.url('download')
-        settings.mailer = (mailer==True) and Mail() or mailer
-        settings.login_captcha = None
-        settings.register_captcha = None
-        settings.retrieve_username_captcha = None
-        settings.retrieve_password_captcha = None
-        settings.captcha = None
-        settings.expiration = 3600            # one hour
-        settings.long_expiration = 3600*30*24 # one month
-        settings.remember_me_form = True
-        settings.allow_basic_login = False
-        settings.allow_basic_login_only = False
-        settings.on_failed_authorization = \
-            self.url(function, args='not_authorized')
-
-        settings.on_failed_authentication = lambda x: redirect(x)
-
-        settings.formstyle = 'table3cols'
-        settings.label_separator = ': '
-
-        # ## table names to be used
-
-        settings.password_field = 'password'
-        settings.table_user_name = 'auth_user'
-        settings.table_group_name = 'auth_group'
-        settings.table_membership_name = 'auth_membership'
-        settings.table_permission_name = 'auth_permission'
-        settings.table_event_name = 'auth_event'
-        settings.table_cas_name = 'auth_cas'
-
-        # ## if none, they will be created, unless DAL(lazy_tables=True)!!!
-
-        settings.table_user = None
-        settings.table_group = None
-        settings.table_membership = None
-        settings.table_permission = None
-        settings.table_event = None
-        settings.table_cas = None
-
-        # ##
-
-        settings.showid = False
-
-        # ## these should be functions or lambdas
-
-        settings.login_next = self.url('index')
-        settings.login_onvalidation = []
-        settings.login_onaccept = []
-        settings.login_methods = [self]
-        settings.login_form = self
-        settings.login_email_validate = True
-        settings.login_userfield = None
-
-        settings.logout_next = self.url('index')
-        settings.logout_onlogout = None
-
-        settings.register_next = self.url('index')
-        settings.register_onvalidation = []
-        settings.register_onaccept = []
-        settings.register_fields = None
-        settings.register_verify_password = True
-
-        settings.verify_email_next = self.url(function, args='login')
-        settings.verify_email_onaccept = []
-
-        settings.profile_next = self.url('index')
-        settings.profile_onvalidation = []
-        settings.profile_onaccept = []
-        settings.profile_fields = None
-        settings.retrieve_username_next = self.url('index')
-        settings.retrieve_password_next = self.url('index')
-        settings.request_reset_password_next = self.url(function, args='login')
-        settings.reset_password_next = self.url(function, args='login')
-
-        settings.change_password_next = self.url('index')
-        settings.change_password_onvalidation = []
-        settings.change_password_onaccept = []
-
-        settings.retrieve_password_onvalidation = []
-        settings.reset_password_onvalidation = []
-        settings.reset_password_onaccept = []
-
-        settings.email_case_sensitive = True
-        settings.username_case_sensitive = True
-
-        settings.hmac_key = hmac_key
+            
+        settings = self.settings = Settings()
+        settings.update(Auth.default_settings)
+        settings.update({
+        'cas_domains': [request.env.http_host],
+        'cas_provider': cas_provider,
+        'cas_actions': {'login':'login',
+                        'validate':'validate',
+                        'servicevalidate':'serviceValidate',
+                        'proxyvalidate':'proxyValidate',
+                        'logout':'logout'},
+        'extra_fields': {},
+        'actions_disabled': [],
+        'controller': controller,
+        'function': function,
+        'login_url': url_login,
+        'logged_url': URL(controller, function, args='profile'),
+        'download_url': URL(controller,'download'),
+        'mailer': (mailer==True) and Mail() or mailer,
+        'on_failed_authorization': \
+            URL(controller,function, args='not_authorized'),
+        'login_next': url_index,
+        'login_onvalidation': [],
+        'login_onaccept': [],
+        'login_methods': [self],
+        'login_form': self,
+        'logout_next': url_index,
+        'logout_onlogout': None,
+        'register_next': url_index,
+        'register_onvalidation': [],
+        'register_onaccept': [],
+        'verify_email_next': url_login,
+        'verify_email_onaccept': [],
+        'profile_next': url_index,
+        'profile_onvalidation': [],
+        'profile_onaccept': [],
+        'retrieve_username_next': url_index,
+        'retrieve_password_next': url_index,
+        'request_reset_password_next': url_login,
+        'reset_password_next': url_index,
+        'change_password_next': url_index,
+        'change_password_onvalidation': [],
+        'change_password_onaccept': [],
+        'retrieve_password_onvalidation': [],
+        'reset_password_onvalidation': [],
+        'reset_password_onaccept': [],
+        'hmac_key': hmac_key,
+        })
         settings.lock_keys = True
 
         # ## these are messages that can be customized
         messages = self.messages = Messages(current.T)
-        messages.login_button = 'Login'
-        messages.register_button = 'Register'
-        messages.password_reset_button = 'Request reset password'
-        messages.password_change_button = 'Change password'
-        messages.profile_save_button = 'Save profile'
-        messages.submit_button = 'Submit'
-        messages.verify_password = 'Verify Password'
-        messages.delete_label = 'Check to delete'
-        messages.function_disabled = 'Function disabled'
-        messages.access_denied = 'Insufficient privileges'
-        messages.registration_verifying = 'Registration needs verification'
-        messages.registration_pending = 'Registration is pending approval'
-        messages.login_disabled = 'Login disabled by administrator'
-        messages.logged_in = 'Logged in'
-        messages.email_sent = 'Email sent'
-        messages.unable_to_send_email = 'Unable to send email'
-        messages.email_verified = 'Email verified'
-        messages.logged_out = 'Logged out'
-        messages.registration_successful = 'Registration successful'
-        messages.invalid_email = 'Invalid email'
-        messages.unable_send_email = 'Unable to send email'
-        messages.invalid_login = 'Invalid login'
-        messages.invalid_user = 'Invalid user'
-        messages.invalid_password = 'Invalid password'
-        messages.is_empty = "Cannot be empty"
-        messages.mismatched_password = "Password fields don't match"
-        messages.verify_email = \
-            'Click on the link ' + \
-            URL('default','user',args='verify_email',scheme=True) + \
-            '/%(key)s to verify your email'
-        messages.verify_email_subject = 'Email verification'
-        messages.username_sent = 'Your username was emailed to you'
-        messages.new_password_sent = 'A new password was emailed to you'
-        messages.password_changed = 'Password changed'
-        messages.retrieve_username = 'Your username is: %(username)s'
-        messages.retrieve_username_subject = 'Username retrieve'
-        messages.retrieve_password = 'Your password is: %(password)s'
-        messages.retrieve_password_subject = 'Password retrieve'
-        messages.reset_password = \
-            'Click on the link ' + \
-            URL('default','user',args='reset_password',scheme=True) + \
-            '/%(key)s to reset your password'
-        messages.reset_password_subject = 'Password reset'
-        messages.invalid_reset_password = 'Invalid reset password'
-        messages.profile_updated = 'Profile updated'
-        messages.new_password = 'New password'
-        messages.old_password = 'Old password'
-        messages.group_description = \
-            'Group uniquely assigned to user %(id)s'
-
-        messages.register_log = 'User %(id)s Registered'
-        messages.login_log = 'User %(id)s Logged-in'
-        messages.login_failed_log = None
-        messages.logout_log = 'User %(id)s Logged-out'
-        messages.profile_log = 'User %(id)s Profile updated'
-        messages.verify_email_log = 'User %(id)s Verification email sent'
-        messages.retrieve_username_log = 'User %(id)s Username retrieved'
-        messages.retrieve_password_log = 'User %(id)s Password retrieved'
-        messages.reset_password_log = 'User %(id)s Password reset'
-        messages.change_password_log = 'User %(id)s Password changed'
-        messages.add_group_log = 'Group %(group_id)s created'
-        messages.del_group_log = 'Group %(group_id)s deleted'
-        messages.add_membership_log = None
-        messages.del_membership_log = None
-        messages.has_membership_log = None
-        messages.add_permission_log = None
-        messages.del_permission_log = None
-        messages.has_permission_log = None
-        messages.impersonate_log = 'User %(id)s is impersonating %(other_id)s'
-
-        messages.label_first_name = 'First name'
-        messages.label_last_name = 'Last name'
-        messages.label_username = 'Username'
-        messages.label_email = 'E-mail'
-        messages.label_password = 'Password'
-        messages.label_registration_key = 'Registration key'
-        messages.label_reset_password_key = 'Reset Password key'
-        messages.label_registration_id = 'Registration identifier'
-        messages.label_role = 'Role'
-        messages.label_description = 'Description'
-        messages.label_user_id = 'User ID'
-        messages.label_group_id = 'Group ID'
-        messages.label_name = 'Name'
-        messages.label_table_name = 'Object or table name'
-        messages.label_record_id = 'Record ID'
-        messages.label_time_stamp = 'Timestamp'
-        messages.label_client_ip = 'Client IP'
-        messages.label_origin = 'Origin'
-        messages.label_remember_me = "Remember me (for 30 days)"
-        messages['T'] = current.T
-        messages.verify_password_comment = 'please input your password again'
+        messages.update(Auth.default_messages)
         messages.lock_keys = True
 
         # for "remember me" option
@@ -1336,37 +1319,39 @@ class Auth(object):
         settings = self.settings
         request = current.request
         T = current.T
-        def lazy_user (auth = self): return auth.user_id
         reference_user = 'reference %s' % settings.table_user_name
+        def lazy_user (auth = self):
+            return auth.user_id        
         def represent(id,record=None,s=settings):
             try:
                 user = s.table_user(id)
                 return '%(first_name)s %(last_name)s' % user
-            except: return id
-        self.signature = db.Table(self.db,'auth_signature',
-                                  Field('is_active','boolean',
-                                        default=True,
-                                        readable=False, writable=False,
-                                        label=T('Is Active')),
-                                  Field('created_on','datetime',
-                                        default=request.now,
-                                        writable=False, readable=False,
-                                        label=T('Created On')),
-                                  Field('created_by',
-                                        reference_user,
-                                        default=lazy_user, represent=represent,
-                                        writable=False, readable=False,
-                                        label=T('Created By')),
-                                  Field('modified_on','datetime',
-                                        update=request.now,default=request.now,
-                                        writable=False,readable=False,
-                                        label=T('Modified On')),
-                                  Field('modified_by',
-                                        reference_user,represent=represent,
-                                        default=lazy_user,update=lazy_user,
-                                        writable=False,readable=False,
-                                        label=T('Modified By')))
-
+            except:
+                return id
+        self.signature = db.Table(
+            self.db,'auth_signature',
+            Field('is_active','boolean',
+                  default=True,
+                  readable=False, writable=False,
+                  label=T('Is Active')),
+            Field('created_on','datetime',
+                  default=request.now,
+                  writable=False, readable=False,
+                  label=T('Created On')),
+            Field('created_by',
+                  reference_user,
+                  default=lazy_user, represent=represent,
+                  writable=False, readable=False,
+                  label=T('Created By')),
+            Field('modified_on','datetime',
+                  update=request.now,default=request.now,
+                  writable=False,readable=False,
+                  label=T('Modified On')),
+            Field('modified_by',
+                  reference_user,represent=represent,
+                  default=lazy_user,update=lazy_user,
+                  writable=False,readable=False,
+                  label=T('Modified By')))
 
     def define_tables(self, username=False, signature=None,
                       migrate=True, fake_migrate=False):
@@ -1396,8 +1381,7 @@ class Auth(object):
         elif isinstance(signature,self.db.Table):
             signature_list = [signature]
         else:
-            signature_list = signature
-        lazy_tables, db._lazy_tables = db._lazy_tables, False
+            signature_list = signature        
         is_not_empty = IS_NOT_EMPTY(error_message=self.messages.is_empty)
         is_crypted = CRYPT(key=settings.hmac_key,
                            min_length=settings.password_min_length)
@@ -1600,7 +1584,6 @@ class Auth(object):
                 actions=actions,
                 maps=maps)
 
-
     def log_event(self, description, vars=None, origin='auth'):
         """
         usage:
@@ -1667,6 +1650,11 @@ class Auth(object):
         return user
 
     def basic(self):
+        """
+        perform basic login.
+        reads current.request.env.http_authorization
+        and returns basic_allowed,basic_accepted,user 
+        """
         if not self.settings.allow_basic_login:
             return (False,False,False)
         basic = current.request.env.http_authorization
@@ -1675,11 +1663,23 @@ class Auth(object):
         (username, password) = base64.b64decode(basic[6:]).split(':')
         return (True, True, self.login_bare(username, password))
 
+    def login_user(self,user):
+        """
+        login the user = db.auth_user(id)
+        """
+        # user=Storage(self.table_user()._filter_fields(user,id=True))
+        current.session.auth = Storage(
+            user = user, 
+            last_visit = current.request.now,
+            expiration = self.settings.expiration,
+            hmac_key = web2py_uuid())
+        self.user = user
+        self.update_groups()
+
     def login_bare(self, username, password):
         """
-        logins user
+        logins user as specified by usernname (or email) and password
         """
-
         request = current.request
         session = current.session
         table_user = self.table_user()
@@ -1694,12 +1694,7 @@ class Auth(object):
         if user and user.get(passfield,False):
             password = table_user[passfield].validate(password)[0]
             if not user.registration_key and password == user[passfield]:
-                user = Storage(table_user._filter_fields(user, id=True))
-                session.auth = Storage(user=user, last_visit=request.now,
-                                       expiration=self.settings.expiration,
-                                       hmac_key = web2py_uuid())
-                self.user = user
-                self.update_groups()
+                self.login_user(user)
                 return user
         else:
             # user not in database try other login methods
@@ -1738,15 +1733,15 @@ class Auth(object):
                              renew=interactivelogin)
             service = session._cas_service
             del session._cas_service
-            if request.vars.has_key('warn') and not interactivelogin:
+            if 'warn' in request.vars and not interactivelogin:
                 response.headers['refresh'] = "5;URL=%s"%service+"?ticket="+ticket
                 return A("Continue to %s"%service,
                     _href=service+"?ticket="+ticket)
             else:
                 redirect(service+"?ticket="+ticket)
-        if self.is_logged_in() and not request.vars.has_key('renew'):
+        if self.is_logged_in() and not 'renew' in request.vars:
             return allow_access()
-        elif not self.is_logged_in() and request.vars.has_key('gateway'):
+        elif not self.is_logged_in() and 'gateway' in request.vars:
             redirect(service)
         def cas_onaccept(form, onaccept=onaccept):
             if not onaccept is DEFAULT: onaccept(form)
@@ -1759,7 +1754,7 @@ class Auth(object):
         db, table = self.db, self.table_cas()
         current.response.headers['Content-Type']='text'
         ticket = request.vars.ticket
-        renew = True if request.vars.has_key('renew') else False
+        renew = 'renew' in request.vars
         row = table(ticket=ticket)
         success = False
         if row:
@@ -1973,25 +1968,18 @@ class Auth(object):
 
         # process authenticated users
         if user:
-            user = Storage(table_user._filter_fields(user, id=True))
-
+            user = Row(table_user._filter_fields(user, id=True))
             # process authenticated users
             # user wants to be logged in for longer
-            session.auth = Storage(
-                user = user,
-                last_visit = request.now,
-                expiration = request.vars.get("remember",False) and \
-                    self.settings.long_expiration or self.settings.expiration,
-                remember = request.vars.has_key("remember"),
-                hmac_key = web2py_uuid()
-                )
-
-            self.user = user
+            self.login_user(user)
+            session.auth.expiration = \
+                request.vars.get('remember',False) and \
+                self.settings.long_expiration or \
+                self.settings.expiration
+            session.auth.remember = 'remember' in request.vars
             self.log_event(log, user)
             session.flash = self.messages.logged_in
 
-        self.update_groups()
-
         # how to continue
         if self.settings.login_form == self:
             if accepted_form:
@@ -2131,11 +2119,14 @@ class Auth(object):
             if self.settings.everybody_group_id:
                 self.add_membership(self.settings.everybody_group_id, form.vars.id)
             if self.settings.registration_requires_verification:
+                link = self.url('user',args=('verify_email',key),scheme=True)
+                                
                 if not self.settings.mailer or \
-                   not self.settings.mailer.send(to=form.vars.email,
-                        subject=self.messages.verify_email_subject,
-                        message=self.messages.verify_email
-                         % dict(key=key)):
+                   not self.settings.mailer.send(
+                    to=form.vars.email,
+                    subject=self.messages.verify_email_subject,
+                    message=self.messages.verify_email \
+                        % dict(key=key,link=link)):
                     self.db.rollback()
                     response.flash = self.messages.unable_send_email
                     return form
@@ -2149,13 +2140,10 @@ class Auth(object):
                 if not self.settings.registration_requires_verification:
                     table_user[form.vars.id] = dict(registration_key='')
                 session.flash = self.messages.registration_successful
-                user = self.db(table_user[username] == form.vars[username]).select().first()
-                user = Storage(table_user._filter_fields(user, id=True))
-                session.auth = Storage(user=user, last_visit=request.now,
-                                       expiration=self.settings.expiration,
-                                       hmac_key = web2py_uuid())
-                self.user = user
-                self.update_groups()
+                user = self.db(
+                    table_user[username] == form.vars[username]
+                    ).select().first()                
+                self.login_user(user)
                 session.flash = self.messages.logged_in
             self.log_event(log, form.vars)
             callback(onaccept,form)
@@ -2192,7 +2180,7 @@ class Auth(object):
 
         key = getarg(-1)
         table_user = self.table_user()
-        user = self.db(table_user.registration_key == key).select().first()
+        user = table_user(registration_key=key)
         if not user:
             redirect(self.settings.login_url)
         if self.settings.registration_requires_approval:
@@ -2267,7 +2255,7 @@ class Auth(object):
         if form.accepts(request, session,
                         formname='retrieve_username', dbio=False,
                         onvalidation=onvalidation,hideerror=self.settings.hideerror):
-            user = self.db(table_user.email == form.vars.email).select().first()
+            user = table_user(email=form.vars.email)
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
@@ -2345,7 +2333,7 @@ class Auth(object):
         if form.accepts(request, session,
                         formname='retrieve_password', dbio=False,
                         onvalidation=onvalidation,hideerror=self.settings.hideerror):
-            user = self.db(table_user.email == form.vars.email).select().first()
+            user = table_user(email=form.vars.email)
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
@@ -2398,12 +2386,12 @@ class Auth(object):
         session = current.session
 
         if next is DEFAULT:
-            next = self.next or self.settings.reset_password_next
+            next = self.settings.reset_password_next
         try:
             key = request.vars.key or getarg(-1)
             t0 = int(key.split('-')[0])
             if time.time()-t0 > 60*60*24: raise Exception
-            user = self.db(table_user.reset_password_key == key).select().first()
+            user = table_user(reset_password_key=key)
             if not user: raise Exception
         except Exception:
             session.flash = self.messages.invalid_reset_password
@@ -2422,11 +2410,15 @@ class Auth(object):
             formstyle=self.settings.formstyle,
             separator=self.settings.label_separator
         )
-        if form.accepts(request,session,hideerror=self.settings.hideerror):
-            user.update_record(**{passfield:str(form.vars.new_password),
-                                  'registration_key':'',
-                                  'reset_password_key':''})
+        if form.accepts(request,session,
+                        hideerror=self.settings.hideerror):
+            user.update_record(
+                **{passfield:str(form.vars.new_password),
+                   'registration_key':'',
+                   'reset_password_key':''})
             session.flash = self.messages.password_changed
+            if self.settings.login_after_password_change:
+                self.login_user(user)
             redirect(next)
         return form
 
@@ -2481,7 +2473,7 @@ class Auth(object):
                         formname='reset_password', dbio=False,
                         onvalidation=onvalidation,
                         hideerror=self.settings.hideerror):
-            user = self.db(table_user.email == form.vars.email).select().first()
+            user = table_user(email=form.vars.email)
             if not user:
                 session.flash = self.messages.invalid_email
                 redirect(self.url(args=request.args))
@@ -2504,11 +2496,14 @@ class Auth(object):
 
     def email_reset_password(self,user):
         reset_password_key = str(int(time.time()))+'-' + web2py_uuid()
+        link = self.url('user',
+                        args=('reset_password',reset_password_key),
+                        scheme=True) 
         if self.settings.mailer.send(
             to=user.email,
             subject=self.messages.reset_password_subject,
             message=self.messages.reset_password % \
-                dict(key=reset_password_key)):
+                dict(key=reset_password_key,link=link)):
             user.update_record(reset_password_key=reset_password_key)
             return True
         return False
@@ -2684,7 +2679,8 @@ class Auth(object):
             self.user = auth.user
             if self.settings.login_onaccept:
                 form = Storage(dict(vars=self.user))
-                self.settings.login_onaccept(form)
+                for callback in self.settings.login_onaccept:
+                    callback(form)
             log = self.messages.impersonate_log
             self.log_event(log,dict(id=current_id, other_id=auth.user.id))
         elif user_id in (0, '0') and self.is_impersonating():
@@ -3142,18 +3138,21 @@ class Auth(object):
                 elif form.record and fieldname in form.record:
                     new_record[fieldname]=form.record[fieldname]
         if fields:
-            for key,value in fields.items():
-                new_record[key] = value
+            new_record.update(fields)
         id = archive_table.insert(**new_record)
         return id
-    def wiki(self,slug=None,env=None,manage_permissions=False,force_prefix=''):
+
+    def wiki(self,slug=None,env=None,manage_permissions=False,force_prefix='', resolve=True):
         if not hasattr(self,'_wiki'):
             self._wiki = Wiki(self,
                               manage_permissions=manage_permissions,
                               force_prefix=force_prefix,env=env)
         else:
             self._wiki.env.update(env or {})
-        return self._wiki.read(slug)['content'] if slug else self._wiki()
+        # if resolve is set to True, process request as wiki call
+        # resolve=False allows initial setup without wiki redirection
+        if resolve:
+            return self._wiki.read(slug)['content'] if slug else self._wiki()
 
 class Crud(object):
 
@@ -3714,14 +3713,14 @@ def universal_caller(f, *a, **b):
     # There might be pos_args left, that are sent as named_values. Gather them as well.
     # If a argument already is populated with values we simply replaces them.
     for arg_name in pos_args[len(arg_dict):]:
-        if b.has_key(arg_name):
+        if arg_name in b:
             arg_dict[arg_name] = b[arg_name]
 
     if len(arg_dict) >= len(pos_args):
         # All the positional arguments is found. The function may now be called.
         # However, we need to update the arg_dict with the values from the named arguments as well.
         for arg_name in named_args:
-            if b.has_key(arg_name):
+            if arg_name in b:
                 arg_dict[arg_name] = b[arg_name]
 
         return f(**arg_dict)
@@ -4136,7 +4135,7 @@ class Service(object):
             prefix='pys',
             documentation = documentation,
             ns = True)
-        for method, (function, returns, args, doc) in procedures.items():
+        for method, (function, returns, args, doc) in procedures.iteritems():
             dispatcher.register_function(method, function, returns, args, doc)
         if request.env.request_method == 'POST':
             # Process normal Soap Operation
@@ -4392,8 +4391,9 @@ class PluginManager(object):
             self.__dict__.clear()
         settings = self.__getattr__(plugin)
         settings.installed = True
-        [settings.update({key:value}) for key,value in defaults.items() \
-            if not key in settings]
+        settings.update(
+            (k,v) for k,v in defaults.items() if not k in settings)
+                            
     def __getattr__(self, key):
         if not key in self.__dict__:
             self.__dict__[key] = Storage()
@@ -4494,32 +4494,48 @@ class Wiki(object):
         self.host = current.request.env.http_host
         perms = self.manage_permissions = manage_permissions
         db = auth.db
-        db.define_table(
-            'wiki_page',
-            Field('slug',
-                  requires=[IS_SLUG(),IS_NOT_IN_DB(db,'wiki_page.slug')],
-                  readable=False,writable=False),
-            Field('title',unique=True),
-            Field('body','text',notnull=True),
-            Field('tags','list:string'),
-            Field('can_read','list:string',writable=perms,readable=perms,
-                  default=[Wiki.everybody]),
-            Field('can_edit','list:string',writable=perms,readable=perms,
-                  default=[Wiki.everybody]),
-            Field('changelog'),
-            Field('html','text',readable=False,writable=False,compute=render),
-            auth.signature,format='%(title)s')
-        db.define_table(
-            'wiki_tag',
-            Field('name'),
-            Field('wiki_page',db.wiki_page),
-            auth.signature,format='%(name)s')
-        db.define_table(
-            'wiki_media',
-            Field('wiki_page',db.wiki_page),
-            Field('title',required=True),
-            Field('file','upload',required=True),
-            auth.signature,format='%(title)s')
+        table_definitions = {
+            'wiki_page':{
+                'args':[
+                    Field('slug',
+                          requires=[IS_SLUG(),
+                                    IS_NOT_IN_DB(db,'wiki_page.slug')],
+                          readable=False,writable=False),
+                    Field('title',unique=True),
+                    Field('body','text',notnull=True),
+                    Field('tags','list:string'),
+                    Field('can_read','list:string',
+                          writable=perms,
+                          readable=perms,
+                          default=[Wiki.everybody]),
+                    Field('can_edit', 'list:string',
+                          writable=perms,readable=perms,
+                          default=[Wiki.everybody]),
+                    Field('changelog'),
+                    Field('html','text',compute=render,
+                          readable=False, writable=False),
+                    auth.signature],
+                'vars':{'format':'%(title)s'}},
+             'wiki_tag':{
+                'args':[
+                    Field('name'),
+                    Field('wiki_page','reference wiki_page'),
+                    auth.signature],
+                'vars':{'format':'%(name)s'}},
+           'wiki_media':{
+                'args':[
+                    Field('wiki_page','reference wiki_page'),
+                    Field('title',required=True),
+                    Field('file','upload',required=True),
+                    auth.signature],
+                'vars':{'format':'%(title)s'}}
+            }
+
+        # define only non-existent tables
+        for key, value in table_definitions.iteritems():
+            if not key in db.tables():
+                db.define_table(key, *value['args'], **value['vars'])
+
         def update_tags_insert(page,id,db=db):
             for tag in page.tags or []:
                 tag = tag.strip().lower()