faster and leaner languages, should be 100% backward compatible, plurals work again

VERSION

@@ -1 +1 @@
-Version 2.00.0 (2012-08-27 10:13:44) dev
+Version 2.00.0 (2012-08-27 10:57:21) dev

applications/admin/controllers/default.py

@@ -23,9 +23,9 @@ except ImportError:
     GIT_MISSING = 'requires python-git module, but not installed or incompatible version'
 
 from gluon.languages import (regex_language, read_possible_languages,
-                             read_possible_plurals, lang_sampling,
+                             lang_sampling,
                              read_dict, write_dict, read_plural_dict,
-                             write_plural_dict)
+                             write_plural_dict, PLURAL_RULES)
 
 
 if DEMO_MODE and request.function in ['change_password','pack','pack_plugin','upgrade_web2py','uninstall','cleanup','compile_app','remove_compiled_app','delete','delete_plugin','create_file','upload_file','update_languages','reload_routes','git_push','git_pull']:
@@ -949,8 +949,8 @@ def design():
                                         # get only existed files
     languages = sorted(all_languages)
 
-    plural_rules={}
+    plural_rules = {}
-    all_plurals=read_possible_plurals()
+    all_plurals = PLURAL_RULES
     for langfile,lang in all_languages.iteritems():
         lang=lang.strip()
         match_language = regex_language.match(lang)
@@ -961,7 +961,7 @@ def design():
             plang = lang_sampling(match_language, all_plurals.keys())
             if plang:
                plural=all_plurals[plang]
-               plural_rules[langfile]=(plural[0],plang,plural[1],plural[3])
+               plural_rules[langfile]=(plural[0],plang,plural[4],plural[3])
             else:
                plural_rules[langfile]=(0,lang,'plural_rules-%s.py'%lang,'')
 

gluon/languages.py

@@ -34,7 +34,7 @@ osep = os.sep
 isdir = os.path.isdir
 is_gae = settings.global_settings.web2py_runtime_gae
 
-DEFAULT_ACCEPT_LANGUAGE = 'en'
+DEFAULT_LANGUAGE = 'en'
 
 # DEFAULT PLURAL-FORMS RULES:
 # language doesn't use plural forms
@@ -76,23 +76,24 @@ regex_param=re.compile(r'{(?P<s>.+?)}')
 regex_language = \
     re.compile('^([a-zA-Z]{2})(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$')
 regex_langfile = re.compile('^[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
-regex_langinfo = re.compile("^[^'\"]*['\"]([^'\"]*)['\"]\s*:\s*['\"]([^'\"]*)['\"].*$")
 regex_backslash = re.compile(r"\\([\\{}%])")
 regex_plural = re.compile('%({.+?})')
 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)\)}$')  # %%{word(varname or number)}
 regex_plural_tuple = re.compile('^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word}
-regex_plural_q = re.compile('^asdf$') # %%{?word?cnt}, %%{??cnt} or %%{?cnt}
 regex_plural_rules = re.compile('^plural_rules-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
 
-upper_fun = lambda s: unicode(s,'utf-8').upper().encode('utf-8')
+# UTF8 helper functions
-title_fun = lambda s: unicode(s,'utf-8').title().encode('utf-8')
+def upper_fun(s):
-cap_fun   = lambda s: unicode(s,'utf-8').capitalize().encode('utf-8')
+    return unicode(s,'utf-8').upper().encode('utf-8')
-
+def title_fun(s):
+    return unicode(s,'utf-8').title().encode('utf-8')
+def cap_fun(s):
+    return lambda s: unicode(s,'utf-8').capitalize().encode('utf-8')
 ttab_in  = maketrans("\\%{}", '\x1c\x1d\x1e\x1f')
 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
 
 # cache of translated messages:
-# of structure:
+# global_language_cache:
 # { 'languages/xx.py':
 #     ( {"def-message": "xx-message",
 #        ...
@@ -100,35 +101,38 @@ ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
 #  'languages/yy.py': ( {dict}, lock_object )
 #  ...
 # }
-tcache={}
+
+global_language_cache={}
 
 def get_from_cache(cache, val, fun):
-    lock=cache[1]
+    lang_dict, lock = cache
     lock.acquire()
     try:
-        result=cache[0].get(val);
+        result = lang_dict.get(val);
     finally:
         lock.release()
     if result:
         return result
     lock.acquire()
     try:
-        result=cache[0].setdefault(val, fun())
+        result = lang_dict.setdefault(val, fun())
     finally:
         lock.release()
     return result
 
-def clear_cache(cache):
+def clear_cache(filename):
-    lock=cache[1]
+    cache = global_language_cache.setdefault(
+        filename, ({}, allocate_lock()))
+    lang_dict, lock = cache
     lock.acquire()
     try:
-        cache[0].clear();
+        lang_dict.clear();
     finally:
         lock.release()
 
-
 def lang_sampling(lang_tuple, langlist):
-    """ search *lang_tuple* in *langlist*
+    """
+    search *lang_tuple* in *langlist*
 
     Args:
         lang_tuple (tuple of strings): ('aa'[[,'-bb'],'-cc'])
@@ -159,11 +163,8 @@ def lang_sampling(lang_tuple, langlist):
 
 
 def read_dict_aux(filename):
-    fp = portalocker.LockedFile(filename, 'r')
+    lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
-    lang_text = fp.read().replace('\r\n', '\n')
+    clear_cache(filename)
-    fp.close()
-    # clear cache of processed messages:
-    clear_cache(tcache.setdefault(filename, ({}, allocate_lock())))
     try:
         return safe_eval(lang_text) or {}
     except Exception, e:
@@ -180,9 +181,10 @@ def read_dict(filename):
 
 
 def get_lang_info(lang, langdir):
-    """retrieve lang information from *langdir*/*lang*.py file.
+    """
-       Read few strings from lang.py file until keys !langname!,
+    retrieve lang information from *langdir*/*lang*.py file.
-       !langcode! or keys greater then '!*' were found
+    Read few strings from lang.py file until keys !langname!,
+    !langcode! or keys greater then '!*' were found
 
     args:
         lang (str): lang-code or 'default'
@@ -193,27 +195,10 @@ def get_lang_info(lang, langdir):
         e.g.: ('en', 'English', 1338549043.0)
     """
     filename = ospath.join(langdir, lang+'.py')
-    langcode=langname=''
+    d = read_dict(filename)    
-    f = portalocker.LockedFile(filename, 'r')
+    langcode = d.get('!langcode!',DEFAULT_LANGUAGE)
-    try:
+    langname = d.get('!langname!',langcode)
-        while not (langcode and langname):
+    return (langcode, langname or langcode, ostat(filename).st_mtime)
-            line = f.readline()
-            if not line:
-               break
-            match=regex_langinfo.match(line)
-            if match:
-                k = match.group(1)
-                if k == '!langname!':
-                    langname = match.group(2)
-                elif k == '!langcode!':
-                    langcode = match.group(2)
-                elif k[0:1] > '!':
-                    break
-    finally:
-        f.close()
-    if not langcode:
-        langcode = lang if lang != 'default' else 'en'
-    return langcode, langname or langcode, ostat(filename).st_mtime
 
 def read_possible_languages(appdir):
     langs = {}
@@ -222,10 +207,10 @@ def read_possible_languages(appdir):
     for filename in os.listdir(langdir):
         if regex_langfile.match(filename) or filename=='default.py':
             lang = filename[:-3]
-            langs[lang]=get_lang_info(lang, langdir)
+            langs[lang] = get_lang_info(lang, langdir)
-    if not 'default' in langs:
+    if not 'en' in langs:
         # if default.py is not found, add default value:
-        langs['default'] = ('en', 'English', 0)
+        langs['en'] = ('en', 'English', 0)
     return langs
 
 def read_global_plural_rules(filename):
@@ -240,14 +225,13 @@ def read_global_plural_rules(filename):
         e.g.: (3, <function>, <function>, ok)
     """
     env = {}
-    lock = portalocker.LockedFile(filename, 'r')
+    data = portalocker.read_locked(filename)
     try:
-        execfile(filename) in env
+        exec(data) in env
         status='ok'
     except Exception, e:
         status='Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
-    lock.close()        
     nplurals = env.get('nplurals', DEFAULT_NPLURALS)
     get_plural_id = env.get('get_plural_id', DEFAULT_GET_PLURAL_ID)
     construct_plural_form = env.get('construct_plural_form',
@@ -256,8 +240,9 @@ def read_global_plural_rules(filename):
 
 
 def read_possible_plurals():
-    """ create list of all possible plural rules files
+    """
-        result is cached to increase speed
+    create list of all possible plural rules files
+    result is cached to increase speed
     """
     pdir = abspath('gluon','contrib','rules')
     plurals = {}
@@ -268,21 +253,20 @@ def read_possible_plurals():
             fname = ospath.join(pdir, pname)
             n, f1, f2, status = read_global_plural_rules(fname)
             if status == 'ok':
-                plurals[lang] = (lang, n, f1, f2)
+                plurals[lang] = (lang, n, f1, f2, pname)
     plurals['default'] = ('default',
                           DEFAULT_NPLURALS,
                           DEFAULT_GET_PLURAL_ID,
-                          DEFAULT_CONSTRUCTOR_PLURAL_FORM)
+                          DEFAULT_CONSTRUCTOR_PLURAL_FORM,
+                          None)
     return plurals
 
 PLURAL_RULES = read_possible_plurals()
 
 def read_plural_dict_aux(filename):
-    fp = portalocker.LockedFile(filename, 'r')
+    lang_text = portalocker.read_locked(filename).replace('\r\n', '\n')
-    lang_text = fp.read().replace('\r\n', '\n')
-    fp.close()
     try:
-        return safe_eval(lang_text) or {}
+        return eval(lang_text) or {}
     except Exception, e:
         status='Syntax error in %s (%s)' % (filename, e)
         logging.error(status)
@@ -422,13 +406,10 @@ class lazyT(object):
         return lazyT(self.m, symbols, self.T, self.f, self.t, self.M)
 
 class translator(object):
-
     """
     this class is instantiated by gluon.compileapp.build_environment
     as the T object
-
     ::
-
         T.force(None) # turns off translation
         T.force('fr, it') # forces web2py to translate using fr.py or it.py
 
@@ -436,20 +417,21 @@ class translator(object):
 
     notice 1: there is no need to force since, by default, T uses
     http_accept_language to determine a translation file.
-
+    notice 2: 
-    notice 2: en and en-en are considered different languages!
+       en and en-en are considered different languages!
-
+    notice 3: 
-    notice 3: if language xx-yy is not found force() probes other similar
+       if language xx-yy is not found force() probes other similar
-    languages using such algorithm: xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py
+    languages using such algorithm: 
+        xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py
     """
 
     def __init__(self, request):
         self.request = request
         self.folder = request.folder
         self.langpath = ospath.join(self.folder,'languages')
-        self.filenames = set(os.listdir(self.langpath))
+        self.filenames = set(os.listdir(self.langpath))        
-        self.cache = tcache.setdefault(None, ({},allocate_lock()))
         self.http_accept_language = request.env.http_accept_language
+        # self.cache                        # filled in self.force()
         # self.accepted_language = None     # filled in self.force()
         # self.language_file = None         # filled in self.force()
         # self.plural_language = None       # filled in self.force()
@@ -468,12 +450,9 @@ class translator(object):
         self.filter = markmin
         self.ftag = 'markmin'
 
-
     def get_possible_languages(self):
-        possible_languages = [
+        return [lang[:-3] for lang in self.filenames \
-            lang[:-3] for lang in self.filenames \
+                    if regex_langfile.match(lang)]
-                if regex_langfile.match(lang)]
-        return possible_languages
 
     def set_current_languages(self, *languages):
         """
@@ -488,27 +467,31 @@ class translator(object):
         self.force(self.http_accept_language)
 
     def set_plural(self, language):
-        """ initialize plural forms subsystem
+        """
-            invoked from self.force()
+        initialize plural forms subsystem
+        invoked from self.force()
         """
         lang = language[:2] 
         (self.plural_language,
          self.nplurals,
          self.get_plural_id,
          self.construct_plural_form,
+         self.plural_filename
          ) = PLURAL_RULES.get(language,PLURAL_RULES['default'])
-        filename = 'plural-%s.py' % self.plural_language
+        for lang in (language, language[:5], language[:2]):
-        if filename in self.filenames:
+            filename = 'plural-%s.py' % lang
-            self.plural_file = ospath.join(self.langpath,filename)
+            if filename in self.filenames:
-            self.plural_dict = read_plural_dict(self.plural_file)
+                self.plural_file = ospath.join(self.langpath,filename)
+                self.plural_dict = read_plural_dict(self.plural_file)
+                break
         else:
             self.plural_file = None
             self.plural_dict = {}
-
+ 
     def plural(self, word, n):
         """
         get plural form of word for number *n*
-            NOTE: *word" MUST be defined in current language
+            NOTE: *word* MUST be defined in current language
                   (T.accepted_language)
 
             invoked from T()/M() in %%{} tag
@@ -520,25 +503,27 @@ class translator(object):
             (str): word in appropriate singular/plural form
         """
         nplurals = self.nplurals
-        if word:
+        if int(n)==1:
-            id = self.get_plural_id(abs(int(n)))
+            return word
-            if id > 0:
+        elif word:
-                forms = self.plural_dict.get(word, [])
+            id = min(int(n)-1,1) # self.get_plural_id(abs(int(n)))
-                if forms:
+            # id = 0 first plural form
-                    form = forms.get(id-1)
+            # id = 1 second plural form
-                    if form:
+            # etc.
-                        return form
+            forms = self.plural_dict.get(word, [])
+            if len(forms)>=id:
+                # have this plural form
+                return forms[id-1]
+            else:
+                # guessing this plural form
+                forms += ['']*(nplurals-len(forms)-1)
                 form = self.construct_plural_form(word, id)
-                if len(forms) < nplurals-1:
-                    forms.extend('' for i in xrange(nplurals-len(forms)-1))
                 forms[id-1] = form
                 self.plural_dict[word] = forms
-                if (self.plural_file and
+                if self.plural_file and not is_gae:
-                    not settings.global_settings.web2py_runtime_gae):
                     write_plural_dict(self.plural_file,
                                       self.plural_dict)
                 return form
-        return word
 
     def get_possible_languages_info(self, lang=None):
         """
@@ -556,9 +541,9 @@ class translator(object):
                                          langname(from !langname! key),
                                          langfile_mtime ) }
         """
-        if lang:
+        info = read_possible_languages(self.folder)
-            return read_possible_languages(self.folder).get(lang)
+        if lang: info = ingo.get(lang)
-        return read_possible_languages(self.folder)
+        return info
 
     def force(self, *languages):
         """
@@ -572,7 +557,6 @@ class translator(object):
         default language will be selected if none
         of them matches possible_languages.
         """
-        global tcache
         language = ''
         if isinstance(languages,str):
             languages = regex_language.findall(languages.lower())
@@ -593,21 +577,20 @@ class translator(object):
                 break
         else:
             if 'default.py' in self.filenames:
-                language = DEFAULT_ACCEPT_LANGUAGE
+                language = DEFAULT_LANGUAGE
                 langfile = 'default.py'
             else:
-                language = DEFAULT_ACCEPT_LANGUAGE
+                language = DEFAULT_LANGUAGE
                 langfile = None
         self.accepted_language = language
         if langfile:
             self.language_file = ospath.join(self.langpath,langfile)
             self.t = read_dict(self.language_file)
-            self.cache = tcache.setdefault(self.language_file,
-                                           ({},allocate_lock()))
         else:
             self.language_file = None
             self.t = {}
-            self.cache = tcache[None]
+        self.cache = global_language_cache.setdefault(
+            self.language_file,({},allocate_lock()))
         self.set_plural(language)
         return languages
 
@@ -639,26 +622,30 @@ class translator(object):
             prefix = '@'+(ftag or 'userdef')+'\x01'
         else:
             prefix = '@'+self.ftag+'\x01'
-        message = get_from_cache(self.cache, prefix+message,
+        message = get_from_cache(
-                                 lambda: get_tr(message, prefix, filter))
+            self.cache, prefix+message,
+            lambda: get_tr(message, prefix, filter))
         if symbols or symbols == 0 or symbols == "":
             if isinstance(symbols, dict):
-                symbols.update( (key, xmlescape(value).translate(ttab_in))
+                symbols.update(
-                                for key, value in symbols.iteritems()
+                    (key, xmlescape(value).translate(ttab_in))
-                                 if not isinstance(value, NUMBERS) )
+                    for key, value in symbols.iteritems()
+                    if not isinstance(value, NUMBERS) )
             else:
                 if not isinstance(symbols, tuple):
                     symbols = (symbols,)
-                symbols = tuple(value if isinstance(value, NUMBERS)
+                symbols = tuple(
-                                    else xmlescape(value).translate(ttab_in)
+                    value if isinstance(value, NUMBERS)
-                                     for value in symbols)
+                    else xmlescape(value).translate(ttab_in)
+                    for value in symbols)
             message = self.params_substitution(message, symbols)
         return XML(message.translate(ttab_out))
 
-    def M(self, message, symbols={}, language=None, lazy=None, filter=None, ftag=None):
+    def M(self, message, symbols={}, language=None, 
-        """ get cached translated markmin-message with inserted parametes
+          lazy=None, filter=None, ftag=None):
-
+        """
-            if lazy==True lazyT object is returned
+        get cached translated markmin-message with inserted parametes
+        if lazy==True lazyT object is returned
         """
         if lazy is None:
             lazy = self.lazy
@@ -804,19 +791,21 @@ class translator(object):
         """
         get cached translated message with inserted parameters(symbols)
         """
-        message = get_from_cache(self.cache, message, lambda: self.get_t(message))
+        message = get_from_cache(self.cache, message, 
+                                 lambda: self.get_t(message))
         if symbols or symbols == 0 or symbols == "":
             if isinstance(symbols, dict):
-                symbols.update( (key, str(value).translate(ttab_in))
+                symbols.update(
-                                for key, value in symbols.iteritems()
+                    (key, str(value).translate(ttab_in))
-                                 if not isinstance(value, NUMBERS) )
+                    for key, value in symbols.iteritems()
+                    if not isinstance(value, NUMBERS) )
             else:
                 if not isinstance(symbols, tuple):
                     symbols = (symbols,)
-                symbols = tuple(value if isinstance(value, NUMBERS)
+                symbols = tuple(
-                                    else str(value).translate(ttab_in)
+                    value if isinstance(value, NUMBERS)
-                                     for value in symbols)
+                    else str(value).translate(ttab_in)
-
+                    for value in symbols)
             message = self.params_substitution(message, symbols)
         return message.translate(ttab_out)
 
@@ -830,26 +819,25 @@ def findT(path, language='en'):
     cp = ospath.join(path, 'controllers')
     vp = ospath.join(path, 'views')
     mop = ospath.join(path, 'modules')
-    for file in listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\
+    for filename in \
-         + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0):
+            listdir(mp, '^.+\.py$', 0)+listdir(cp, '^.+\.py$', 0)\
-        fp = portalocker.LockedFile(file, 'r')
+            +listdir(vp, '^.+\.html$', 0)+listdir(mop, '^.+\.py$', 0):
-        data = fp.read()
+        data = portalocker.read_locked(filename)
-        fp.close()
         items = regex_translate.findall(data)
         for item in items:
             try:
-                message = eval(item)
+                message = safe_eval(item)
-                if not message.startswith('#') and not '\n' in message:
-                    tokens = message.rsplit('##', 1)
-                else:
-                    # this allows markmin syntax in translations
-                    tokens = [message]
-                if len(tokens) == 2:
-                    message = tokens[0].strip() + '##' + tokens[1].strip()
-                if message and not message in sentences:
-                    sentences[message] = message
             except:
-                pass
+                continue # silently ignore inproperly formatted strings
+            if not message.startswith('#') and not '\n' in message:
+                tokens = message.rsplit('##', 1)
+            else:
+                # this allows markmin syntax in translations
+                tokens = [message]
+            if len(tokens) == 2:
+                message = tokens[0].strip()+'##'+tokens[1].strip()
+            if message and not message in sentences:
+                sentences[message] = message
     if not '!langcode!' in sentences:
         sentences['!langcode!'] = (
             'en' if language in ('default', 'en') else language)
@@ -875,8 +863,3 @@ def update_all_languages(application_path):
 if __name__ == '__main__':
     import doctest
     doctest.testmod()
-
-
-
-
-