#!/usr/bin/env python # -*- coding: utf-8 -*- """ This file is part of the web2py Web Framework Copyrighted by Massimo Di Pierro License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) """ import os import re import pkgutil from utf8 import Utf8 from cgi import escape import portalocker import logging import marshal import copy_reg from fileutils import listdir import settings from cfs import getcfs from thread import allocate_lock from html import XML, xmlescape from contrib.markmin.markmin2html import render, markmin_escape from string import maketrans __all__ = ['translator', 'findT', 'update_all_languages'] ostat = os.stat oslistdir = os.listdir pjoin = os.path.join pexists = os.path.exists pdirname = os.path.dirname isdir = os.path.isdir is_gae = settings.global_settings.web2py_runtime_gae DEFAULT_LANGUAGE = 'en' DEFAULT_LANGUAGE_NAME = 'English' # DEFAULT PLURAL-FORMS RULES: # language doesn't use plural forms DEFAULT_NPLURALS = 1 # only one singular/plural form is used DEFAULT_GET_PLURAL_ID = lambda n: 0 # word is unchangeable DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word NUMBERS = (int,long,float) # pattern to find T(blah blah blah) expressions PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P'\ + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\ + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\ + r'(?:"(?:[^"\\]|\\.)*"))' regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL) regex_param=re.compile(r'{(?P.+?)}') # pattern for a valid accept_language regex_language = \ re.compile('([a-z]{2}(?:\-[a-z]{2})?(?:\-[a-z]{2})?)(?:[,;]|$)') regex_langfile = re.compile('^[a-z]{2}(-[a-z]{2})?\.py$') regex_backslash = re.compile(r"\\([\\{}%])") regex_plural = re.compile('%({.+?})') regex_plural_dict = re.compile('^{(?P[^()[\]][^()[\]]*?)\((?P[^()\[\]]+)\)}$') # %%{word(varname or number)} regex_plural_tuple = re.compile('^{(?P[^[\]()]+)(?:\[(?P\d+)\])?}$') # %%{word[index]} or %%{word} regex_plural_file = re.compile('^plural-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$') def safe_eval(text): if text.strip(): try: import ast return ast.literal_eval(text) except ImportError: return eval(text,{},{}) return None # used as default filter in translator.M() def markmin(s): def markmin_aux(m): return '{%s}' % markmin_escape(m.group('s')) return render(regex_param.sub(markmin_aux,s), sep='br', autolinks=None, id_prefix='') # UTF8 helper functions def upper_fun(s): return unicode(s,'utf-8').upper().encode('utf-8') def title_fun(s): return unicode(s,'utf-8').title().encode('utf-8') def cap_fun(s): return unicode(s,'utf-8').capitalize().encode('utf-8') ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f') ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}") # cache of translated messages: # global_language_cache: # { 'languages/xx.py': # ( {"def-message": "xx-message", # ... # "def-message": "xx-message"}, lock_object ) # 'languages/yy.py': ( {dict}, lock_object ) # ... # } global_language_cache={} def get_from_cache(cache, val, fun): lang_dict, lock = cache lock.acquire() try: result = lang_dict.get(val); finally: lock.release() if result: return result lock.acquire() try: result = lang_dict.setdefault(val, fun()) finally: lock.release() return result def clear_cache(filename): cache = global_language_cache.setdefault( filename, ({}, allocate_lock())) lang_dict, lock = cache lock.acquire() try: lang_dict.clear(); finally: lock.release() def read_dict_aux(filename): lang_text = portalocker.read_locked(filename).replace('\r\n', '\n') clear_cache(filename) try: return safe_eval(lang_text) or {} except Exception, e: status = 'Syntax error in %s (%s)' % (filename, e) logging.error(status) return {'__corrupted__':status} def read_dict(filename): """ return dictionary with translation messages """ return getcfs('lang:'+filename, filename, lambda: read_dict_aux(filename)) def read_possible_plural_rules(): """ create list of all possible plural rules files result is cached in PLURAL_RULES dictionary to increase speed """ plurals = {} try: import contrib.plural_rules as package for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): if len(modname)==2: module = __import__(package.__name__+'.'+modname, fromlist=[modname]) lang = modname pname = modname+'.py' nplurals = getattr(module,'nplurals', DEFAULT_NPLURALS) get_plural_id = getattr( module,'get_plural_id', DEFAULT_GET_PLURAL_ID) construct_plural_form = getattr( module,'construct_plural_form', DEFAULT_CONSTRUCT_PLURAL_FORM) plurals[lang] = (lang, nplurals, get_plural_id, construct_plural_form) except ImportError, e: logging.warn('Unable to import plural rules: %s' % e) return plurals PLURAL_RULES = read_possible_plural_rules() def read_possible_languages_aux(langdir): def get_lang_struct(lang, langcode, langname, langfile_mtime): if lang == 'default': real_lang = langcode.lower() else: real_lang = lang (prules_langcode, nplurals, get_plural_id, construct_plural_form ) = PLURAL_RULES.get(real_lang[:2],('default', DEFAULT_NPLURALS, DEFAULT_GET_PLURAL_ID, DEFAULT_CONSTRUCT_PLURAL_FORM)) if prules_langcode != 'default': (pluraldict_fname, pluraldict_mtime) = plurals.get(real_lang, plurals.get(real_lang[:2], ('plural-%s.py'%real_lang,0))) else: pluraldict_fname = None pluraldict_mtime = 0 return (langcode, # language code from !langcode! langname, # language name in national spelling from !langname! langfile_mtime, # m_time of language file pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist) pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist prules_langcode, # code of plural rules language or 'default' nplurals, # nplurals for current language get_plural_id, # get_plural_id() for current language construct_plural_form) # construct_plural_form() for current language plurals = {} flist = oslistdir(langdir) # scan languages directory for plural dict files: for pname in flist: if regex_plural_file.match(pname): plurals[pname[7:-3]] = (pname, ostat(pjoin(langdir,pname)).st_mtime) langs = {} # scan languages directory for langfiles: for fname in flist: if regex_langfile.match(fname) or fname == 'default.py': fname_with_path = pjoin(langdir,fname) d = read_dict(fname_with_path) lang = fname[:-3] langcode = d.get('!langcode!', lang if lang != 'default' else DEFAULT_LANGUAGE) langname = d.get('!langname!',langcode) langfile_mtime = ostat(fname_with_path).st_mtime langs[lang] = get_lang_struct(lang, langcode, langname, langfile_mtime) if 'default' not in langs: # if default.py is not found, # add DEFAULT_LANGUAGE as default language: langs['default'] = get_lang_struct('default', DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_NAME, 0) deflang = langs['default'] deflangcode = deflang[0] if deflangcode not in langs: # create language from default.py: langs[deflangcode] = deflang[:2]+(0,)+deflang[3:] return langs def read_possible_languages(appdir): langdir = pjoin(appdir,'languages') return getcfs('langs:'+langdir, langdir, lambda: read_possible_languages_aux(langdir)) def read_plural_dict_aux(filename): lang_text = portalocker.read_locked(filename).replace('\r\n', '\n') try: return eval(lang_text) or {} except Exception, e: status='Syntax error in %s (%s)' % (filename, e) logging.error(status) return {'__corrupted__':status} def read_plural_dict(filename): return getcfs('plurals:'+filename, filename, lambda: read_plural_dict_aux(filename)) def write_plural_dict(filename, contents): if '__corrupted__' in contents: return try: fp = portalocker.LockedFile(filename, 'w') fp.write('#!/usr/bin/env python\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n') # coding: utf8\n{\n') for key in sorted(contents,lambda x,y: cmp(unicode(x,'utf-8').lower(), unicode(y,'utf-8').lower())): forms = '['+','.join([repr(Utf8(form)) for form in contents[key]])+']' fp.write('%s: %s,\n' % (repr(Utf8(key)), forms)) fp.write('}\n') except (IOError, OSError): if not is_gae: logging.warning('Unable to write to file %s' % filename) return finally: fp.close() def write_dict(filename, contents): if '__corrupted__' in contents: return try: fp = portalocker.LockedFile(filename, 'w') except (IOError, OSError): if not settings.global_settings.web2py_runtime_gae: logging.warning('Unable to write to file %s' % filename) return fp.write('# coding: utf8\n{\n') for key in sorted(contents,lambda x,y: cmp(unicode(x,'utf-8').lower(), unicode(y,'utf-8').lower())): fp.write('%s: %s,\n' % (repr(Utf8(key)), repr(Utf8(contents[key])))) fp.write('}\n') fp.close() class lazyT(object): """ never to be called explicitly, returned by translator.__call__() or translator.M() """ m = s = T = f = t = None M = is_copy = False def __init__( self, message, symbols = {}, T = None, filter = None, ftag = None, M = False ): if isinstance(message, lazyT): self.m = message.m self.s = message.s self.T = message.T self.f = message.f self.t = message.t self.M = message.M self.is_copy = True else: self.m = message self.s = symbols self.T = T self.f = filter self.t = ftag self.M = M self.is_copy = False def __repr__(self): return "" % (repr(Utf8(self.m)), ) def __str__(self): return str(self.T.apply_filter(self.m, self.s, self.f, self.t) if self.M else self.T.translate(self.m, self.s)) def __eq__(self, other): return str(self) == str(other) def __ne__(self, other): return str(self) != str(other) def __add__(self, other): return '%s%s' % (self, other) def __radd__(self, other): return '%s%s' % (other, self) def __mul__(self, other): return str(self) * other def __cmp__(self,other): return cmp(str(self), str(other)) def __hash__(self): return hash(str(self)) def __getattr__(self, name): return getattr(str(self), name) def __getitem__(self, i): return str(self)[i] def __getslice__(self, i, j): return str(self)[i:j] def __iter__(self): for c in str(self): yield c def __len__(self): return len(str(self)) def xml(self): return str(self) if self.M else escape(str(self)) def encode(self, *a, **b): return str(self).encode(*a, **b) def decode(self, *a, **b): return str(self).decode(*a, **b) def read(self): return str(self) def __mod__(self, symbols): if self.is_copy: return lazyT(self) return lazyT(self.m, symbols, self.T, self.f, self.t, self.M) class translator(object): """ this class is instantiated by gluon.compileapp.build_environment as the T object :: T.force(None) # turns off translation T.force('fr, it') # forces web2py to translate using fr.py or it.py T(\"Hello World\") # translates \"Hello World\" using the selected file notice 1: there is no need to force since, by default, T uses http_accept_language to determine a translation file. notice 2: en and en-en are considered different languages! notice 3: if language xx-yy is not found force() probes other similar languages using such algorithm: xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py """ def __init__(self, request): self.request = request self.folder = request.folder self.langpath = pjoin(self.folder, 'languages') self.http_accept_language = request.env.http_accept_language # filled in self.force(): #------------------------ # self.cache # self.accepted_language # self.language_file # self.plural_language # self.nplurals # self.get_plural_id # self.construct_plural_form # self.plural_file # self.plural_dict # self.requested_languages #---------------------------------------- # filled in self.set_current_languages(): #---------------------------------------- # self.default_language_file # self.default_t # self.current_languages self.set_current_languages() self.lazy = True self.otherTs = {} self.filter = markmin self.ftag = 'markmin' def get_possible_languages_info(self, lang=None): """ return info for selected language or dictionary with all possible languages info from APP/languages/*.py args: *lang* (str): language returns: if *lang* is defined: return tuple(langcode, langname, langfile_mtime, pluraldict_fname, pluraldict_mtime, prules_langcode, nplurals, get_plural_id, construct_plural_form) or None if *lang* is NOT defined: returns dictionary with all possible languages: { langcode(from filename): ( langcode, # language code from !langcode! langname, # language name in national spelling from !langname! langfile_mtime, # m_time of language file pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist) pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist prules_langcode, # code of plural rules language or 'default' nplurals, # nplurals for current language get_plural_id, # get_plural_id() for current language construct_plural_form) # construct_plural_form() for current language } """ info = read_possible_languages(self.folder) if lang: info = info.get(lang) return info def get_possible_languages(self): """ get list of all possible languages for current applications """ return list(set(self.current_languages + [lang for lang in read_possible_languages(self.folder).iterkeys() if lang != 'default'])) def set_current_languages(self, *languages): """ set current AKA "default" languages setting one of this languages makes force() function turn translation off to use default language """ if len(languages) == 1 and isinstance( languages[0], (tuple, list)): languages = languages[0] if not languages or languages[0] is None: # set default language from default.py/DEFAULT_LANGUAGE pl_info = self.get_possible_languages_info('default') if pl_info[2]==0: # langfile_mtime # if languages/default.py is not found self.default_language_file = self.langpath self.default_t = {} self.current_languages = [DEFAULT_LANGUAGE] else: self.default_language_file = pjoin(self.langpath, 'default.py') self.default_t = read_dict(self.default_language_file) self.current_languages = [pl_info[0]] # !langcode! else: self.current_languages = list(languages) self.force(self.http_accept_language) def plural(self, word, n): """ get plural form of word for number *n* NOTE: *word" MUST be defined in current language (T.accepted_language) invoked from T()/T.M() in %%{} tag args: word (str): word in singular n (numeric): number plural form created for returns: (str): word in appropriate singular/plural form """ if int(n) == 1: return word elif word: id = self.get_plural_id(abs(int(n))) # id = 0 singular form # id = 1 first plural form # id = 2 second plural form # etc. if id != 0: forms = self.plural_dict.get(word, []) if len(forms)>=id: # have this plural form: return forms[id-1] else: # guessing this plural form forms += ['']*(self.nplurals-len(forms)-1) form = self.construct_plural_form(word, id) forms[id-1] = form self.plural_dict[word] = forms if self.plural_file and not is_gae: write_plural_dict(self.plural_file, self.plural_dict) return form return word def force(self, *languages): """ select language(s) for translation if a list of languages is passed as a parameter, first language from this list that matches the ones from the possible_languages dictionary will be selected default language will be selected if none of them matches possible_languages. """ pl_info = read_possible_languages(self.folder) def set_plural(language): """ initialize plural forms subsystem """ lang_info = pl_info.get(language) if lang_info: (pname, pmtime, self.plural_language, self.nplurals, self.get_plural_id, self.construct_plural_form ) = lang_info[3:] pdict = {} if pname: pname = pjoin(self.langpath, pname) if pmtime != 0: pdict = read_plural_dict(pname) self.plural_file = pname self.plural_dict = pdict else: self.plural_language = 'default' self.nplurals = DEFAULT_NPLURALS self.get_plural_id = DEFAULT_GET_PLURAL_ID self.construct_plural_form = DEFAULT_CONSTRUCT_PLURAL_FORM self.plural_file = None self.plural_dict = {} language = '' if len(languages)==1 and isinstance(languages[0],str): languages = regex_language.findall(languages[0].lower()) elif not languages or languages[0] is None: languages = [] self.requested_languages = languages = tuple(languages) if languages: all_languages = set(lang for lang in pl_info.iterkeys() if lang != 'default') \ | set(self.current_languages) for lang in languages: # compare "aa-bb" | "aa" from *language* parameter # with strings from langlist using such alghorythm: # xx-yy.py -> xx.py -> xx*.py lang5 = lang[:5] if lang5 in all_languages: language = lang5 else: lang2 = lang[:2] if len(lang5)>2 and lang2 in all_languages: language = lang2 else: for l in all_languages: if l[:2]==lang2: language = l if language: if language in self.current_languages: break self.language_file = pjoin(self.langpath, language+'.py') self.t = read_dict(self.language_file) self.cache = global_language_cache.setdefault( self.language_file, ({},allocate_lock())) set_plural(language) self.accepted_language = language return languages self.accepted_language = language or self.current_languages[0] self.language_file = self.default_language_file self.cache = global_language_cache.setdefault(self.language_file, ({}, allocate_lock())) self.t = self.default_t set_plural(self.accepted_language) return languages def __call__(self, message, symbols={}, language=None, lazy=None): """ get cached translated plain text message with inserted parameters(symbols) if lazy==True lazyT object is returned """ if lazy is None: lazy = self.lazy if not language: if lazy : return lazyT(message, symbols, self) else: return self.translate(message, symbols) else: try: otherT = self.otherTs[language] except KeyError: otherT = self.otherTs[language] = translator(self.request) otherT.force(language) return otherT(message, symbols, lazy=lazy) def apply_filter(self, message, symbols={}, filter=None, ftag=None): def get_tr(message, prefix, filter): s = self.get_t(message, prefix) return filter(s) if filter else self.filter(s) if filter: prefix = '@'+(ftag or 'userdef')+'\x01' else: prefix = '@'+self.ftag+'\x01' message = get_from_cache( self.cache, prefix+message, lambda: get_tr(message, prefix, filter)) if symbols or symbols == 0 or symbols == "": if isinstance(symbols, dict): symbols.update( (key, xmlescape(value).translate(ttab_in)) for key, value in symbols.iteritems() if not isinstance(value, NUMBERS) ) else: if not isinstance(symbols, tuple): symbols = (symbols,) symbols = tuple( value if isinstance(value, NUMBERS) else xmlescape(value).translate(ttab_in) for value in symbols) message = self.params_substitution(message, symbols) return XML(message.translate(ttab_out)) def M(self, message, symbols={}, language=None, lazy=None, filter=None, ftag=None): """ get cached translated markmin-message with inserted parametes if lazy==True lazyT object is returned """ if lazy is None: lazy = self.lazy if not language: if lazy: return lazyT(message, symbols, self, filter, ftag, True) else: return self.apply_filter(message, symbols, filter, ftag) else: try: otherT = self.otherTs[language] except KeyError: otherT = self.otherTs[language] = translator(self.request) otherT.force(language) return otherT.M(message, symbols, lazy=lazy) def get_t(self, message, prefix=''): """ user ## to add a comment into a translation string the comment can be useful do discriminate different possible translations for the same string (for example different locations) T(' hello world ') -> ' hello world ' T(' hello world ## token') -> ' hello world ' T('hello ## world## token') -> 'hello ## world' the ## notation is ignored in multiline strings and strings that start with ##. this is to allow markmin syntax to be translated """ if isinstance(message, unicode): message = message.encode('utf8') if isinstance(prefix, unicode): prefix = prefix.encode('utf8') key = prefix+message mt = self.t.get(key, None) if mt is not None: return mt # we did not find a translation if message.find('##')>0 and not '\n' in message: # remove comments message = message.rsplit('##', 1)[0] # guess translation same as original self.t[key] = mt = self.default_t.get(key, message) # update language file for latter translation if self.language_file != self.default_language_file and not is_gae: write_dict(self.language_file, self.t) return regex_backslash.sub( lambda m: m.group(1).translate(ttab_in), mt) def params_substitution(self, message, symbols): """ substitute parameters from symbols into message using %. also parse %%{} placeholders for plural-forms processing. returns: string with parameters NOTE: *symbols* MUST BE OR tuple OR dict of parameters! """ def sub_plural(m): """string in %{} is transformed by this rules: If string starts with \\, ! or ? such transformations take place: "!string of words" -> "String of word" (Capitalize) "!!string of words" -> "String Of Word" (Title) "!!!string of words" -> "STRING OF WORD" (Upper) "\\!string of words" -> "!string of word" (remove \\ and disable transformations) "?word?number" -> "word" (return word, if number == 1) "?number" or "??number" -> "" (remove number, if number == 1) "?word?number" -> "number" (if number != 1) """ def sub_tuple(m): """ word[number], !word[number], !!word[number], !!!word[number] word, !word, !!word, !!!word, ?word?number, ??number, ?number ?word?word[number], ?word?[number], ??word[number] """ w,i = m.group('w','i') c = w[0] if c not in '!?': return self.plural(w, symbols[int(i or 0)]) elif c == '?': (p1, sep, p2) = w[1:].partition("?") part1 = p1 if sep else "" (part2, sep, part3) = (p2 if sep else p1).partition("?") if not sep: part3 = part2 if i is None: # ?[word]?number[?number] or ?number if not part2: return m.group(0) num = int(part2) else: # ?[word]?word2[?word3][number] num = int(symbols[int(i or 0)]) return part1 if num==1 else part3 if num==0 else part2 elif w.startswith('!!!'): word = w[3:] fun = upper_fun elif w.startswith('!!'): word = w[2:] fun = title_fun else: word = w[1:] fun = cap_fun if i is not None: return fun(self.plural(word, symbols[int(i)])) return fun(word) def sub_dict(m): """ word(var), !word(var), !!word(var), !!!word(var) word(num), !word(num), !!word(num), !!!word(num) ?word2(var), ?word1?word2(var), ?word1?word2?word0(var) ?word2(num), ?word1?word2(num), ?word1?word2?word0(num) """ w,n = m.group('w','n') c = w[0] n = int(n) if n.isdigit() else symbols[n] if c not in '!?': return self.plural(w, n) elif c == '?': # ?[word1]?word2[?word0](var or num), ?[word1]?word2(var or num) or ?word2(var or num) (p1, sep, p2) = w[1:].partition("?") part1 = p1 if sep else "" (part2, sep, part3) = (p2 if sep else p1).partition("?") if not sep: part3 = part2 num = int(n) return part1 if num==1 else part3 if num==0 else part2 elif w.startswith('!!!'): word = w[3:] fun = upper_fun elif w.startswith('!!'): word = w[2:] fun = title_fun else: word = w[1:] fun = cap_fun return fun(self.plural(word, n)) s = m.group(1) part = regex_plural_tuple.sub(sub_tuple, s) if part == s: part = regex_plural_dict.sub(sub_dict, s) if part == s: return m.group(0) return part message = message % symbols message = regex_plural.sub(sub_plural, message ) return message def translate(self, message, symbols): """ get cached translated message with inserted parameters(symbols) """ message = get_from_cache(self.cache, message, lambda: self.get_t(message)) if symbols or symbols == 0 or symbols == "": if isinstance(symbols, dict): symbols.update( (key, str(value).translate(ttab_in)) for key, value in symbols.iteritems() if not isinstance(value, NUMBERS) ) else: if not isinstance(symbols, tuple): symbols = (symbols,) symbols = tuple( value if isinstance(value, NUMBERS) else str(value).translate(ttab_in) for value in symbols) message = self.params_substitution(message, symbols) return message.translate(ttab_out) def findT(path, language=DEFAULT_LANGUAGE): """ must be run by the admin app """ lang_file = pjoin(path, 'languages', language + '.py') sentences = read_dict(lang_file) mp = pjoin(path, 'models') cp = pjoin(path, 'controllers') vp = pjoin(path, 'views') mop = pjoin(path, 'modules') for filename in \ listdir(mp, '^.+\.py$', 0)+listdir(cp, '^.+\.py$', 0)\ +listdir(vp, '^.+\.html$', 0)+listdir(mop, '^.+\.py$', 0): data = portalocker.read_locked(filename) items = regex_translate.findall(data) for item in items: try: message = safe_eval(item) except: continue # silently ignore inproperly formatted strings if not message.startswith('#') and not '\n' in message: tokens = message.rsplit('##', 1) else: # this allows markmin syntax in translations tokens = [message] if len(tokens) == 2: message = tokens[0].strip()+'##'+tokens[1].strip() if message and not message in sentences: sentences[message] = message if not '!langcode!' in sentences: sentences['!langcode!'] = ( DEFAULT_LANGUAGE if language in ('default', DEFAULT_LANGUAGE) else language) if not '!langname!' in sentences: sentences['!langname!'] = ( DEFAULT_LANGUAGE_NAME if language in ('default', DEFAULT_LANGUAGE) else sentences['!langcode!']) write_dict(lang_file, sentences) ### important to allow safe session.flash=T(....) def lazyT_unpickle(data): return marshal.loads(data) def lazyT_pickle(data): return lazyT_unpickle, (marshal.dumps(str(data)),) copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle) def update_all_languages(application_path): path = pjoin(application_path, 'languages/') for language in oslistdir(path): if regex_langfile.match(language): findT(application_path, language[:-3]) if __name__ == '__main__': import doctest doctest.testmod()