From 02e7f38da7995e711698bf0fb2294d4243a96a73 Mon Sep 17 00:00:00 2001 From: Massimo Di Pierro Date: Wed, 29 Feb 2012 13:38:59 -0600 Subject: [PATCH] fixed a major problem with locking in languages, thanks Dan McGee fr the help --- VERSION | 2 +- gluon/dal.py | 21 ++++++---------- gluon/globals.py | 3 +-- gluon/languages.py | 12 +++------ gluon/portalocker.py | 56 ++++++++++++++++++++++++++++------------- gluon/storage.py | 15 +++++------ gluon/tests/test_dal.py | 5 +++- 7 files changed, 61 insertions(+), 53 deletions(-) diff --git a/VERSION b/VERSION index 50b1cc8f..06902d87 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 1.99.4 (2012-02-29 11:11:18) stable +Version 1.99.4 (2012-02-29 13:38:37) stable diff --git a/gluon/dal.py b/gluon/dal.py index 93f3b164..b315cc3d 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -541,24 +541,17 @@ class BaseAdapter(ConnectionPool): """ to be used ONLY for files that on GAE may not be on filesystem """ - fileobj = open(filename, mode) if have_portalocker and lock: - if mode in ('r','rb'): - portalocker.lock(fileobj, portalocker.LOCK_SH) - elif mode in ('w','wb','a'): - portalocker.lock(fileobj, portalocker.LOCK_EX) - else: - fileobj.close() - raise RuntimeError, "Unsupported file_open mode" + fileobj = portalocker.LockedFile(filename,mode) + else: + fileobj = open(filename,mode) return fileobj - def file_close(self, fileobj, unlock=True): + def file_close(self, fileobj): """ to be used ONLY for files that on GAE may not be on filesystem """ if fileobj: - if have_portalocker and unlock: - portalocker.unlock(fileobj) fileobj.close() def file_delete(self, filename): @@ -3293,7 +3286,7 @@ class UseDatabaseStoredFile: def file_open(self, filename, mode='rb', lock=True): return DatabaseStoredFile(self.db,filename,mode) - def file_close(self, fileobj, unlock=True): + def file_close(self, fileobj): fileobj.close() def file_delete(self,filename): @@ -3503,7 +3496,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): def file_exists(self, filename): pass def file_open(self, filename, mode='rb', lock=True): pass - def file_close(self, fileobj, unlock=True): pass + def file_close(self, fileobj): pass def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', credential_decoder=lambda x:x, driver_args={}, @@ -3854,7 +3847,7 @@ class CouchDBAdapter(NoSQLAdapter): def file_exists(self, filename): pass def file_open(self, filename, mode='rb', lock=True): pass - def file_close(self, fileobj, unlock=True): pass + def file_close(self, fileobj): pass def expand(self,expression,field_type=None): if isinstance(expression,Field): diff --git a/gluon/globals.py b/gluon/globals.py index 92e76f33..bf3d7ae8 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -437,8 +437,7 @@ class Session(Storage): response.session_file = \ open(response.session_filename, 'rb+') try: - portalocker.lock(response.session_file, - portalocker.LOCK_EX) + portalocker.lock(response.session_file,portalocker.LOCK_EX) response.session_locked = True self.update(cPickle.load(response.session_file)) response.session_file.seek(0) diff --git a/gluon/languages.py b/gluon/languages.py index 07d6afdd..cf222401 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -38,10 +38,8 @@ regex_language = \ def read_dict_aux(filename): - fp = open(filename, 'r') - portalocker.lock(fp, portalocker.LOCK_SH) + fp = portalocker.LockedFile(filename, 'r') lang_text = fp.read().replace('\r\n', '\n') - portalocker.unlock(fp) # needed or test_languages.py fails fp.close() if not lang_text.strip(): return {} @@ -91,17 +89,15 @@ def utf8_repr(s): def write_dict(filename, contents): try: - fp = open(filename, 'w') + fp = portalocker.LockedFile(filename, 'w') except (IOError, OSError): if not is_gae: logging.warning('Unable to write to file %s' % filename) return - portalocker.lock(fp, portalocker.LOCK_EX) fp.write('# coding: utf8\n{\n') for key in sorted(contents): fp.write('%s: %s,\n' % (utf8_repr(key), utf8_repr(contents[key]))) fp.write('}\n') - portalocker.unlock(fp) fp.close() @@ -312,10 +308,8 @@ def findT(path, language='en-us'): vp = os.path.join(path, 'views') for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\ + listdir(vp, '.+\.html', 0): - fp = open(file, 'r') - portalocker.lock(fp, portalocker.LOCK_EX) + fp = portalocker.LockedFile(file, 'r') data = fp.read() - portalocker.unlock(fp) fp.close() items = regex_translate.findall(data) for item in items: diff --git a/gluon/portalocker.py b/gluon/portalocker.py index c3bb007d..eb002025 100644 --- a/gluon/portalocker.py +++ b/gluon/portalocker.py @@ -108,22 +108,44 @@ else: pass -if __name__ == '__main__': - from time import time, strftime, localtime - import sys - - log = open('log.txt', 'a+') - lock(log, LOCK_EX) - - timestamp = strftime('%m/%d/%Y %H:%M:%S\n', localtime(time())) - log.write(timestamp) - - print 'Wrote lines. Hit enter to release lock.' - dummy = sys.stdin.readline() - - log.close() - - - +class LockedFile(object): + def __init__(self,filename, mode='rb'): + self.filename = filename + self.mode = mode + self.file = None + if 'r' in mode: + self.file = open(filename,mode) + lock(self.file,LOCK_SH) + elif 'w' in mode or 'a' in mode: + self.file = open(filename,mode.replace('w','a')) + lock(self.file,LOCK_EX) + if not 'a' in mode: + self.file.seek(0) + self.file.truncate() + else: + raise RuntimeError, "invalid LockedFile(...,mode)" + def read(self,size=None): + return self.file.read() if size is None else self.file.read(size) + def readline(self): + return self.file.readline() + def readlines(self): + return self.file.readlines() + def write(self,data): + self.file.write(data) + self.file.flush() + def close(self): + if not self.file is None: + unlock(self.file) + self.file.close() + self.file = None + def __del__(self): + self.close() +if __name__=='__main__': + f = LockedFile('test.txt',mode='wb') + f.write('test ok') + f.close() + f = LockedFile('test.txt',mode='rb') + print f.read() + f.close() diff --git a/gluon/storage.py b/gluon/storage.py index efa8f966..0e00bff7 100644 --- a/gluon/storage.py +++ b/gluon/storage.py @@ -172,25 +172,22 @@ class StorageList(Storage): return self[key] def load_storage(filename): - fp = open(filename, 'rb') + fp = None try: - portalocker.lock(fp, portalocker.LOCK_EX) + fp = portalocker.LockFile(filename, 'rb') storage = cPickle.load(fp) - portalocker.unlock(fp) finally: - fp.close() + if fp: fp.close() return Storage(storage) def save_storage(storage, filename): - fp = open(filename, 'wb') + fp = None try: - portalocker.lock(fp, portalocker.LOCK_EX) + fp = portalocker.LockFile(filename, 'wb') cPickle.dump(dict(storage), fp) - portalocker.unlock(fp) finally: - fp.close() - + if fp: fp.close() class Settings(Storage): diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index f5aaaf28..101bee95 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -421,7 +421,10 @@ class TestMigrations(unittest.TestCase): db.commit() def tearDown(self): - os.unlink('.storage.db') + if os.path.exists('.storage.db'): + os.unlink('.storage.db') + if os.path.exists('.storage.table'): + os.unlink('.storage.table') class TestReferece(unittest.TestCase):