From 2bb348582718e585645be2c2e30aaf74a08b125a Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 22 Oct 2012 18:50:18 -0500 Subject: [PATCH 01/27] minor fix in dal for rare condition --- VERSION | 2 +- gluon/dal.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/VERSION b/VERSION index d833dcfe..c301a9a6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-22 15:07:32) stable +Version 2.2.1 (2012-10-22 18:50:13) stable diff --git a/gluon/dal.py b/gluon/dal.py index e2088dc9..91f05762 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -8613,7 +8613,6 @@ class Field(Expression): if not isinstance(requires, (list, tuple)): requires = [requires] for validator in requires: - print validator (value, error) = validator(value) if error: return (value, error) From 9e897dcc46a20db04ce65a87fcb54774b6257d81 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 22 Oct 2012 22:14:40 -0500 Subject: [PATCH 02/27] language changes --- VERSION | 2 +- applications/admin/controllers/default.py | 6 +++--- applications/welcome/languages/it.py | 1 + gluon/compileapp.py | 3 ++- gluon/languages.py | 24 +++++++++++------------ gluon/tests/test_languages.py | 12 +++++------- 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/VERSION b/VERSION index c301a9a6..8c46a8a0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-22 18:50:13) stable +Version 2.2.1 (2012-10-22 22:14:35) stable diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 97293161..3daad36b 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -998,11 +998,11 @@ def design(): statics.sort() # Get all languages + langpath = os.path.join(apath(app, r=request),'language') languages = dict([(lang, info) for lang, info - in read_possible_languages( - apath(app, r=request)).iteritems() + in read_possible_languages(langpath).iteritems() if info[2] != 0]) # info[2] is langfile_mtime: - # get only existed files + # get only existed files #Get crontab cronfolder = apath('%s/cron' % app, r=request) diff --git a/applications/welcome/languages/it.py b/applications/welcome/languages/it.py index fcd9c933..17988e3c 100644 --- a/applications/welcome/languages/it.py +++ b/applications/welcome/languages/it.py @@ -6,6 +6,7 @@ '"update" is an optional expression like "field1=\'newvalue\'". You cannot update or delete the results of a JOIN': '"update" è un\'espressione opzionale come "campo1=\'nuovo valore\'". Non si può fare "update" o "delete" dei risultati di un JOIN ', '%(nrows)s records found': '%(nrows)s records found', '%d seconds ago': '%d seconds ago', +'%m/%d/%Y': '%m/%d/%Y', '%s %%{row} deleted': '%s righe ("record") cancellate', '%s %%{row} updated': '%s righe ("record") modificate', '%s selected': '%s selezionato', diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 5f145888..33ee0564 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -396,7 +396,8 @@ def build_environment(request, response, session, store_current=True): response.models_to_run = [r'^\w+\.py$', r'^%s/\w+\.py$' % request.controller, r'^%s/%s/\w+\.py$' % (request.controller, request.function)] - t = environment['T'] = translator(request) + t = environment['T'] = translator(os.path.join(request.folder,'languages'), + request.env.http_accept_langauge) c = environment['cache'] = Cache(request) if store_current: diff --git a/gluon/languages.py b/gluon/languages.py index 7f25c5d8..8ed8b6e1 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -261,10 +261,9 @@ def read_possible_languages_aux(langdir): return langs -def read_possible_languages(appdir): - langdir = pjoin(appdir, 'languages') - return getcfs('langs:' + langdir, langdir, - lambda: read_possible_languages_aux(langdir)) +def read_possible_languages(langpath): + return getcfs('langs:' + langpath, langpath, + lambda: read_possible_languages_aux(langpath)) def read_plural_dict_aux(filename): @@ -434,11 +433,9 @@ class translator(object): xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py """ - def __init__(self, request): - self.request = request - self.folder = request.folder - self.langpath = pjoin(self.folder, 'languages') - self.http_accept_language = request.env.http_accept_language + def __init__(self, langpath, http_accept_language): + self.langpath = langpath + self.http_accept_language = http_accept_language self.is_writable = not is_gae # filled in self.force(): #------------------------ @@ -493,7 +490,7 @@ class translator(object): construct_plural_form) # construct_plural_form() for current language } """ - info = read_possible_languages(self.folder) + info = read_possible_languages(self.langpath) if lang: info = info.get(lang) return info @@ -501,7 +498,7 @@ class translator(object): def get_possible_languages(self): """ get list of all possible languages for current applications """ return list(set(self.current_languages + - [lang for lang in read_possible_languages(self.folder).iterkeys() + [lang for lang in read_possible_languages(self.langpath).iterkeys() if lang != 'default'])) def set_current_languages(self, *languages): @@ -581,7 +578,7 @@ class translator(object): default language will be selected if none of them matches possible_languages. """ - pl_info = read_possible_languages(self.folder) + pl_info = read_possible_languages(self.langpath) def set_plural(language): """ @@ -670,7 +667,8 @@ class translator(object): try: otherT = self.otherTs[language] except KeyError: - otherT = self.otherTs[language] = translator(self.request) + otherT = self.otherTs[language] = translator( + self.langpath, self.http_accept_language) otherT.force(language) return otherT(message, symbols, lazy=lazy) diff --git a/gluon/tests/test_languages.py b/gluon/tests/test_languages.py index 911ddcd2..ece446c8 100644 --- a/gluon/tests/test_languages.py +++ b/gluon/tests/test_languages.py @@ -57,20 +57,18 @@ try: class TestTranslations(unittest.TestCase): def setUp(self): - self.request = Storage() if os.path.isdir('gluon'): - self.request.folder = 'applications/welcome' + self.langpath = 'applications/welcome/languages' else: - self.request.folder = os.path.realpath( - '../../applications/welcome') - self.request.env = Storage() - self.request.env.http_accept_language = 'en' + self.langpath = os.path.realpath( + '../../applications/welcome/languages') + self.http_accept_language = 'en' def tearDown(self): pass def test_plain(self): - T = languages.translator(self.request) + T = languages.translator(self.langpath, self.http_accept_language) self.assertEqual(str(T('Hello World')), 'Hello World') self.assertEqual(str(T('Hello World## comment')), From 752bb7e048276c2f40d421c03b2c3ee9c87a63c3 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 22 Oct 2012 22:21:11 -0500 Subject: [PATCH 03/27] portable languages --- VERSION | 2 +- applications/admin/controllers/default.py | 2 +- applications/welcome/languages/it.py | 1 - gluon/compileapp.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 8c46a8a0..6bbf2868 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-22 22:14:35) stable +Version 2.2.1 (2012-10-22 22:21:06) stable diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 3daad36b..1de5098e 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -998,7 +998,7 @@ def design(): statics.sort() # Get all languages - langpath = os.path.join(apath(app, r=request),'language') + langpath = os.path.join(apath(app, r=request),'languages') languages = dict([(lang, info) for lang, info in read_possible_languages(langpath).iteritems() if info[2] != 0]) # info[2] is langfile_mtime: diff --git a/applications/welcome/languages/it.py b/applications/welcome/languages/it.py index 17988e3c..fcd9c933 100644 --- a/applications/welcome/languages/it.py +++ b/applications/welcome/languages/it.py @@ -6,7 +6,6 @@ '"update" is an optional expression like "field1=\'newvalue\'". You cannot update or delete the results of a JOIN': '"update" è un\'espressione opzionale come "campo1=\'nuovo valore\'". Non si può fare "update" o "delete" dei risultati di un JOIN ', '%(nrows)s records found': '%(nrows)s records found', '%d seconds ago': '%d seconds ago', -'%m/%d/%Y': '%m/%d/%Y', '%s %%{row} deleted': '%s righe ("record") cancellate', '%s %%{row} updated': '%s righe ("record") modificate', '%s selected': '%s selezionato', diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 33ee0564..b98e77aa 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -397,7 +397,7 @@ def build_environment(request, response, session, store_current=True): r'^%s/%s/\w+\.py$' % (request.controller, request.function)] t = environment['T'] = translator(os.path.join(request.folder,'languages'), - request.env.http_accept_langauge) + request.env.http_accept_language) c = environment['cache'] = Cache(request) if store_current: From 0eab35842d33f5e403eaccbe366d3d730ff4cd87 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 22 Oct 2012 22:49:35 -0500 Subject: [PATCH 04/27] THREAD_LOCAL folder not reset in dal --- VERSION | 2 +- gluon/dal.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/VERSION b/VERSION index 6bbf2868..da4a1fe3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-22 22:21:06) stable +Version 2.2.1 (2012-10-22 22:49:31) stable diff --git a/gluon/dal.py b/gluon/dal.py index 91f05762..ad6969f0 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -6625,14 +6625,6 @@ class DAL(object): db._db_uid = db_uid return db - @staticmethod - def set_folder(folder): - """ - # ## this allows gluon to set a folder for this thread - # ## <<<<<<<<< Should go away as new DAL replaces old sql.py - """ - BaseAdapter.set_folder(folder) - @staticmethod def distributed_transaction_begin(*instances): if not instances: @@ -6712,8 +6704,6 @@ class DAL(object): credential_decoder = lambda cred: cred else: credential_decoder = lambda cred: urllib.unquote(cred) - if folder: - self.set_folder(folder) self._uri = uri self._pool_size = pool_size self._db_codec = db_codec From 8c514df1200573478c9179365a2aa7f94237e887 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 22 Oct 2012 22:52:40 -0500 Subject: [PATCH 05/27] reverted previous change but still problems with concurency multiple dbs in different folders in same app --- VERSION | 2 +- gluon/dal.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index da4a1fe3..08e7b922 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-22 22:49:31) stable +Version 2.2.1 (2012-10-22 22:52:36) stable diff --git a/gluon/dal.py b/gluon/dal.py index ad6969f0..91f05762 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -6625,6 +6625,14 @@ class DAL(object): db._db_uid = db_uid return db + @staticmethod + def set_folder(folder): + """ + # ## this allows gluon to set a folder for this thread + # ## <<<<<<<<< Should go away as new DAL replaces old sql.py + """ + BaseAdapter.set_folder(folder) + @staticmethod def distributed_transaction_begin(*instances): if not instances: @@ -6704,6 +6712,8 @@ class DAL(object): credential_decoder = lambda cred: cred else: credential_decoder = lambda cred: urllib.unquote(cred) + if folder: + self.set_folder(folder) self._uri = uri self._pool_size = pool_size self._db_codec = db_codec From ab066397b201cfaf32cfddd227b5447fe1da2579 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 09:44:34 -0500 Subject: [PATCH 06/27] aes new raise compliant --- VERSION | 2 +- gluon/contrib/aes.py | 10 +++++----- gluon/rocket.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/VERSION b/VERSION index 08e7b922..b53a93fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-22 22:52:36) stable +Version 2.2.1 (2012-10-23 09:44:29) stable diff --git a/gluon/contrib/aes.py b/gluon/contrib/aes.py index 24e40878..cecf2d90 100644 --- a/gluon/contrib/aes.py +++ b/gluon/contrib/aes.py @@ -64,7 +64,7 @@ def new(key, mode=MODE_CBC, IV=None): return ECBMode(AES(key)) elif mode == MODE_CBC: if IV is None: - raise ValueError, "CBC mode needs an IV value!" + raise ValueError("CBC mode needs an IV value!") return CBCMode(AES(key), IV) else: @@ -91,7 +91,7 @@ class AES(object): elif self.key_size == 32: self.rounds = 14 else: - raise ValueError, "Key length must be 16, 24 or 32 bytes" + raise ValueError("Key length must be 16, 24 or 32 bytes") self.expand_key() @@ -313,7 +313,7 @@ class ECBMode(object): """Perform ECB mode with the given function""" if len(data) % self.block_size != 0: - raise ValueError, "Plaintext length must be multiple of 16" + raise ValueError("Plaintext length must be multiple of 16") block_size = self.block_size data = array('B', data) @@ -357,7 +357,7 @@ class CBCMode(object): block_size = self.block_size if len(data) % block_size != 0: - raise ValueError, "Plaintext length must be multiple of 16" + raise ValueError("Plaintext length must be multiple of 16") data = array('B', data) IV = self.IV @@ -381,7 +381,7 @@ class CBCMode(object): block_size = self.block_size if len(data) % block_size != 0: - raise ValueError, "Ciphertext length must be multiple of 16" + raise ValueError("Ciphertext length must be multiple of 16") data = array('B', data) IV = self.IV diff --git a/gluon/rocket.py b/gluon/rocket.py index af5e7133..c5dfff76 100644 --- a/gluon/rocket.py +++ b/gluon/rocket.py @@ -1670,8 +1670,8 @@ class WSGIWorker(Worker): peercert = conn.socket.getpeercert(binary_form=True) environ['SSL_CLIENT_RAW_CERT'] = \ peercert and ssl.DER_cert_to_PEM_cert(peercert) - except Exception, e: - print e + except Exception: + print sys.exc_info()[1] if environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked': environ['wsgi.input'] = ChunkedReader(sock_file) From 4152c72de51d73b936d200251a3bb58a1424a609 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 09:54:07 -0500 Subject: [PATCH 07/27] aes python 3 compiles (but does work well?) --- VERSION | 2 +- gluon/contrib/aes.py | 115 ++++++++++++++++++++++++------------------- 2 files changed, 64 insertions(+), 53 deletions(-) diff --git a/VERSION b/VERSION index b53a93fb..2fc3e7fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 09:44:29) stable +Version 2.2.1 (2012-10-23 09:54:01) stable diff --git a/gluon/contrib/aes.py b/gluon/contrib/aes.py index cecf2d90..9ca2a3b3 100644 --- a/gluon/contrib/aes.py +++ b/gluon/contrib/aes.py @@ -45,8 +45,16 @@ there clears it up. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +# +# Ported to python 3 by Massimo Di Pierro +# #### +try: + import binascii # python 3 + str2hex = lambda s: binascii.b2a_hex(s.encode('utf8')) +except ImportError: # python 2 + str2hex = lambda s: s.decode('hex') from array import array @@ -436,67 +444,70 @@ gf_mul_by_14 = array('B', [galois_multiply(x, 14) for x in range(256)]) # # More information: http://en.wikipedia.org/wiki/Rijndael_S-box -aes_sbox = array('B', - '637c777bf26b6fc53001672bfed7ab76' - 'ca82c97dfa5947f0add4a2af9ca472c0' - 'b7fd9326363ff7cc34a5e5f171d83115' - '04c723c31896059a071280e2eb27b275' - '09832c1a1b6e5aa0523bd6b329e32f84' - '53d100ed20fcb15b6acbbe394a4c58cf' - 'd0efaafb434d338545f9027f503c9fa8' - '51a3408f929d38f5bcb6da2110fff3d2' - 'cd0c13ec5f974417c4a77e3d645d1973' - '60814fdc222a908846eeb814de5e0bdb' - 'e0323a0a4906245cc2d3ac629195e479' - 'e7c8376d8dd54ea96c56f4ea657aae08' - 'ba78252e1ca6b4c6e8dd741f4bbd8b8a' - '703eb5664803f60e613557b986c11d9e' - 'e1f8981169d98e949b1e87e9ce5528df' - '8ca1890dbfe6426841992d0fb054bb16'.decode('hex') +aes_sbox = array( + 'B', + str2hex('637c777bf26b6fc53001672bfed7ab76' + 'ca82c97dfa5947f0add4a2af9ca472c0' + 'b7fd9326363ff7cc34a5e5f171d83115' + '04c723c31896059a071280e2eb27b275' + '09832c1a1b6e5aa0523bd6b329e32f84' + '53d100ed20fcb15b6acbbe394a4c58cf' + 'd0efaafb434d338545f9027f503c9fa8' + '51a3408f929d38f5bcb6da2110fff3d2' + 'cd0c13ec5f974417c4a77e3d645d1973' + '60814fdc222a908846eeb814de5e0bdb' + 'e0323a0a4906245cc2d3ac629195e479' + 'e7c8376d8dd54ea96c56f4ea657aae08' + 'ba78252e1ca6b4c6e8dd741f4bbd8b8a' + '703eb5664803f60e613557b986c11d9e' + 'e1f8981169d98e949b1e87e9ce5528df' + '8ca1890dbfe6426841992d0fb054bb16') ) # This is the inverse of the above. In other words: # aes_inv_sbox[aes_sbox[val]] == val -aes_inv_sbox = array('B', - '52096ad53036a538bf40a39e81f3d7fb' - '7ce339829b2fff87348e4344c4dee9cb' - '547b9432a6c2233dee4c950b42fac34e' - '082ea16628d924b2765ba2496d8bd125' - '72f8f66486689816d4a45ccc5d65b692' - '6c704850fdedb9da5e154657a78d9d84' - '90d8ab008cbcd30af7e45805b8b34506' - 'd02c1e8fca3f0f02c1afbd0301138a6b' - '3a9111414f67dcea97f2cfcef0b4e673' - '96ac7422e7ad3585e2f937e81c75df6e' - '47f11a711d29c5896fb7620eaa18be1b' - 'fc563e4bc6d279209adbc0fe78cd5af4' - '1fdda8338807c731b11210592780ec5f' - '60517fa919b54a0d2de57a9f93c99cef' - 'a0e03b4dae2af5b0c8ebbb3c83539961' - '172b047eba77d626e169146355210c7d'.decode('hex') -) +aes_inv_sbox = array( + 'B', + str2hex('52096ad53036a538bf40a39e81f3d7fb' + '7ce339829b2fff87348e4344c4dee9cb' + '547b9432a6c2233dee4c950b42fac34e' + '082ea16628d924b2765ba2496d8bd125' + '72f8f66486689816d4a45ccc5d65b692' + '6c704850fdedb9da5e154657a78d9d84' + '90d8ab008cbcd30af7e45805b8b34506' + 'd02c1e8fca3f0f02c1afbd0301138a6b' + '3a9111414f67dcea97f2cfcef0b4e673' + '96ac7422e7ad3585e2f937e81c75df6e' + '47f11a711d29c5896fb7620eaa18be1b' + 'fc563e4bc6d279209adbc0fe78cd5af4' + '1fdda8338807c731b11210592780ec5f' + '60517fa919b54a0d2de57a9f93c99cef' + 'a0e03b4dae2af5b0c8ebbb3c83539961' + '172b047eba77d626e169146355210c7d') + ) # The Rcon table is used in AES's key schedule (key expansion) # It's a pre-computed table of exponentation of 2 in AES's finite field # # More information: http://en.wikipedia.org/wiki/Rijndael_key_schedule -aes_Rcon = array('B', - '8d01020408102040801b366cd8ab4d9a' - '2f5ebc63c697356ad4b37dfaefc59139' - '72e4d3bd61c29f254a943366cc831d3a' - '74e8cb8d01020408102040801b366cd8' - 'ab4d9a2f5ebc63c697356ad4b37dfaef' - 'c5913972e4d3bd61c29f254a943366cc' - '831d3a74e8cb8d01020408102040801b' - '366cd8ab4d9a2f5ebc63c697356ad4b3' - '7dfaefc5913972e4d3bd61c29f254a94' - '3366cc831d3a74e8cb8d010204081020' - '40801b366cd8ab4d9a2f5ebc63c69735' - '6ad4b37dfaefc5913972e4d3bd61c29f' - '254a943366cc831d3a74e8cb8d010204' - '08102040801b366cd8ab4d9a2f5ebc63' - 'c697356ad4b37dfaefc5913972e4d3bd' - '61c29f254a943366cc831d3a74e8cb'.decode('hex') +aes_Rcon = array( + 'B', + str2hex('8d01020408102040801b366cd8ab4d9a' + '2f5ebc63c697356ad4b37dfaefc59139' + '72e4d3bd61c29f254a943366cc831d3a' + '74e8cb8d01020408102040801b366cd8' + 'ab4d9a2f5ebc63c697356ad4b37dfaef' + 'c5913972e4d3bd61c29f254a943366cc' + '831d3a74e8cb8d01020408102040801b' + '366cd8ab4d9a2f5ebc63c697356ad4b3' + '7dfaefc5913972e4d3bd61c29f254a94' + '3366cc831d3a74e8cb8d010204081020' + '40801b366cd8ab4d9a2f5ebc63c69735' + '6ad4b37dfaefc5913972e4d3bd61c29f' + '254a943366cc831d3a74e8cb8d010204' + '08102040801b366cd8ab4d9a2f5ebc63' + 'c697356ad4b37dfaefc5913972e4d3bd' + '61c29f254a943366cc831d3a74e8cb') ) From 96ee37727968bcae2f2bd1a1f86add038ca5175e Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 10:11:33 -0500 Subject: [PATCH 08/27] utils compiles with python 3 --- VERSION | 2 +- gluon/utils.py | 36 +++++++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/VERSION b/VERSION index 2fc3e7fb..3c945c5e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 09:54:01) stable +Version 2.2.1 (2012-10-23 10:11:28) stable diff --git a/gluon/utils.py b/gluon/utils.py index 3ef03909..df629463 100644 --- a/gluon/utils.py +++ b/gluon/utils.py @@ -21,20 +21,32 @@ import os import re import logging import socket -import cPickle import base64 import zlib +try: + import cPickle as pickle # python 2 +except ImportError: + import pickle # python 3 + + try: from Crypto.Cipher import AES except ImportError: - from contrib import aes as AES + try: + from .aes import AES + except ImportError: + from contrib.aes import AES try: from contrib.pbkdf2 import pbkdf2_hex HAVE_PBKDF2 = True except ImportError: - HAVE_PBKDF2 = False + try: + from .pbkdf2 import pbkdf2_hex + HAVE_PBKDF2 = True + except (ImportError, ValueError): + HAVE_PBKDF2 = False logger = logging.getLogger("web2py") @@ -115,7 +127,7 @@ def pad(s, n=32, padchar='.'): def secure_dumps(data, encryption_key, hash_key=None, compression_level=None): if not hash_key: hash_key = hashlib.sha1(encryption_key).hexdigest() - dump = cPickle.dumps(data) + dump = pickle.dumps(data) if compression_level: dump = zlib.compress(dump, compression_level) key = pad(encryption_key[:32]) @@ -141,8 +153,8 @@ def secure_loads(data, encryption_key, hash_key=None, compression_level=None): data = data.rstrip(' ') if compression_level: data = zlib.decompress(data) - return cPickle.loads(data) - except (TypeError, cPickle.UnpicklingError): + return pickle.loads(data) + except (TypeError, pickle.UnpicklingError): return None ### compute constant CTOKENS @@ -173,7 +185,10 @@ def initialize_urandom(): # try to add process-specific entropy frandom = open('/dev/urandom', 'wb') try: - frandom.write(''.join(chr(t) for t in ctokens)) + try: + frandom.write(''.join(chr(t) for t in ctokens)) # python 2 + except: + frandom.write(bytes([]).join(bytes([t]) for t in ctokens)) # python 3 finally: frandom.close() except IOError: @@ -185,8 +200,11 @@ def initialize_urandom(): """Cryptographically secure session management is not possible on your system because your system does not provide a cryptographically secure entropy source. This is not specific to web2py; consider deploying on a different operating system.""") - unpacked_ctokens = struct.unpack('=QQ', string.join( - (chr(x) for x in ctokens), '')) + try: + packed = ''.join(chr(x) for x in ctokens) # python 2 + except: + packed = bytes([]).join(bytes([x]) for x in ctokens) # python 3 + unpacked_ctokens = struct.unpack('=QQ', packed) return unpacked_ctokens, have_urandom UNPACKED_CTOKENS, HAVE_URANDOM = initialize_urandom() From a4cef60c49f510dbf6938a9275bfe8341237e438 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 10:13:17 -0500 Subject: [PATCH 09/27] reverted changs to utils --- VERSION | 2 +- gluon/contrib/aes.py | 115 +++++++++++++++++++------------------------ gluon/utils.py | 36 ++++---------- 3 files changed, 62 insertions(+), 91 deletions(-) diff --git a/VERSION b/VERSION index 3c945c5e..767f5e08 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 10:11:28) stable +Version 2.2.1 (2012-10-23 10:13:13) stable diff --git a/gluon/contrib/aes.py b/gluon/contrib/aes.py index 9ca2a3b3..cecf2d90 100644 --- a/gluon/contrib/aes.py +++ b/gluon/contrib/aes.py @@ -45,16 +45,8 @@ there clears it up. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -# -# Ported to python 3 by Massimo Di Pierro -# #### -try: - import binascii # python 3 - str2hex = lambda s: binascii.b2a_hex(s.encode('utf8')) -except ImportError: # python 2 - str2hex = lambda s: s.decode('hex') from array import array @@ -444,70 +436,67 @@ gf_mul_by_14 = array('B', [galois_multiply(x, 14) for x in range(256)]) # # More information: http://en.wikipedia.org/wiki/Rijndael_S-box -aes_sbox = array( - 'B', - str2hex('637c777bf26b6fc53001672bfed7ab76' - 'ca82c97dfa5947f0add4a2af9ca472c0' - 'b7fd9326363ff7cc34a5e5f171d83115' - '04c723c31896059a071280e2eb27b275' - '09832c1a1b6e5aa0523bd6b329e32f84' - '53d100ed20fcb15b6acbbe394a4c58cf' - 'd0efaafb434d338545f9027f503c9fa8' - '51a3408f929d38f5bcb6da2110fff3d2' - 'cd0c13ec5f974417c4a77e3d645d1973' - '60814fdc222a908846eeb814de5e0bdb' - 'e0323a0a4906245cc2d3ac629195e479' - 'e7c8376d8dd54ea96c56f4ea657aae08' - 'ba78252e1ca6b4c6e8dd741f4bbd8b8a' - '703eb5664803f60e613557b986c11d9e' - 'e1f8981169d98e949b1e87e9ce5528df' - '8ca1890dbfe6426841992d0fb054bb16') +aes_sbox = array('B', + '637c777bf26b6fc53001672bfed7ab76' + 'ca82c97dfa5947f0add4a2af9ca472c0' + 'b7fd9326363ff7cc34a5e5f171d83115' + '04c723c31896059a071280e2eb27b275' + '09832c1a1b6e5aa0523bd6b329e32f84' + '53d100ed20fcb15b6acbbe394a4c58cf' + 'd0efaafb434d338545f9027f503c9fa8' + '51a3408f929d38f5bcb6da2110fff3d2' + 'cd0c13ec5f974417c4a77e3d645d1973' + '60814fdc222a908846eeb814de5e0bdb' + 'e0323a0a4906245cc2d3ac629195e479' + 'e7c8376d8dd54ea96c56f4ea657aae08' + 'ba78252e1ca6b4c6e8dd741f4bbd8b8a' + '703eb5664803f60e613557b986c11d9e' + 'e1f8981169d98e949b1e87e9ce5528df' + '8ca1890dbfe6426841992d0fb054bb16'.decode('hex') ) # This is the inverse of the above. In other words: # aes_inv_sbox[aes_sbox[val]] == val -aes_inv_sbox = array( - 'B', - str2hex('52096ad53036a538bf40a39e81f3d7fb' - '7ce339829b2fff87348e4344c4dee9cb' - '547b9432a6c2233dee4c950b42fac34e' - '082ea16628d924b2765ba2496d8bd125' - '72f8f66486689816d4a45ccc5d65b692' - '6c704850fdedb9da5e154657a78d9d84' - '90d8ab008cbcd30af7e45805b8b34506' - 'd02c1e8fca3f0f02c1afbd0301138a6b' - '3a9111414f67dcea97f2cfcef0b4e673' - '96ac7422e7ad3585e2f937e81c75df6e' - '47f11a711d29c5896fb7620eaa18be1b' - 'fc563e4bc6d279209adbc0fe78cd5af4' - '1fdda8338807c731b11210592780ec5f' - '60517fa919b54a0d2de57a9f93c99cef' - 'a0e03b4dae2af5b0c8ebbb3c83539961' - '172b047eba77d626e169146355210c7d') - ) +aes_inv_sbox = array('B', + '52096ad53036a538bf40a39e81f3d7fb' + '7ce339829b2fff87348e4344c4dee9cb' + '547b9432a6c2233dee4c950b42fac34e' + '082ea16628d924b2765ba2496d8bd125' + '72f8f66486689816d4a45ccc5d65b692' + '6c704850fdedb9da5e154657a78d9d84' + '90d8ab008cbcd30af7e45805b8b34506' + 'd02c1e8fca3f0f02c1afbd0301138a6b' + '3a9111414f67dcea97f2cfcef0b4e673' + '96ac7422e7ad3585e2f937e81c75df6e' + '47f11a711d29c5896fb7620eaa18be1b' + 'fc563e4bc6d279209adbc0fe78cd5af4' + '1fdda8338807c731b11210592780ec5f' + '60517fa919b54a0d2de57a9f93c99cef' + 'a0e03b4dae2af5b0c8ebbb3c83539961' + '172b047eba77d626e169146355210c7d'.decode('hex') +) # The Rcon table is used in AES's key schedule (key expansion) # It's a pre-computed table of exponentation of 2 in AES's finite field # # More information: http://en.wikipedia.org/wiki/Rijndael_key_schedule -aes_Rcon = array( - 'B', - str2hex('8d01020408102040801b366cd8ab4d9a' - '2f5ebc63c697356ad4b37dfaefc59139' - '72e4d3bd61c29f254a943366cc831d3a' - '74e8cb8d01020408102040801b366cd8' - 'ab4d9a2f5ebc63c697356ad4b37dfaef' - 'c5913972e4d3bd61c29f254a943366cc' - '831d3a74e8cb8d01020408102040801b' - '366cd8ab4d9a2f5ebc63c697356ad4b3' - '7dfaefc5913972e4d3bd61c29f254a94' - '3366cc831d3a74e8cb8d010204081020' - '40801b366cd8ab4d9a2f5ebc63c69735' - '6ad4b37dfaefc5913972e4d3bd61c29f' - '254a943366cc831d3a74e8cb8d010204' - '08102040801b366cd8ab4d9a2f5ebc63' - 'c697356ad4b37dfaefc5913972e4d3bd' - '61c29f254a943366cc831d3a74e8cb') +aes_Rcon = array('B', + '8d01020408102040801b366cd8ab4d9a' + '2f5ebc63c697356ad4b37dfaefc59139' + '72e4d3bd61c29f254a943366cc831d3a' + '74e8cb8d01020408102040801b366cd8' + 'ab4d9a2f5ebc63c697356ad4b37dfaef' + 'c5913972e4d3bd61c29f254a943366cc' + '831d3a74e8cb8d01020408102040801b' + '366cd8ab4d9a2f5ebc63c697356ad4b3' + '7dfaefc5913972e4d3bd61c29f254a94' + '3366cc831d3a74e8cb8d010204081020' + '40801b366cd8ab4d9a2f5ebc63c69735' + '6ad4b37dfaefc5913972e4d3bd61c29f' + '254a943366cc831d3a74e8cb8d010204' + '08102040801b366cd8ab4d9a2f5ebc63' + 'c697356ad4b37dfaefc5913972e4d3bd' + '61c29f254a943366cc831d3a74e8cb'.decode('hex') ) diff --git a/gluon/utils.py b/gluon/utils.py index df629463..3ef03909 100644 --- a/gluon/utils.py +++ b/gluon/utils.py @@ -21,32 +21,20 @@ import os import re import logging import socket +import cPickle import base64 import zlib -try: - import cPickle as pickle # python 2 -except ImportError: - import pickle # python 3 - - try: from Crypto.Cipher import AES except ImportError: - try: - from .aes import AES - except ImportError: - from contrib.aes import AES + from contrib import aes as AES try: from contrib.pbkdf2 import pbkdf2_hex HAVE_PBKDF2 = True except ImportError: - try: - from .pbkdf2 import pbkdf2_hex - HAVE_PBKDF2 = True - except (ImportError, ValueError): - HAVE_PBKDF2 = False + HAVE_PBKDF2 = False logger = logging.getLogger("web2py") @@ -127,7 +115,7 @@ def pad(s, n=32, padchar='.'): def secure_dumps(data, encryption_key, hash_key=None, compression_level=None): if not hash_key: hash_key = hashlib.sha1(encryption_key).hexdigest() - dump = pickle.dumps(data) + dump = cPickle.dumps(data) if compression_level: dump = zlib.compress(dump, compression_level) key = pad(encryption_key[:32]) @@ -153,8 +141,8 @@ def secure_loads(data, encryption_key, hash_key=None, compression_level=None): data = data.rstrip(' ') if compression_level: data = zlib.decompress(data) - return pickle.loads(data) - except (TypeError, pickle.UnpicklingError): + return cPickle.loads(data) + except (TypeError, cPickle.UnpicklingError): return None ### compute constant CTOKENS @@ -185,10 +173,7 @@ def initialize_urandom(): # try to add process-specific entropy frandom = open('/dev/urandom', 'wb') try: - try: - frandom.write(''.join(chr(t) for t in ctokens)) # python 2 - except: - frandom.write(bytes([]).join(bytes([t]) for t in ctokens)) # python 3 + frandom.write(''.join(chr(t) for t in ctokens)) finally: frandom.close() except IOError: @@ -200,11 +185,8 @@ def initialize_urandom(): """Cryptographically secure session management is not possible on your system because your system does not provide a cryptographically secure entropy source. This is not specific to web2py; consider deploying on a different operating system.""") - try: - packed = ''.join(chr(x) for x in ctokens) # python 2 - except: - packed = bytes([]).join(bytes([x]) for x in ctokens) # python 3 - unpacked_ctokens = struct.unpack('=QQ', packed) + unpacked_ctokens = struct.unpack('=QQ', string.join( + (chr(x) for x in ctokens), '')) return unpacked_ctokens, have_urandom UNPACKED_CTOKENS, HAVE_URANDOM = initialize_urandom() From 2f1f2dccc11be093c2941cc6d87db760105f834e Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 10:17:23 -0500 Subject: [PATCH 10/27] utils compiles with python 3 --- VERSION | 2 +- gluon/utils.py | 39 ++++++++++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/VERSION b/VERSION index 767f5e08..4cb4771e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 10:13:13) stable +Version 2.2.1 (2012-10-23 10:17:19) stable diff --git a/gluon/utils.py b/gluon/utils.py index 3ef03909..10fb2457 100644 --- a/gluon/utils.py +++ b/gluon/utils.py @@ -19,22 +19,37 @@ import random import time import os import re +import sys import logging import socket -import cPickle import base64 import zlib +python_version = sys.version_info[0] + +if python_version == 2: + import cPickle as pickle +else: + import pickle + + try: from Crypto.Cipher import AES except ImportError: - from contrib import aes as AES + try: + from .aes import AES + except ImportError: + from contrib.aes import AES try: from contrib.pbkdf2 import pbkdf2_hex HAVE_PBKDF2 = True except ImportError: - HAVE_PBKDF2 = False + try: + from .pbkdf2 import pbkdf2_hex + HAVE_PBKDF2 = True + except (ImportError, ValueError): + HAVE_PBKDF2 = False logger = logging.getLogger("web2py") @@ -115,7 +130,7 @@ def pad(s, n=32, padchar='.'): def secure_dumps(data, encryption_key, hash_key=None, compression_level=None): if not hash_key: hash_key = hashlib.sha1(encryption_key).hexdigest() - dump = cPickle.dumps(data) + dump = pickle.dumps(data) if compression_level: dump = zlib.compress(dump, compression_level) key = pad(encryption_key[:32]) @@ -141,8 +156,8 @@ def secure_loads(data, encryption_key, hash_key=None, compression_level=None): data = data.rstrip(' ') if compression_level: data = zlib.decompress(data) - return cPickle.loads(data) - except (TypeError, cPickle.UnpicklingError): + return pickle.loads(data) + except (TypeError, pickle.UnpicklingError): return None ### compute constant CTOKENS @@ -173,7 +188,10 @@ def initialize_urandom(): # try to add process-specific entropy frandom = open('/dev/urandom', 'wb') try: - frandom.write(''.join(chr(t) for t in ctokens)) + if python_version == 2: + frandom.write(''.join(chr(t) for t in ctokens)) # python 2 + else: + frandom.write(bytes([]).join(bytes([t]) for t in ctokens)) # python 3 finally: frandom.close() except IOError: @@ -185,8 +203,11 @@ def initialize_urandom(): """Cryptographically secure session management is not possible on your system because your system does not provide a cryptographically secure entropy source. This is not specific to web2py; consider deploying on a different operating system.""") - unpacked_ctokens = struct.unpack('=QQ', string.join( - (chr(x) for x in ctokens), '')) + if python_version == 2: + packed = ''.join(chr(x) for x in ctokens) # python 2 + else: + packed = bytes([]).join(bytes([x]) for x in ctokens) # python 3 + unpacked_ctokens = struct.unpack('=QQ', packed) return unpacked_ctokens, have_urandom UNPACKED_CTOKENS, HAVE_URANDOM = initialize_urandom() From 55a0dbeb86552592ab12f9093e7f08f736521107 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 10:28:46 -0500 Subject: [PATCH 11/27] languages.py closer to python 3.0 support --- VERSION | 2 +- gluon/languages.py | 43 ++++++++++++++++++++++++++----------------- gluon/portalocker.py | 2 +- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index 4cb4771e..ad3573d4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 10:17:19) stable +Version 2.2.1 (2012-10-23 10:28:41) stable diff --git a/gluon/languages.py b/gluon/languages.py index 8ed8b6e1..35474e4c 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -13,16 +13,22 @@ Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) import os import re import pkgutil -from utf8 import Utf8 -from cgi import escape -import portalocker import logging import marshal -import copy_reg +from cgi import escape +from threading import RLock + +try: + import copyreg as copy_reg # python 3 +except ImportError: + import copy_reg # python 2 + +from portalocker import read_locked, LockedFile +from utf8 import Utf8 + from fileutils import listdir import settings from cfs import getcfs -from thread import allocate_lock from html import XML, xmlescape from contrib.markmin.markmin2html import render, markmin_escape from string import maketrans @@ -35,7 +41,7 @@ pjoin = os.path.join pexists = os.path.exists pdirname = os.path.dirname isdir = os.path.isdir -is_gae = settings.global_settings.web2py_runtime_gae +is_gae = False # settings.global_settings.web2py_runtime_gae DEFAULT_LANGUAGE = 'en' DEFAULT_LANGUAGE_NAME = 'English' @@ -137,7 +143,7 @@ def get_from_cache(cache, val, fun): def clear_cache(filename): cache = global_language_cache.setdefault( - filename, ({}, allocate_lock())) + filename, ({}, RLock())) lang_dict, lock = cache lock.acquire() try: @@ -147,11 +153,12 @@ def clear_cache(filename): def read_dict_aux(filename): - lang_text = portalocker.read_locked(filename).replace('\r\n', '\n') + lang_text = read_locked(filename).replace('\r\n', '\n') clear_cache(filename) try: return safe_eval(lang_text) or {} - except Exception, e: + except Exception: + e = sys.exc_info()[1] status = 'Syntax error in %s (%s)' % (filename, e) logging.error(status) return {'__corrupted__': status} @@ -187,7 +194,8 @@ def read_possible_plural_rules(): DEFAULT_CONSTRUCT_PLURAL_FORM) plurals[lang] = (lang, nplurals, get_plural_id, construct_plural_form) - except ImportError, e: + except ImportError: + e = sys.exc_info()[1] logging.warn('Unable to import plural rules: %s' % e) return plurals @@ -267,10 +275,11 @@ def read_possible_languages(langpath): def read_plural_dict_aux(filename): - lang_text = portalocker.read_locked(filename).replace('\r\n', '\n') + lang_text = read_locked(filename).replace('\r\n', '\n') try: return eval(lang_text) or {} - except Exception, e: + except Exception: + e = sys.exc_info()[1] status = 'Syntax error in %s (%s)' % (filename, e) logging.error(status) return {'__corrupted__': status} @@ -285,7 +294,7 @@ def write_plural_dict(filename, contents): if '__corrupted__' in contents: return try: - fp = portalocker.LockedFile(filename, 'w') + fp = LockedFile(filename, 'w') fp.write('#!/usr/bin/env python\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n') # coding: utf8\n{\n') for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())): @@ -305,7 +314,7 @@ def write_dict(filename, contents): if '__corrupted__' in contents: return try: - fp = portalocker.LockedFile(filename, 'w') + fp = LockedFile(filename, 'w') except (IOError, OSError): if not settings.global_settings.web2py_runtime_gae: logging.warning('Unable to write to file %s' % filename) @@ -639,14 +648,14 @@ class translator(object): self.t = read_dict(self.language_file) self.cache = global_language_cache.setdefault( self.language_file, - ({}, allocate_lock())) + ({}, RLock())) set_plural(language) self.accepted_language = language return languages self.accepted_language = language or self.current_languages[0] self.language_file = self.default_language_file self.cache = global_language_cache.setdefault(self.language_file, - ({}, allocate_lock())) + ({}, RLock())) self.t = self.default_t set_plural(self.accepted_language) return languages @@ -890,7 +899,7 @@ def findT(path, language=DEFAULT_LANGUAGE): for filename in \ listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\ + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0): - data = portalocker.read_locked(filename) + data = read_locked(filename) items = regex_translate.findall(data) for item in items: try: diff --git a/gluon/portalocker.py b/gluon/portalocker.py index 37e5ffe3..5090535c 100644 --- a/gluon/portalocker.py +++ b/gluon/portalocker.py @@ -167,5 +167,5 @@ if __name__ == '__main__': f.write('test ok') f.close() f = LockedFile('test.txt', mode='rb') - print f.read() + sys.stdout.write(f.read()+'\n') f.close() From 43d4c2831d7b204944a5e32b7c16f0255b33090c Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 10:45:17 -0500 Subject: [PATCH 12/27] template compiles with python 3.0 --- VERSION | 2 +- gluon/template.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/VERSION b/VERSION index ad3573d4..05d96ec2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 10:28:41) stable +Version 2.2.1 (2012-10-23 10:45:12) stable diff --git a/gluon/template.py b/gluon/template.py index 0c1cb913..50312f97 100644 --- a/gluon/template.py +++ b/gluon/template.py @@ -16,9 +16,13 @@ Contributors: import os import cgi -import cStringIO import logging from re import compile, sub, escape, DOTALL +try: + import cStringIO as StringIO +except: + from io import StringIO + try: # have web2py from restricted import RestrictedError @@ -791,13 +795,13 @@ def get_parsed(text): class DummyResponse(): def __init__(self): - self.body = cStringIO.StringIO() + self.body = StringIO.StringIO() def write(self, data, escape=True): if not escape: self.body.write(str(data)) - elif hasattr(data, 'xml') and callable(data.xml): - self.body.write(data.xml()) + elif hasattr(data, 'as_html') and callable(data.as_html): + self.body.write(data.as_html()) else: # make it a string if not isinstance(data, (str, unicode)): @@ -870,7 +874,7 @@ def render(content="hello world", # save current response class if context and 'response' in context: old_response_body = context['response'].body - context['response'].body = cStringIO.StringIO() + context['response'].body = StringIO.StringIO() else: old_response_body = None context['response'] = Response() @@ -887,7 +891,7 @@ def render(content="hello world", stream = open(filename, 'rb') close_stream = True elif content: - stream = cStringIO.StringIO(content) + stream = StringIO.StringIO(content) # Execute the template. code = str(TemplateParser(stream.read( From 093e8b356ea86dfbc34a4b381abe86029c00c645 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 23 Oct 2012 11:13:29 -0500 Subject: [PATCH 13/27] dal compiles with python 3.3 --- VERSION | 2 +- gluon/dal.py | 537 +++++++++++++++++++++++++++------------------------ 2 files changed, 282 insertions(+), 257 deletions(-) diff --git a/VERSION b/VERSION index 05d96ec2..4acb0bb2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 10:45:12) stable +Version 2.2.1 (2012-10-23 11:13:22) stable diff --git a/gluon/dal.py b/gluon/dal.py index 91f05762..9e8a7b42 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -150,17 +150,14 @@ import sys import locale import os import types -import cPickle import datetime import threading import time -import cStringIO import csv import cgi import copy import socket import logging -import copy_reg import base64 import shutil import marshal @@ -173,6 +170,17 @@ import glob import traceback import platform +python_version = sys.version_info[0] +if python_version == 2: + import cPickle as pickle + import cStringIO as StringIO + import copy_reg as copyreg +else: + import pickle + from io import StringIO as StringIO + import copyreg + long = int + CALLABLETYPES = (types.LambdaType, types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.BuiltinMethodType) @@ -196,7 +204,7 @@ pjoin = os.path.join ################################################################################### try: from utils import web2py_uuid -except ImportError: +except (ImportError, SystemError): import uuid def web2py_uuid(): return str(uuid.uuid4()) @@ -215,7 +223,7 @@ except ImportError: try: import validators have_validators = True -except ImportError: +except (ImportError, SyntaxError): have_validators = False LOGGER = logging.getLogger("web2py.dal") @@ -687,12 +695,12 @@ class BaseAdapter(ConnectionPool): self.driver_name = request_driver self.driver = globals().get(request_driver) else: - raise RuntimeError, "driver %s not available" % request_driver + raise RuntimeError("driver %s not available" % request_driver) elif drivers_available: self.driver_name = drivers_available[0] self.driver = globals().get(self.driver_name) else: - raise RuntimeError, "no driver available %s" % self.drivers + raise RuntimeError("no driver available %s" % self.drivers) def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8', @@ -797,9 +805,9 @@ class BaseAdapter(ConnectionPool): srid = self.srid geotype, parms = field_type[:-1].split('(') if not geotype in types: - raise SyntaxError, \ + raise SyntaxError( 'Field: unknown field type: %s for %s' \ - % (field_type, field_name) + % (field_type, field_name)) ftype = types[geotype] if self.dbengine == 'postgres' and geotype == 'geometry': # parameters: schema, srid, dimension @@ -818,8 +826,8 @@ class BaseAdapter(ConnectionPool): dimension=dimension) postcreation_fields.append(ftype) elif not field_type in types: - raise SyntaxError, 'Field: unknown field type: %s for %s' % \ - (field_type, field_name) + raise SyntaxError('Field: unknown field type: %s for %s' % \ + (field_type, field_name)) else: ftype = types[field_type]\ % dict(length=field.length) @@ -927,7 +935,7 @@ class BaseAdapter(ConnectionPool): table._db.commit() if table._dbt: tfile = self.file_open(table._dbt, 'w') - cPickle.dump(sql_fields, tfile) + pickle.dump(sql_fields, tfile) self.file_close(tfile) if fake_migrate: logfile.write('faked!\n') @@ -936,11 +944,11 @@ class BaseAdapter(ConnectionPool): else: tfile = self.file_open(table._dbt, 'r') try: - sql_fields_old = cPickle.load(tfile) + sql_fields_old = pickle.load(tfile) except EOFError: self.file_close(tfile) self.file_close(logfile) - raise RuntimeError, 'File %s appears corrupted' % table._dbt + raise RuntimeError('File %s appears corrupted' % table._dbt) self.file_close(tfile) if sql_fields != sql_fields_old: self.migrate_table(table, @@ -1059,21 +1067,21 @@ class BaseAdapter(ConnectionPool): if db._adapter.commit_on_alter_table: db.commit() tfile = self.file_open(table._dbt, 'w') - cPickle.dump(sql_fields_current, tfile) + pickle.dump(sql_fields_current, tfile) self.file_close(tfile) logfile.write('success!\n') else: logfile.write('faked!\n') elif metadata_change: tfile = self.file_open(table._dbt, 'w') - cPickle.dump(sql_fields_current, tfile) + pickle.dump(sql_fields_current, tfile) self.file_close(tfile) if metadata_change and \ not (query and self.dbengine in ('mysql','oracle','firebird')): db.commit() tfile = self.file_open(table._dbt, 'w') - cPickle.dump(sql_fields_current, tfile) + pickle.dump(sql_fields_current, tfile) self.file_close(tfile) def LOWER(self, first): @@ -1155,7 +1163,8 @@ class BaseAdapter(ConnectionPool): query = self._insert(table,fields) try: self.execute(query) - except Exception, e: + except Exception: + e = sys.exc_info()[1] if isinstance(e,self.integrity_error_class()): return None raise e @@ -1231,25 +1240,25 @@ class BaseAdapter(ConnectionPool): def LT(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s < None" % first + raise RuntimeError("Cannot compare %s < None" % first) return '(%s < %s)' % (self.expand(first), self.expand(second,first.type)) def LE(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s <= None" % first + raise RuntimeError("Cannot compare %s <= None" % first) return '(%s <= %s)' % (self.expand(first), self.expand(second,first.type)) def GT(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s > None" % first + raise RuntimeError("Cannot compare %s > None" % first) return '(%s > %s)' % (self.expand(first), self.expand(second,first.type)) def GE(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s >= None" % first + raise RuntimeError("Cannot compare %s >= None" % first) return '(%s >= %s)' % (self.expand(first), self.expand(second,first.type)) @@ -1411,9 +1420,9 @@ class BaseAdapter(ConnectionPool): if len(tablenames)==1: return tablenames[0] elif len(tablenames)<1: - raise RuntimeError, "No table selected" + raise RuntimeError("No table selected") else: - raise RuntimeError, "Too many tables selected" + raise RuntimeError("Too many tables selected") def expand_all(self, fields, tablenames): db = self.db @@ -1440,7 +1449,7 @@ class BaseAdapter(ConnectionPool): def _select(self, query, fields, attributes): tables = self.tables for key in set(attributes.keys())-SELECT_ARGS: - raise SyntaxError, 'invalid select attribute: %s' % key + raise SyntaxError('invalid select attribute: %s' % key) args_get = attributes.get tablenames = tables(query) for field in fields: @@ -1456,7 +1465,7 @@ class BaseAdapter(ConnectionPool): query = self.common_filter(query,tablenames) if len(tablenames) < 1: - raise SyntaxError, 'Set: no tables selected' + raise SyntaxError('Set: no tables selected') sql_f = ', '.join(map(self.expand, fields)) self._colnames = [c.strip() for c in sql_f.split(', ')] if query: @@ -1474,7 +1483,7 @@ class BaseAdapter(ConnectionPool): limitby = args_get('limitby', False) for_update = args_get('for_update', False) if self.can_select_for_update is False and for_update is True: - raise SyntaxError, 'invalid select attribute: for_update' + raise SyntaxError('invalid select attribute: for_update') if distinct is True: sql_s += 'DISTINCT' elif distinct: @@ -2354,20 +2363,20 @@ class MySQLAdapter(BaseAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, \ - "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError( + "Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') port = int(m.group('port') or '3306') charset = m.group('charset') or 'utf8' driver_args.update(db=db, @@ -2481,19 +2490,19 @@ class PostgreSQLAdapter(BaseAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, "Invalid URI string in DAL" + raise SyntaxError("Invalid URI string in DAL") user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') port = m.group('port') or '5432' sslmode = m.group('sslmode') if sslmode: @@ -2630,7 +2639,7 @@ class PostgreSQLAdapter(BaseAdapter): elif field_is_type('geography'): value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj) # else: -# raise SyntaxError, 'Invalid field type %s' %fieldtype +# raise SyntaxError('Invalid field type %s' %fieldtype) return value return BaseAdapter.represent(self, obj, fieldtype) @@ -2706,19 +2715,19 @@ class JDBCPostgreSQLAdapter(PostgreSQLAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, "Invalid URI string in DAL" + raise SyntaxError("Invalid URI string in DAL") user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') port = m.group('port') or '5432' msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password) def connector(msg=msg,driver_args=driver_args): @@ -2991,12 +3000,13 @@ class MSSQLAdapter(BaseAdapter): try: m = self.REGEX_DSN.match(ruri) if not m: - raise SyntaxError, \ - 'Parsing uri string(%s) has no result' % self.uri + raise SyntaxError( + 'Parsing uri string(%s) has no result' % self.uri) dsn = m.group('dsn') if not dsn: - raise SyntaxError, 'DSN required' - except SyntaxError, e: + raise SyntaxError('DSN required') + except SyntaxError: + e = sys.exc_info()[1] LOGGER.error('NdGpatch error') raise e # was cnxn = 'DSN=%s' % dsn @@ -3004,20 +3014,20 @@ class MSSQLAdapter(BaseAdapter): else: m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, \ - "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError( + "Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') port = m.group('port') or '1433' # Parse the optional url name-value arg pairs after the '?' # (in the form of arg1=value1&arg2=value2&...) @@ -3095,7 +3105,7 @@ class MSSQLAdapter(BaseAdapter): srid = parms return "geography::STGeomFromText('%s',%s)" %(obj, srid) # else: -# raise SyntaxError, 'Invalid field type %s' %fieldtype +# raise SyntaxError('Invalid field type %s' %fieldtype) return "geometry::STGeomFromText('%s',%s)" %(obj, srid) return BaseAdapter.represent(self, obj, fieldtype) @@ -3188,31 +3198,32 @@ class SybaseAdapter(MSSQLAdapter): try: m = self.REGEX_DSN.match(ruri) if not m: - raise SyntaxError, \ - 'Parsing uri string(%s) has no result' % self.uri + raise SyntaxError( + 'Parsing uri string(%s) has no result' % self.uri) dsn = m.group('dsn') if not dsn: - raise SyntaxError, 'DSN required' - except SyntaxError, e: + raise SyntaxError('DSN required') + except SyntaxError: + e = sys.exc_info()[1] LOGGER.error('NdGpatch error') raise e else: m = self.REGEX_URI.match(uri) if not m: - raise SyntaxError, \ - "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError( + "Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') port = m.group('port') or '1433' dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db) @@ -3314,20 +3325,20 @@ class FireBirdAdapter(BaseAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError("Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') port = int(m.group('port') or 3050) db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') charset = m.group('charset') or 'UTF8' driver_args.update(dsn='%s/%s:%s' % (host,port,db), user = credential_decoder(user), @@ -3373,17 +3384,17 @@ class FireBirdEmbeddedAdapter(FireBirdAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, \ - "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError( + "Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' pathdb = m.group('path') if not pathdb: - raise SyntaxError, 'Path required' + raise SyntaxError('Path required') charset = m.group('charset') if not charset: charset = 'UTF8' @@ -3480,20 +3491,20 @@ class InformixAdapter(BaseAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, \ - "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError( + "Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') user = credential_decoder(user) password = credential_decoder(password) dsn = '%s@%s' % (db,host) @@ -3862,19 +3873,19 @@ class SAPDBAdapter(BaseAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, "Invalid URI string in DAL" + raise SyntaxError("Invalid URI string in DAL") user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') def connector(user=user, password=password, database=db, host=host, driver_args=driver_args): return self.driver.Connection(user, password, database, @@ -3905,20 +3916,20 @@ class CubridAdapter(MySQLAdapter): ruri = uri.split('://',1)[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, \ - "Invalid URI string in DAL: %s" % self.uri + raise SyntaxError( + "Invalid URI string in DAL: %s" % self.uri) user = credential_decoder(m.group('user')) if not user: - raise SyntaxError, 'User required' + raise SyntaxError('User required') password = credential_decoder(m.group('password')) if not password: password = '' host = m.group('host') if not host: - raise SyntaxError, 'Host name required' + raise SyntaxError('Host name required') db = m.group('db') if not db: - raise SyntaxError, 'Database name required' + raise SyntaxError('Database name required') port = int(m.group('port') or '30000') charset = m.group('charset') or 'utf8' user = credential_decoder(user) @@ -3945,7 +3956,7 @@ class DatabaseStoredFile: def __init__(self,db,filename,mode): if db._adapter.dbengine != 'mysql': - raise RuntimeError, "only MySQL can store metadata .table files in database for now" + raise RuntimeError("only MySQL can store metadata .table files in database for now") self.db = db self.filename = filename self.mode = mode @@ -3967,7 +3978,7 @@ class DatabaseStoredFile: finally: datafile.close() elif mode in ('r','rw'): - raise RuntimeError, "File %s does not exist" % filename + raise RuntimeError("File %s does not exist" % filename) def read(self, bytes): data = self.data[self.p:self.p+bytes] @@ -4039,7 +4050,7 @@ class GoogleSQLAdapter(UseDatabaseStoredFile,MySQLAdapter): ruri = uri.split("://")[1] m = self.REGEX_URI.match(ruri) if not m: - raise SyntaxError, "Invalid URI string in SQLDB: %s" % self.uri + raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri) instance = credential_decoder(m.group('instance')) self.dbstring = db = credential_decoder(m.group('db')) driver_args['instance'] = instance @@ -4083,7 +4094,7 @@ class NoSQLAdapter(BaseAdapter): if isinstance(fieldtype, SQLCustomType): return fieldtype.encoder(obj) if isinstance(obj, (Expression, Field)): - raise SyntaxError, "non supported on GAE" + raise SyntaxError("non supported on GAE") if self.dbengine == 'google:datastore': if isinstance(fieldtype, gae.Property): return obj @@ -4182,41 +4193,41 @@ class NoSQLAdapter(BaseAdapter): # these functions should never be called! - def OR(self,first,second): raise SyntaxError, "Not supported" - def AND(self,first,second): raise SyntaxError, "Not supported" - def AS(self,first,second): raise SyntaxError, "Not supported" - def ON(self,first,second): raise SyntaxError, "Not supported" - def STARTSWITH(self,first,second=None): raise SyntaxError, "Not supported" - def ENDSWITH(self,first,second=None): raise SyntaxError, "Not supported" - def ADD(self,first,second): raise SyntaxError, "Not supported" - def SUB(self,first,second): raise SyntaxError, "Not supported" - def MUL(self,first,second): raise SyntaxError, "Not supported" - def DIV(self,first,second): raise SyntaxError, "Not supported" - def LOWER(self,first): raise SyntaxError, "Not supported" - def UPPER(self,first): raise SyntaxError, "Not supported" - def EXTRACT(self,first,what): raise SyntaxError, "Not supported" - def AGGREGATE(self,first,what): raise SyntaxError, "Not supported" - def LEFT_JOIN(self): raise SyntaxError, "Not supported" - def RANDOM(self): raise SyntaxError, "Not supported" - def SUBSTRING(self,field,parameters): raise SyntaxError, "Not supported" - def PRIMARY_KEY(self,key): raise SyntaxError, "Not supported" - def ILIKE(self,first,second): raise SyntaxError, "Not supported" - def drop(self,table,mode): raise SyntaxError, "Not supported" - def alias(self,table,alias): raise SyntaxError, "Not supported" - def migrate_table(self,*a,**b): raise SyntaxError, "Not supported" - def distributed_transaction_begin(self,key): raise SyntaxError, "Not supported" - def prepare(self,key): raise SyntaxError, "Not supported" - def commit_prepared(self,key): raise SyntaxError, "Not supported" - def rollback_prepared(self,key): raise SyntaxError, "Not supported" - def concat_add(self,table): raise SyntaxError, "Not supported" - def constraint_name(self, table, fieldname): raise SyntaxError, "Not supported" + def OR(self,first,second): raise SyntaxError("Not supported") + def AND(self,first,second): raise SyntaxError("Not supported") + def AS(self,first,second): raise SyntaxError("Not supported") + def ON(self,first,second): raise SyntaxError("Not supported") + def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported") + def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported") + def ADD(self,first,second): raise SyntaxError("Not supported") + def SUB(self,first,second): raise SyntaxError("Not supported") + def MUL(self,first,second): raise SyntaxError("Not supported") + def DIV(self,first,second): raise SyntaxError("Not supported") + def LOWER(self,first): raise SyntaxError("Not supported") + def UPPER(self,first): raise SyntaxError("Not supported") + def EXTRACT(self,first,what): raise SyntaxError("Not supported") + def AGGREGATE(self,first,what): raise SyntaxError("Not supported") + def LEFT_JOIN(self): raise SyntaxError("Not supported") + def RANDOM(self): raise SyntaxError("Not supported") + def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported") + def PRIMARY_KEY(self,key): raise SyntaxError("Not supported") + def ILIKE(self,first,second): raise SyntaxError("Not supported") + def drop(self,table,mode): raise SyntaxError("Not supported") + def alias(self,table,alias): raise SyntaxError("Not supported") + def migrate_table(self,*a,**b): raise SyntaxError("Not supported") + def distributed_transaction_begin(self,key): raise SyntaxError("Not supported") + def prepare(self,key): raise SyntaxError("Not supported") + def commit_prepared(self,key): raise SyntaxError("Not supported") + def rollback_prepared(self,key): raise SyntaxError("Not supported") + def concat_add(self,table): raise SyntaxError("Not supported") + def constraint_name(self, table, fieldname): raise SyntaxError("Not supported") def create_sequence_and_triggers(self, query, table, **args): pass - def log_execute(self,*a,**b): raise SyntaxError, "Not supported" - def execute(self,*a,**b): raise SyntaxError, "Not supported" - def represent_exceptions(self, obj, fieldtype): raise SyntaxError, "Not supported" - def lastrowid(self,table): raise SyntaxError, "Not supported" - def integrity_error_class(self): raise SyntaxError, "Not supported" - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError, "Not supported" + def log_execute(self,*a,**b): raise SyntaxError("Not supported") + def execute(self,*a,**b): raise SyntaxError("Not supported") + def represent_exceptions(self, obj, fieldtype): raise SyntaxError("Not supported") + def lastrowid(self,table): raise SyntaxError("Not supported") + def integrity_error_class(self): raise SyntaxError("Not supported") + def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported") class GAEF(object): @@ -4308,7 +4319,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): ftype = self.types[field_type](**attr) elif not field_type in self.types\ or not self.types[field_type]: - raise SyntaxError, 'Field: unknown field type: %s' % field_type + raise SyntaxError('Field: unknown field type: %s' % field_type) else: ftype = self.types[field_type](**attr) myfields[field.name] = ftype @@ -4319,13 +4330,13 @@ class GoogleDatastoreAdapter(NoSQLAdapter): elif isinstance(polymodel,Table): table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields) else: - raise SyntaxError, "polymodel must be None, True, a table or a tablename" + raise SyntaxError("polymodel must be None, True, a table or a tablename") return None def expand(self,expression,field_type=None): if isinstance(expression,Field): if expression.type in ('text','blob'): - raise SyntaxError, 'AppEngine does not index by: %s' % expression.type + raise SyntaxError('AppEngine does not index by: %s' % expression.type) return expression.name elif isinstance(expression, (Expression, Query)): if not expression.second is None: @@ -4398,7 +4409,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): def BELONGS(self,first,second=None): if not isinstance(second,(list, tuple)): - raise SyntaxError, "Not supported" + raise SyntaxError("Not supported") if first.type != 'id': return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)] else: @@ -4407,7 +4418,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): def CONTAINS(self,first,second): if not first.type.startswith('list:'): - raise SyntaxError, "Not supported" + raise SyntaxError("Not supported") return [GAEF(first.name,'=',self.expand(second,first.type[5:]),lambda a,b:b in a)] def NOT(self,first): @@ -4418,10 +4429,10 @@ class GoogleDatastoreAdapter(NoSQLAdapter): self.LE: self.GT, self.GE: self.LT} if not isinstance(first,Query): - raise SyntaxError, "Not suported" + raise SyntaxError("Not suported") nop = nops.get(first.op,None) if not nop: - raise SyntaxError, "Not suported %s" % first.op.__name__ + raise SyntaxError("Not suported %s" % first.op.__name__) first.op = nop return self.expand(first) @@ -4446,7 +4457,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): tablename = fields[0].tablename query = db._adapter.id_query(fields[0].table) else: - raise SyntaxError, "Unable to determine a tablename" + raise SyntaxError("Unable to determine a tablename") if query: if use_common_filters(query): @@ -4464,8 +4475,8 @@ class GoogleDatastoreAdapter(NoSQLAdapter): projection = [] for f in fields: if f.type in ['text', 'blob']: - raise SyntaxError, \ - "text and blob field types not allowed in projection queries" + raise SyntaxError( + "text and blob field types not allowed in projection queries") else: projection.append(f.name) @@ -4486,8 +4497,8 @@ class GoogleDatastoreAdapter(NoSQLAdapter): if args_get('projection') == True and \ filter.name in query_projection and \ filter.op in ['=', '<=', '>=']: - raise SyntaxError, \ - "projection fields cannot have equality filters" + raise SyntaxError( + "projection fields cannot have equality filters") if filter.name=='__key__' and filter.op=='>' and filter.value==0: continue elif filter.name=='__key__' and filter.op=='=': @@ -4515,9 +4526,9 @@ class GoogleDatastoreAdapter(NoSQLAdapter): filter.value) if not isinstance(items,list): if args_get('left', None): - raise SyntaxError, 'Set: no left join in appengine' + raise SyntaxError('Set: no left join in appengine') if args_get('groupby', None): - raise SyntaxError, 'Set: no groupby in appengine' + raise SyntaxError('Set: no groupby in appengine') orderby = args_get('orderby', False) if orderby: ### THIS REALLY NEEDS IMPROVEMENT !!! @@ -4570,7 +4581,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): def count(self,query,distinct=None): if distinct: - raise RuntimeError, "COUNT DISTINCT not supported" + raise RuntimeError("COUNT DISTINCT not supported") (items, tablename, fields) = self.select_raw(query) # self.db['_lastsql'] = self._count(query) try: @@ -4731,9 +4742,9 @@ class CouchDBAdapter(NoSQLAdapter): def _select(self,query,fields,attributes): if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") for key in set(attributes.keys())-SELECT_ARGS: - raise SyntaxError, 'invalid select attribute: %s' % key + raise SyntaxError('invalid select attribute: %s' % key) new_fields=[] for item in fields: if isinstance(item,SQLALL): @@ -4758,7 +4769,7 @@ class CouchDBAdapter(NoSQLAdapter): def select(self,query,fields,attributes): if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") fn, colnames = self._select(query,fields,attributes) tablename = colnames[0].split('.')[0] ctable = self.connection[tablename] @@ -4768,7 +4779,7 @@ class CouchDBAdapter(NoSQLAdapter): def delete(self,tablename,query): if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") if query.first.type=='id' and query.op==self.EQ: id = query.second tablename = query.first.tablename @@ -4789,7 +4800,7 @@ class CouchDBAdapter(NoSQLAdapter): def update(self,tablename,query,fields): if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") if query.first.type=='id' and query.op==self.EQ: id = query.second tablename = query.first.tablename @@ -4816,9 +4827,9 @@ class CouchDBAdapter(NoSQLAdapter): def count(self,query,distinct=None): if distinct: - raise RuntimeError, "COUNT DISTINCT not supported" + raise RuntimeError("COUNT DISTINCT not supported") if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") tablename = self.get_table(query) rows = self.select(query,[self.db[tablename]._id],{}) return len(rows) @@ -4828,7 +4839,7 @@ def cleanup(text): validates that the given text is clean: only contains [0-9a-zA-Z_] """ if not REGEX_ALPHANUMERIC.match(text): - raise SyntaxError, 'invalid table or field name: %s' % text + raise SyntaxError('invalid table or field name: %s' % text) return text class MongoDBAdapter(NoSQLAdapter): @@ -4898,9 +4909,11 @@ class MongoDBAdapter(NoSQLAdapter): def connector(uri=self.uri,m=m): try: return self.driver.Connection(uri)[m.get('database')] - except self.driver.errors.ConnectionFailure, inst: - raise SyntaxError, "The connection to " + uri + " could not be made" - except Exception, inst: + except self.driver.errors.ConnectionFailure: + inst = sys.exc_info()[1] + raise SyntaxError("The connection to " + uri + " could not be made") + except Exception: + inst = sys.exc_info()[1] if inst == "cannot specify database without a username and password": raise SyntaxError("You are probebly running version 1.1 of pymongo which contains a bug which requires authentication. Update your pymongo.") else: @@ -4920,7 +4933,7 @@ class MongoDBAdapter(NoSQLAdapter): d = datetime.date(2000, 1, 1) #this piece of data can be stripped of based on the fieldtype return datetime.datetime.combine(d, value) #mongodb doesn't has a time object and so it must datetime, string or integer elif fieldtype == 'list:string' or fieldtype == 'list:integer' or fieldtype == 'list:reference': - return value #raise SyntaxError, "Not Supported" + return value #raise SyntaxError("Not Supported") return value #Safe determines whether a asynchronious request is done or a synchronious action is done @@ -4935,15 +4948,15 @@ class MongoDBAdapter(NoSQLAdapter): def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None, isCapped=False): if isCapped: - raise RuntimeError, "Not implemented" + raise RuntimeError("Not implemented") else: pass def count(self,query,distinct=None,snapshot=True): if distinct: - raise RuntimeError, "COUNT DISTINCT not supported" + raise RuntimeError("COUNT DISTINCT not supported") if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") tablename = self.get_table(query) return int(self.select(query,[self.db[tablename]._id],{},count=True,snapshot=snapshot)['count']) #Maybe it would be faster if we just implemented the pymongo .count() function which is probably quicker? @@ -4976,13 +4989,13 @@ class MongoDBAdapter(NoSQLAdapter): # databases expression.second = ObjectId(("%X" % expression.second)) except: - raise SyntaxError, 'The second argument must by an integer that can represent an objectid.' + raise SyntaxError('The second argument must by an integer that can represent an objectid.') else: try: #But a direct id is also possible expression.second = ObjectId(expression.second) except: - raise SyntaxError, 'second argument must be of type ObjectId or an objectid representable integer' + raise SyntaxError('second argument must be of type ObjectId or an objectid representable integer') elif expression.second == 0: expression.second = ObjectId('000000000000000000000000') return expression.op(expression.first, expression.second) @@ -5018,7 +5031,7 @@ class MongoDBAdapter(NoSQLAdapter): logging.warn('mongodb does not support for_update') for key in set(attributes.keys())-set(('limitby','orderby','for_update')): if attributes[key]!=None: - raise SyntaxError, 'invalid select attribute: %s' % key + raise SyntaxError('invalid select attribute: %s' % key) new_fields=[] mongosort_list = [] @@ -5057,7 +5070,7 @@ class MongoDBAdapter(NoSQLAdapter): elif len(fields) != 0: tablename = fields[0].tablename else: - raise SyntaxError, "The table name could not be found in the query nor from the select statement." + raise SyntaxError("The table name could not be found in the query nor from the select statement.") mongoqry_dict = self.expand(query) fields = fields or self.db[tablename] for field in fields: @@ -5128,7 +5141,7 @@ class MongoDBAdapter(NoSQLAdapter): #the update function should return a string def oupdate(self,tablename,query,fields): if not isinstance(query,Query): - raise SyntaxError, "Not Supported" + raise SyntaxError("Not Supported") filter = None if query: filter = self.expand(query) @@ -5147,7 +5160,7 @@ class MongoDBAdapter(NoSQLAdapter): safe=self.safe #return amount of adjusted rows or zero, but no exceptions related not finding the result if not isinstance(query,Query): - raise RuntimeError, "Not implemented" + raise RuntimeError("Not implemented") amount = self.count(query,False) modify,filter = self.oupdate(tablename,query,fields) try: @@ -5217,14 +5230,14 @@ class MongoDBAdapter(NoSQLAdapter): def LT(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s < None" % first + raise RuntimeError("Cannot compare %s < None" % first) result = {} result[self.expand(first)] = {'$lt': self.expand(second)} return result def LE(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s <= None" % first + raise RuntimeError("Cannot compare %s <= None" % first) result = {} result[self.expand(first)] = {'$lte': self.expand(second)} return result @@ -5236,38 +5249,38 @@ class MongoDBAdapter(NoSQLAdapter): def GE(self,first,second=None): if second is None: - raise RuntimeError, "Cannot compare %s >= None" % first + raise RuntimeError("Cannot compare %s >= None" % first) result = {} result[self.expand(first)] = {'$gte': self.expand(second)} return result def ADD(self, first, second): - raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry" + raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") return '%s + %s' % (self.expand(first), self.expand(second, first.type)) def SUB(self, first, second): - raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry" + raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") return '(%s - %s)' % (self.expand(first), self.expand(second, first.type)) def MUL(self, first, second): - raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry" + raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") return '(%s * %s)' % (self.expand(first), self.expand(second, first.type)) def DIV(self, first, second): - raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry" + raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") return '(%s / %s)' % (self.expand(first), self.expand(second, first.type)) def MOD(self, first, second): - raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry" + raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type)) def AS(self, first, second): - raise NotImplementedError, "This must yet be replaced with javascript in order to accomplish this. Sorry" + raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") return '%s AS %s' % (self.expand(first), second) #We could implement an option that simulates a full featured SQL database. But I think the option should be set explicit or implemented as another library. def ON(self, first, second): - raise NotImplementedError, "This is not possible in NoSQL, but can be simulated with a wrapper." + raise NotImplementedError("This is not possible in NoSQL, but can be simulated with a wrapper.") return '%s ON %s' % (self.expand(first), self.expand(second)) # @@ -5588,7 +5601,8 @@ class IMAPAdapter(NoSQLAdapter): try: result = self.connection.select(self.connection.mailbox_names[tablename]) last_message = int(result[1][0]) - except (IndexError, ValueError, TypeError, KeyError), e: + except (IndexError, ValueError, TypeError, KeyError): + e = sys.exc_info()[1] LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e)) return last_message @@ -6078,12 +6092,14 @@ class IMAPAdapter(NoSQLAdapter): # the uid format implemented try: pedestal, threshold = self.get_uid_bounds(first.tablename) - except TypeError, e: + except TypeError: + e = sys.exc_info()[1] LOGGER.debug("Error requesting uid bounds: %s", str(e)) return "" try: lower_limit = int(self.expand(second)) + 1 - except (ValueError, TypeError), e: + except (ValueError, TypeError): + e = sys.exc_info()[1] raise Exception("Operation not supported (non integer UID)") result = "UID %s:%s" % (lower_limit, threshold) elif name == "DATE": @@ -6106,7 +6122,8 @@ class IMAPAdapter(NoSQLAdapter): # the uid format implemented try: pedestal, threshold = self.get_uid_bounds(first.tablename) - except TypeError, e: + except TypeError: + e = sys.exc_info()[1] LOGGER.debug("Error requesting uid bounds: %s", str(e)) return "" lower_limit = self.expand(second) @@ -6125,12 +6142,14 @@ class IMAPAdapter(NoSQLAdapter): elif name == "UID": try: pedestal, threshold = self.get_uid_bounds(first.tablename) - except TypeError, e: + except TypeError: + e = sys.exc_info()[1] LOGGER.debug("Error requesting uid bounds: %s", str(e)) return "" try: upper_limit = int(self.expand(second)) - 1 - except (ValueError, TypeError), e: + except (ValueError, TypeError): + e = sys.exc_info()[1] raise Exception("Operation not supported (non integer UID)") result = "UID %s:%s" % (pedestal, upper_limit) elif name == "DATE": @@ -6149,7 +6168,8 @@ class IMAPAdapter(NoSQLAdapter): elif name == "UID": try: pedestal, threshold = self.get_uid_bounds(first.tablename) - except TypeError, e: + except TypeError: + e = sys.exc_info()[1] LOGGER.debug("Error requesting uid bounds: %s", str(e)) return "" upper_limit = int(self.expand(second)) @@ -6482,7 +6502,7 @@ def smart_query(fields,text): for ofield in field: new_fields.append(ofield) else: - raise RuntimeError, "fields must be a list of fields" + raise RuntimeError("fields must be a list of fields") fields = new_fields field_map = {} for field in fields: @@ -6546,13 +6566,13 @@ def smart_query(fields,text): elif item in field_map: field = field_map[item] else: - raise RuntimeError, "Invalid syntax" + raise RuntimeError("Invalid syntax") elif not field is None and op is None: op = item elif not op is None: if item.startswith('#'): if not item[1:] in constants: - raise RuntimeError, "Invalid syntax" + raise RuntimeError("Invalid syntax") value = constants[item[1:]] else: value = item @@ -6569,12 +6589,12 @@ def smart_query(fields,text): elif op == 'like': new_query = field.like(value) elif op == 'startswith': new_query = field.startswith(value) elif op == 'endswith': new_query = field.endswith(value) - else: raise RuntimeError, "Invalid operation" + else: raise RuntimeError("Invalid operation") elif field._db._adapter.dbengine=='google:datastore' and \ field.type in ('list:integer', 'list:string', 'list:reference'): if op == 'contains': new_query = field.contains(value) - else: raise RuntimeError, "Invalid operation" - else: raise RuntimeError, "Invalid operation" + else: raise RuntimeError("Invalid operation") + else: raise RuntimeError("Invalid operation") if neg: new_query = ~new_query if query is None: query = new_query @@ -6642,8 +6662,8 @@ class DAL(object): instances = enumerate(instances) for (i, db) in instances: if not db._adapter.support_distributed_transaction(): - raise SyntaxError, \ - 'distributed transaction not suported by %s' % db._dbname + raise SyntaxError( + 'distributed transaction not suported by %s' % db._dbname) for (i, db) in instances: db._adapter.distributed_transaction_begin(keys[i]) @@ -6656,15 +6676,15 @@ class DAL(object): keys = ['%s.%i' % (thread_key, i) for (i,db) in instances] for (i, db) in instances: if not db._adapter.support_distributed_transaction(): - raise SyntaxError, \ - 'distributed transaction not suported by %s' % db._dbanme + raise SyntaxError( + 'distributed transaction not suported by %s' % db._dbanme) try: for (i, db) in instances: db._adapter.prepare(keys[i]) except: for (i, db) in instances: db._adapter.rollback_prepared(keys[i]) - raise RuntimeError, 'failure to commit distributed transaction' + raise RuntimeError('failure to commit distributed transaction') else: for (i, db) in instances: db._adapter.commit_prepared(keys[i]) @@ -6741,7 +6761,7 @@ class DAL(object): uri = 'jdbc:'+uri self._dbname = REGEX_DBNAME.match(uri).group() if not self._dbname in ADAPTERS: - raise SyntaxError, "Error in URI '%s' or database not supported" % self._dbname + raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname) # notice that driver args or {} else driver_args # defaults to {} global, not correct kwargs = dict(db=self,uri=uri, @@ -6763,7 +6783,7 @@ class DAL(object): break except SyntaxError: raise - except Exception, error: + except Exception: tb = traceback.format_exc() sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb)) if connected: @@ -6771,7 +6791,7 @@ class DAL(object): else: time.sleep(1) if not connected: - raise RuntimeError, "Failure to connect, tried %d times:\n%s" % (attempts, tb) + raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb)) else: self._adapter = BaseAdapter(db=self,pool_size=0, uri='None',folder=folder, @@ -6800,7 +6820,7 @@ class DAL(object): for filename in glob.glob(pattern): tfile = self._adapter.file_open(filename, 'r') try: - sql_fields = cPickle.load(tfile) + sql_fields = pickle.load(tfile) name = filename[len(pattern)-7:-6] mf = [(value['sortable'], Field(key, @@ -6827,7 +6847,8 @@ class DAL(object): """ for backend in self.check_reserved: if name.upper() in self.RSK[backend]: - raise SyntaxError, 'invalid table/column name "%s" is a "%s" reserved SQL keyword' % (name, backend.upper()) + raise SyntaxError( + 'invalid table/column name "%s" is a "%s" reserved SQL keyword' % (name, backend.upper())) def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True): """ @@ -6982,16 +7003,16 @@ def index(): elif tokens[2]=='contains': query = db[table][field].contains(args[i]) else: - raise RuntimeError, "invalid pattern: %s" % pattern + raise RuntimeError("invalid pattern: %s" % pattern) if len(tokens)==4 and tokens[3]=='not': query = ~query elif len(tokens)>=4: - raise RuntimeError, "invalid pattern: %s" % pattern + raise RuntimeError("invalid pattern: %s" % pattern) if not otable and isinstance(queries,dict): dbset = db(queries[table]) dbset=dbset(query) else: - raise RuntimeError, "missing relation in pattern: %s" % pattern + raise RuntimeError("missing relation in pattern: %s" % pattern) elif re2.match(tag) and args[i]==tag[:tag.find('[')]: ref = tag[tag.find('[')+1:-1] if '.' in ref and otable: @@ -7065,20 +7086,20 @@ def index(): **args ): if not isinstance(tablename,str): - raise SyntaxError, "missing table name" + raise SyntaxError("missing table name") elif hasattr(self,tablename) or tablename in self.tables: if not args.get('redefine',False): - raise SyntaxError, 'table already defined: %s' % tablename + raise SyntaxError('table already defined: %s' % tablename) elif tablename.startswith('_') or hasattr(self,tablename) or \ REGEX_PYTHON_KEYWORDS.match(tablename): - raise SyntaxError, 'invalid table name: %s' % tablename + raise SyntaxError('invalid table name: %s' % tablename) elif self.check_reserved: self.check_reserved_keyword(tablename) else: invalid_args = set(args)-TABLE_ARGS if invalid_args: - raise SyntaxError, 'invalid table "%s" attributes: %s' \ - % (tablename,invalid_args) + raise SyntaxError('invalid table "%s" attributes: %s' \ + % (tablename,invalid_args)) if self._lazy_tables and not tablename in self._LAZY_TABLES: self._LAZY_TABLES[tablename] = (tablename,fields,args) table = None @@ -7155,8 +7176,8 @@ def index(): def __setattr__(self, key, value): if key[:1]!='_' and key in self: - raise SyntaxError, \ - 'Object %s exists and cannot be redefined' % key + raise SyntaxError( + 'Object %s exists and cannot be redefined' % key) osetattr(self,key,value) __delitem__ = object.__delattr__ @@ -7247,7 +7268,7 @@ def index(): adapter.execute(query) if as_dict: if not hasattr(adapter.cursor,'description'): - raise RuntimeError, "database does not support executesql(...,as_dict=True)" + raise RuntimeError("database does not support executesql(...,as_dict=True)") # Non-DAL legacy db query, converts cursor results to dict. # sequence of 7-item sequences. each sequence tells about a column. # first item is always the field name according to Python Database API specs @@ -7312,7 +7333,7 @@ def index(): elif line == 'END': return elif not line.startswith('TABLE ') or not line[6:] in self.tables: - raise SyntaxError, 'invalid file format' + raise SyntaxError('invalid file format') else: tablename = line[6:] self[tablename].import_from_csv_file( @@ -7324,7 +7345,7 @@ def DAL_unpickler(db_uid): def DAL_pickler(db): return DAL_unpickler, (db._db_uid,) -copy_reg.pickle(DAL, DAL_pickler, DAL_unpickler) +copyreg.pickle(DAL, DAL_pickler, DAL_unpickler) class SQLALL(object): """ @@ -7347,7 +7368,8 @@ class Reference(int): if not self._record: self._record = self._table[int(self)] if not self._record: - raise RuntimeError, "Using a recursive select but encountered a broken reference: %s %d"%(self._table, int(self)) + raise RuntimeError( + "Using a recursive select but encountered a broken reference: %s %d"%(self._table, int(self))) def __getattr__(self, key): if key == 'id': @@ -7386,7 +7408,7 @@ def Reference_pickler(data): marshal_dump = 'i%s' % struct.pack(' Date: Tue, 23 Oct 2012 11:30:02 -0500 Subject: [PATCH 14/27] dal almost runs with python 3.3 but fails a test on 2.5 --- VERSION | 2 +- gluon/dal.py | 29 ++++++++++++++++++----------- gluon/tests/test_dal.py | 13 ++++++++----- gluon/utils.py | 2 +- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 4acb0bb2..545705e2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 11:13:22) stable +Version 2.2.1 (2012-10-23 11:29:56) stable diff --git a/gluon/dal.py b/gluon/dal.py index 9e8a7b42..fafe0727 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -170,16 +170,20 @@ import glob import traceback import platform -python_version = sys.version_info[0] -if python_version == 2: +PYTHON_VERSION = sys.version_info[0] +if PYTHON_VERSION == 2: import cPickle as pickle import cStringIO as StringIO import copy_reg as copyreg + hashlib_md5 = hashlib.md5 + bytes, unicode = str, unicode else: import pickle from io import StringIO as StringIO import copyreg long = int + hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8')) + bytes, unicode = bytes, str CALLABLETYPES = (types.LambdaType, types.FunctionType, types.BuiltinFunctionType, @@ -1592,7 +1596,7 @@ class BaseAdapter(ConnectionPool): else: (cache_model, time_expire) = cache key = self.uri + '/' + sql + '/rows' - if len(key)>200: key = hashlib.md5(key).hexdigest() + if len(key)>200: key = hashlib_md5(key).hexdigest() def _select_aux2(): self.execute(sql) return self._fetchall() @@ -1615,7 +1619,7 @@ class BaseAdapter(ConnectionPool): del attributes['cache'] (cache_model, time_expire) = cache key = self.uri + '/' + sql - if len(key)>200: key = hashlib.md5(key).hexdigest() + if len(key)>200: key = hashlib_md5(key).hexdigest() args = (sql,fields,attributes) return cache_model( key, @@ -1769,8 +1773,8 @@ class BaseAdapter(ConnectionPool): obj = obj.isoformat()[:10] else: obj = str(obj) - if not isinstance(obj,str): - obj = str(obj) + if not isinstance(obj,bytes): + obj = bytes(obj) try: obj.decode(self.db_codec) except: @@ -2097,8 +2101,11 @@ class SQLiteAdapter(BaseAdapter): else: dbpath = uri.split('://',1)[1] if dbpath[0] != '/': - dbpath = pjoin( - self.folder.decode(path_encoding).encode('utf8'), dbpath) + if PYTHON_VERSION == 2: + dbpath = pjoin( + self.folder.decode(path_encoding).encode('utf8'), dbpath) + else: + dbpath = pjoin(self.folder, dbpath) if not 'check_same_thread' in driver_args: driver_args['check_same_thread'] = False if not 'detect_types' in driver_args: @@ -6633,7 +6640,7 @@ class DAL(object): db = super(DAL, cls).__new__(cls) THREAD_LOCAL.db_instances_zombie[db_uid] = db else: - db_uid = kwargs.get('db_uid',hashlib.md5(repr(uri)).hexdigest()) + db_uid = kwargs.get('db_uid',hashlib_md5(repr(uri)).hexdigest()) if db_uid in THREAD_LOCAL.db_instances_zombie: db = THREAD_LOCAL.db_instances_zombie[db_uid] del THREAD_LOCAL.db_instances_zombie[db_uid] @@ -6798,7 +6805,7 @@ class DAL(object): db_codec=db_codec) migrate = fake_migrate = False adapter = self._adapter - self._uri_hash = hashlib.md5(adapter.uri).hexdigest() + self._uri_hash = hashlib_md5(adapter.uri).hexdigest() self._tables = SQLCallableList() self.check_reserved = check_reserved if self.check_reserved: @@ -8792,7 +8799,7 @@ class Set(object): cache_model, time_expire = cache sql = self._count(distinct=distinct) key = db._uri + '/' + sql - if len(key)>200: key = hashlib.md5(key).hexdigest() + if len(key)>200: key = hashlib_md5(key).hexdigest() return cache_model( key, (lambda self=self,distinct=distinct: \ diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index b639c5e7..d92ac0e6 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -13,7 +13,10 @@ else: import unittest import datetime -import cStringIO +try: + import cStringIO as StringIO +except: + from io import StringIO from dal import DAL, Field, Table, SQLALL ALLOWED_DATATYPES = [ @@ -555,11 +558,11 @@ class TestImportExportFields(unittest.TestCase): id = db.person.insert(name=str(k)) db.pet.insert(friend=id,name=str(k)) db.commit() - stream = cStringIO.StringIO() + stream = StringIO.StringIO() db.export_to_csv_file(stream) db(db.pet).delete() db(db.person).delete() - stream = cStringIO.StringIO(stream.getvalue()) + stream = StringIO.StringIO(stream.getvalue()) db.import_from_csv_file(stream) assert db(db.person.id==db.pet.friend)(db.person.name==db.pet.name).count()==10 db.pet.drop() @@ -579,9 +582,9 @@ class TestImportExportUuidFields(unittest.TestCase): id = db.person.insert(name=str(k),uuid=str(k)) db.pet.insert(friend=id,name=str(k)) db.commit() - stream = cStringIO.StringIO() + stream = StringIO.StringIO() db.export_to_csv_file(stream) - stream = cStringIO.StringIO(stream.getvalue()) + stream = StringIO.StringIO(stream.getvalue()) db.import_from_csv_file(stream) assert db(db.person).count()==10 assert db(db.person.id==db.pet.friend)(db.person.name==db.pet.name).count()==20 diff --git a/gluon/utils.py b/gluon/utils.py index 10fb2457..04bcb649 100644 --- a/gluon/utils.py +++ b/gluon/utils.py @@ -38,7 +38,7 @@ try: except ImportError: try: from .aes import AES - except ImportError: + except (ImportError, ValueError): from contrib.aes import AES try: From b31bfdaaf515858a6d4a0729716ee1a106c9f271 Mon Sep 17 00:00:00 2001 From: Massimo Date: Wed, 24 Oct 2012 16:04:32 -0500 Subject: [PATCH 15/27] fixed missing sys in languages --- VERSION | 2 +- gluon/languages.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 545705e2..289773b4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-23 11:29:56) stable +Version 2.2.1 (2012-10-24 16:04:05) stable diff --git a/gluon/languages.py b/gluon/languages.py index 35474e4c..7a66839a 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -12,6 +12,7 @@ Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) import os import re +import sys import pkgutil import logging import marshal From 0d9e5985e48f92f17c61d346b235f25f7cc07cf0 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 08:17:30 -0500 Subject: [PATCH 16/27] better sessions2trash.py, thanks Jim --- VERSION | 2 +- scripts/sessions2trash.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/VERSION b/VERSION index 289773b4..7ad8bbae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-24 16:04:05) stable +Version 2.2.1 (2012-10-25 08:17:22) stable diff --git a/scripts/sessions2trash.py b/scripts/sessions2trash.py index 11e685fc..3995014e 100755 --- a/scripts/sessions2trash.py +++ b/scripts/sessions2trash.py @@ -23,6 +23,7 @@ Typical usage: """ from __future__ import with_statement +from gluon import current from gluon.storage import Storage from optparse import OptionParser import cPickle @@ -93,10 +94,7 @@ class SessionSetDb(SessionSet): def get(self): """Return list of SessionDb instances for existing sessions.""" sessions = [] - tablename = 'web2py_session' - from gluon import current - (record_id_name, table, record_id, unique_key) = \ - current.response._dbtable_and_field + table = current.response.session_db_table for row in table._db(table.id > 0).select(): sessions.append(SessionDb(row)) return sessions @@ -121,9 +119,7 @@ class SessionDb(object): self.row = row def delete(self): - from gluon import current - (record_id_name, table, record_id, unique_key) = \ - current.response._dbtable_and_field + table = current.response.session_db_table self.row.delete_record() table._db.commit() From c94c192e17def19e7fc2bba0382f15d662a30592 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 08:22:34 -0500 Subject: [PATCH 17/27] wiki edit menu pages conditional to login --- VERSION | 2 +- gluon/tools.py | 47 ++++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/VERSION b/VERSION index 7ad8bbae..0d6e4fbc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 08:17:22) stable +Version 2.2.1 (2012-10-25 08:22:25) stable diff --git a/gluon/tools.py b/gluon/tools.py index 580d4037..e70723dd 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -5035,30 +5035,31 @@ class Wiki(object): if True: submenu = [] menu.append((current.T('[Wiki]'), None, None, submenu)) - if URL() == URL(controller, function): - if not str(request.args(0)).startswith('_'): - slug = request.args(0) or 'index' - mode = 1 - elif request.args(0) == '_edit': - slug = request.args(1) or 'index' - mode = 2 - elif request.args(0) == '_editmedia': - slug = request.args(1) or 'index' - mode = 3 - else: - mode = 0 - if mode in (2, 3): - submenu.append((current.T('View Page'), None, - URL(controller, function, args=slug))) - if mode in (1, 3): - submenu.append((current.T('Edit Page'), None, - URL(controller, function, args=('_edit', slug)))) - if mode in (1, 2): - submenu.append((current.T('Edit Page Media'), None, - URL(controller, function, args=('_editmedia', slug)))) + if self.auth.user: + if URL() == URL(controller, function): + if not str(request.args(0)).startswith('_'): + slug = request.args(0) or 'index' + mode = 1 + elif request.args(0) == '_edit': + slug = request.args(1) or 'index' + mode = 2 + elif request.args(0) == '_editmedia': + slug = request.args(1) or 'index' + mode = 3 + else: + mode = 0 + if mode in (2, 3): + submenu.append((current.T('View Page'), None, + URL(controller, function, args=slug))) + if mode in (1, 3): + submenu.append((current.T('Edit Page'), None, + URL(controller, function, args=('_edit', slug)))) + if mode in (1, 2): + submenu.append((current.T('Edit Page Media'), None, + URL(controller, function, args=('_editmedia', slug)))) - submenu.append((current.T('Create New Page'), None, - URL(controller, function, args=('_create')))) + submenu.append((current.T('Create New Page'), None, + URL(controller, function, args=('_create')))) if self.can_manage(): submenu.append((current.T('Manage Pages'), None, URL(controller, function, args=('_pages')))) From 1f100bbe8844d699fe1ca1fbf2f9ad44d2d06773 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 08:47:36 -0500 Subject: [PATCH 18/27] better pep8 in ldap_auth.py, thanks Gyuris --- VERSION | 2 +- gluon/contrib/login_methods/ldap_auth.py | 45 +++++++++++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/VERSION b/VERSION index 0d6e4fbc..d839a28e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 08:22:25) stable +Version 2.2.1 (2012-10-25 08:47:29) stable diff --git a/gluon/contrib/login_methods/ldap_auth.py b/gluon/contrib/login_methods/ldap_auth.py index a0574318..1d6446a5 100644 --- a/gluon/contrib/login_methods/ldap_auth.py +++ b/gluon/contrib/login_methods/ldap_auth.py @@ -222,7 +222,8 @@ def ldap_auth(server='ldap', port=None, con.set_option(ldap.OPT_PROTOCOL_VERSION, 3) # In cases where ForestDnsZones and DomainDnsZones are found, # result will look like the following: - # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones,DC=domain,DC=com'] + # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones, + # DC=domain,DC=com'] if ldap_binddn: # need to search directory with an admin account 1st con.simple_bind_s(ldap_binddn, ldap_bindpw) @@ -238,8 +239,9 @@ def ldap_auth(server='ldap', port=None, user_mail_attrib]) result = con.search_ext_s( ldap_basedn, ldap.SCOPE_SUBTREE, - "(&(sAMAccountName=%s)(%s))" % (ldap.filter.escape_filter_chars(username_bare), - filterstr), + "(&(sAMAccountName=%s)(%s))" % ( + ldap.filter.escape_filter_chars(username_bare), + filterstr), requested_attrs)[0][1] if not isinstance(result, dict): # result should be a dict in the form @@ -292,8 +294,9 @@ def ldap_auth(server='ldap', port=None, # bind anonymously con.simple_bind_s(dn, pw) # search by e-mail address - filter = '(&(mail=%s)(%s))' % (ldap.filter.escape_filter_chars(username), - filterstr) + filter = '(&(mail=%s)(%s))' % ( + ldap.filter.escape_filter_chars(username), + filterstr) # find the uid attrs = ['uid'] if manage_user: @@ -330,8 +333,10 @@ def ldap_auth(server='ldap', port=None, break except ldap.LDAPError, detail: (exc_type, exc_value) = sys.exc_info()[:2] - logger.warning("ldap_auth: searching %s for %s resulted in %s: %s\n" % - (basedn, filter, exc_type, exc_value)) + logger.warning( + "ldap_auth: searching %s for %s resulted in %s: %s\n" % + (basedn, filter, exc_type, exc_value) + ) if not found: logger.warning('User [%s] not found!' % username) return False @@ -365,8 +370,10 @@ def ldap_auth(server='ldap', port=None, break except ldap.LDAPError, detail: (exc_type, exc_value) = sys.exc_info()[:2] - logger.warning("ldap_auth: searching %s for %s resulted in %s: %s\n" % - (basedn, filter, exc_type, exc_value)) + logger.warning( + "ldap_auth: searching %s for %s resulted in %s: %s\n" % + (basedn, filter, exc_type, exc_value) + ) if not found: logger.warning('User [%s] not found!' % username) return False @@ -502,8 +509,8 @@ def ldap_auth(server='ldap', port=None, 'There is no username or email for %s!' % username) raise db_group_search = db((db.auth_membership.user_id == db_user_id) & - (db.auth_user.id == db.auth_membership.user_id) & - (db.auth_group.id == db.auth_membership.group_id)) + (db.auth_user.id == db.auth_membership.user_id) & + (db.auth_group.id == db.auth_membership.group_id)) db_groups_of_the_user = list() db_group_id = dict() @@ -522,7 +529,8 @@ def ldap_auth(server='ldap', port=None, for group_to_del in db_groups_of_the_user: if ldap_groups_of_the_user.count(group_to_del) == 0: db((db.auth_membership.user_id == db_user_id) & - (db.auth_membership.group_id == db_group_id[group_to_del])).delete() + (db.auth_membership.group_id == \ + db_group_id[group_to_del])).delete() # # Create user membership in groups where user is not in already @@ -531,7 +539,7 @@ def ldap_auth(server='ldap', port=None, if db_groups_of_the_user.count(group_to_add) == 0: if db(db.auth_group.role == group_to_add).count() == 0: gid = db.auth_group.insert(role=group_to_add, - description='Generated from LDAP') + description='Generated from LDAP') else: gid = db(db.auth_group.role == group_to_add).select( db.auth_group.id).first().id @@ -608,7 +616,8 @@ def ldap_auth(server='ldap', port=None, con.set_option(ldap.OPT_PROTOCOL_VERSION, 3) # In cases where ForestDnsZones and DomainDnsZones are found, # result will look like the following: - # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones,DC=domain,DC=com'] + # ['ldap://ForestDnsZones.domain.com/DC=ForestDnsZones, + # DC=domain,DC=com'] if ldap_binddn: # need to search directory with an admin account 1st con.simple_bind_s(ldap_binddn, ldap_bindpw) @@ -620,7 +629,8 @@ def ldap_auth(server='ldap', port=None, # We have to use the full string username = con.search_ext_s(base_dn, ldap.SCOPE_SUBTREE, "(&(sAMAccountName=%s)(%s))" % - (ldap.filter.escape_filter_chars(username_bare), filterstr), ["cn"])[0][0] + (ldap.filter.escape_filter_chars(username_bare), + filterstr), ["cn"])[0][0] else: if ldap_binddn: # need to search directory with an bind_dn account 1st @@ -630,7 +640,9 @@ def ldap_auth(server='ldap', port=None, con.simple_bind_s('', '') # search for groups where user is in - filter = '(&(%s=%s)(%s))' % (ldap.filter.escape_filter_chars(group_member_attrib), + filter = '(&(%s=%s)(%s))' % (ldap.filter.escape_filter_chars( + group_member_attrib + ), ldap.filter.escape_filter_chars(username), group_filterstr) group_search_result = con.search_s(group_dn, @@ -648,3 +660,4 @@ def ldap_auth(server='ldap', port=None, if filterstr[0] == '(' and filterstr[-1] == ')': # rfc4515 syntax filterstr = filterstr[1:-1] # parens added again where used return ldap_auth_aux + From 74024301a57c4126c51f0af77f5742a68c129a0c Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 10:02:40 -0500 Subject: [PATCH 19/27] operators in grid are not translated, thanks Friedrich Weber --- VERSION | 2 +- gluon/sqlhtml.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index d839a28e..a39ba0ac 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 08:47:29) stable +Version 2.2.1 (2012-10-25 10:02:32) stable diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index ef574aee..a2aa12ae 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -1581,7 +1581,7 @@ class SQLFORM(FORM): label = isinstance( field.label, str) and T(field.label) or field.label selectfields.append(OPTION(label, _value=str(field))) - operators = SELECT(*[T(option) for option in options]) + operators = SELECT(*[OPTION(T(option), _value=option) for option in options]) if field.type == 'boolean': value_input = SELECT( OPTION(T("True"), _value="T"), From b6d68f97d63e8e7f24fed6732c59b98d3fb69b1b Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 10:04:04 -0500 Subject: [PATCH 20/27] fixed issue 1114, thanks Friedrich --- VERSION | 2 +- gluon/rocket.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index a39ba0ac..ab68522c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 10:02:32) stable +Version 2.2.1 (2012-10-25 10:03:59) stable diff --git a/gluon/rocket.py b/gluon/rocket.py index c5dfff76..3f88f40b 100644 --- a/gluon/rocket.py +++ b/gluon/rocket.py @@ -1192,7 +1192,7 @@ re_REQUEST_LINE = re.compile(r"""^ (?P[^/]+) # Host )? # (?P(\*|/[^ \?]*)) # Path -(\? (?P[^ ]+))? # Query String +(\? (?P[^ ]*))? # Query String \ # (single space) (?PHTTPS?/1\.[01]) # Protocol $ From 99e239798151cae06e3a3634b48217a0cdcd5b23 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 10:07:39 -0500 Subject: [PATCH 21/27] fixed issue 1113, thanks Nico --- VERSION | 2 +- gluon/contrib/markmin/markmin2html.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ab68522c..e4025c24 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 10:03:59) stable +Version 2.2.1 (2012-10-25 10:07:33) stable diff --git a/gluon/contrib/markmin/markmin2html.py b/gluon/contrib/markmin/markmin2html.py index 94084b0b..252ffc55 100755 --- a/gluon/contrib/markmin/markmin2html.py +++ b/gluon/contrib/markmin/markmin2html.py @@ -1262,6 +1262,10 @@ def render(text, t = t or '' a = escape(a) if a else '' if k: + if '#' in k and not ':' in k.split('#')[0]: + # wikipage, not external url + k=k.replace('#','#'+id_prefix) + if k.startswith('#'): k = '#'+id_prefix+k[1:] k = escape(k) From 35c893d47a4b608f742ad9e1249d4423ad3ef361 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 10:11:51 -0500 Subject: [PATCH 22/27] fixed issue 1109 --- VERSION | 2 +- gluon/globals.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index e4025c24..f48fdbe8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 10:07:33) stable +Version 2.2.1 (2012-10-25 10:11:45) stable diff --git a/gluon/globals.py b/gluon/globals.py index 597eb650..7f588357 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -388,7 +388,10 @@ class Response(Storage): if not items: raise HTTP(404) (t, f) = (items.group('table'), items.group('field')) - field = db[t][f] + try: + field = db[t][f] + except AttributeError: + raise HTTP(404) try: (filename, stream) = field.retrieve(name) except IOError: From 529dda0e6d5010732ca851dbf0fe361dfdd24509 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 10:19:50 -0500 Subject: [PATCH 23/27] fixed issue 1104, thanks Paolo Valleri --- VERSION | 2 +- gluon/html.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index f48fdbe8..390d0f22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 10:11:45) stable +Version 2.2.1 (2012-10-25 10:19:43) stable diff --git a/gluon/html.py b/gluon/html.py index 643d4f1a..648fc405 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -1482,8 +1482,9 @@ class A(DIV): (self['callback'], self['target'] or '', d) self['_href'] = self['_href'] or '#null' elif self['cid']: - self['_onclick'] = 'web2py_component("%s","%s");%sreturn false;' % \ - (self['_href'], self['cid'], d) + pre = self['pre_call'] + ';' if self['pre_call'] else '' + self['_onclick'] = '%sweb2py_component("%s","%s");%sreturn false;' % \ + (pre,self['_href'], self['cid'], d) return DIV.xml(self) From fb22a8843ec44a4fe18ce1d6a71e74c38497064e Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 10:23:50 -0500 Subject: [PATCH 24/27] fixed issue 1096, thanks Howesc --- VERSION | 2 +- appengine_config.py => appengine_config.example.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename appengine_config.py => appengine_config.example.py (100%) diff --git a/VERSION b/VERSION index 390d0f22..5843639e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 10:19:43) stable +Version 2.2.1 (2012-10-25 10:23:42) stable diff --git a/appengine_config.py b/appengine_config.example.py similarity index 100% rename from appengine_config.py rename to appengine_config.example.py From 6af2e859ac47ed1f7d5a8ef0da4998a8db5bf1fc Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 25 Oct 2012 17:23:05 -0500 Subject: [PATCH 25/27] fixed commit mistake, thanks Nico --- VERSION | 2 +- gluon/contrib/markmin/markmin2html.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 5843639e..07725f11 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 10:23:42) stable +Version 2.2.1 (2012-10-25 17:22:57) stable diff --git a/gluon/contrib/markmin/markmin2html.py b/gluon/contrib/markmin/markmin2html.py index 252ffc55..06e3d862 100755 --- a/gluon/contrib/markmin/markmin2html.py +++ b/gluon/contrib/markmin/markmin2html.py @@ -1265,9 +1265,6 @@ def render(text, if '#' in k and not ':' in k.split('#')[0]: # wikipage, not external url k=k.replace('#','#'+id_prefix) - - if k.startswith('#'): - k = '#'+id_prefix+k[1:] k = escape(k) title = ' title="%s"' % a.replace(META, DISABLED_META) if a else '' target = ' target="_blank"' if p == 'popup' else '' From 1621825166e26d4aee64bf77bfc5152012f7f709 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 26 Oct 2012 15:49:47 -0500 Subject: [PATCH 26/27] fixed scheduler bug, thanks Niphlod --- VERSION | 2 +- gluon/scheduler.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/VERSION b/VERSION index 07725f11..27056127 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-25 17:22:57) stable +Version 2.2.1 (2012-10-26 15:49:42) stable diff --git a/gluon/scheduler.py b/gluon/scheduler.py index fb6d5841..2cc7e1c5 100644 --- a/gluon/scheduler.py +++ b/gluon/scheduler.py @@ -63,7 +63,6 @@ import os import time import multiprocessing import sys -import cStringIO import threading import traceback import signal @@ -206,7 +205,6 @@ def executor(queue, task, out): if task.app: os.chdir(os.environ['WEB2PY_PATH']) from gluon.shell import env, parse_path_info - from gluon.dal import BaseAdapter from gluon import current level = logging.getLogger().getEffectiveLevel() logging.getLogger().setLevel(logging.WARN) @@ -788,9 +786,9 @@ class Scheduler(MetaScheduler): #the scheduler): then it wasn't expired, but now it is db(st.status.belongs( (QUEUED, ASSIGNED)))(st.stop_time < now).update(status=EXPIRED) - + all_available = db(st.status.belongs((QUEUED, ASSIGNED)))((st.times_run < st.repeats) | (st.repeats == 0))(st.start_time <= now)((st.stop_time == None) | (st.stop_time > now))(st.next_run_time <= now)(st.enabled == True) - limit = len(all_workers) * (50 / len(wkgroups)) + limit = len(all_workers) * (50 / (len(wkgroups) or 1)) #if there are a moltitude of tasks, let's figure out a maximum of tasks per worker. #this can be adjusted with some added intelligence (like esteeming how many tasks will #a worker complete before the ticker reassign them around, but the gain is quite small @@ -804,11 +802,13 @@ class Scheduler(MetaScheduler): #let's freeze it up db.commit() + x = 0 for group in wkgroups.keys(): tasks = all_available(st.group_name==group).select( limitby=(0, limit), orderby=st.next_run_time) #let's break up the queue evenly among workers for task in tasks: + x += 1 gname = task.group_name ws = wkgroups.get(gname) if ws: @@ -827,10 +827,10 @@ class Scheduler(MetaScheduler): db.commit() #I didn't report tasks but I'm working nonetheless!!!! - if len(tasks) > 0: + if x > 0: self.empty_runs = 0 logger.info('TICKER: workers are %s' % len(all_workers)) - logger.info('TICKER: tasks are %s' % len(tasks)) + logger.info('TICKER: tasks are %s' % x) def sleep(self): time.sleep(self.heartbeat * self.worker_status[1]) From 5a478302f40c026dbd1f588ebd856a07927aca17 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 26 Oct 2012 15:51:17 -0500 Subject: [PATCH 27/27] removed redundant assignment, thanks Niphlod --- VERSION | 2 +- gluon/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 27056127..bd93a497 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.2.1 (2012-10-26 15:49:42) stable +Version 2.2.1 (2012-10-26 15:51:12) stable diff --git a/gluon/main.py b/gluon/main.py index 4cc0f84f..aa64fd22 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -460,7 +460,7 @@ def wsgibase(environ, responder): is_https=env.wsgi_url_scheme in HTTPS_SCHEMES or request.env.http_x_forwarded_proto in HTTPS_SCHEMES or env.https == 'on') - request.uuid = request.compute_uuid() # requires client + request.compute_uuid() # requires client request.url = environ['PATH_INFO'] # ##################################################