From 7259516627cd6c399dae618c25a41f76e988be3e Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 11 Jun 2016 14:54:02 +0200 Subject: [PATCH 1/7] fix tools --- gluon/_compat.py | 7 ++- gluon/contenttype.py | 3 +- gluon/languages.py | 6 +-- gluon/serializers.py | 2 + gluon/sqlhtml.py | 8 +-- gluon/tests/__init__.py | 4 +- gluon/tests/test_tools.py | 14 ++--- gluon/tools.py | 111 ++++++++++++++++++-------------------- 8 files changed, 78 insertions(+), 77 deletions(-) diff --git a/gluon/_compat.py b/gluon/_compat.py index d95b0e82..0baea6bb 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -21,7 +21,10 @@ if PY2: import ConfigParser as configparser from email.MIMEBase import MIMEBase from email.Header import Header - from email import MIMEMultipart, MIMEText, Encoders, Charset + from email import Encoders, Charset + from email.MIMEMultipart import MIMEMultipart + from email.MIMEText import MIMEText + from email.Charset import add_charset, QP as charset_QP from urllib import FancyURLopener, urlencode, urlopen from urllib import quote as urllib_quote, unquote as urllib_unquote from string import maketrans @@ -88,7 +91,7 @@ else: from email.mime.text import MIMEText from email import encoders as Encoders from email.header import Header - from email.charset import Charset + from email.charset import Charset, add_charset, QP as charset_QP from urllib.request import FancyURLopener, urlopen from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode import html diff --git a/gluon/contenttype.py b/gluon/contenttype.py index 209f8e15..7dd28f35 100644 --- a/gluon/contenttype.py +++ b/gluon/contenttype.py @@ -19,6 +19,7 @@ Additions: - .pickle: application/python-pickle - .w2p': application/w2p """ +from gluon._compat import to_native __all__ = ['contenttype'] @@ -842,7 +843,7 @@ def contenttype(filename, default='text/plain'): """ Returns the Content-Type string matching extension of the given filename. """ - + filename=to_native(filename) i = filename.rfind('.') if i >= 0: default = CONTENT_TYPE.get(filename[i:].lower(), default) diff --git a/gluon/languages.py b/gluon/languages.py index 763479ef..2d29a51a 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -18,7 +18,7 @@ import pkgutil import logging from cgi import escape from threading import RLock -from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems +from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, _local_html_escape, to_native from gluon.portalocker import read_locked, LockedFile from gluon.utf8 import Utf8 @@ -426,7 +426,7 @@ class lazyT(object): return len(str(self)) def xml(self): - return str(self) if self.M else escape(str(self)) + return str(self) if self.M else _local_html_escape(str(self), quote=False) def encode(self, *a, **b): return str(self).encode(*a, **b) @@ -821,7 +821,7 @@ class translator(object): self.language_file != self.default_language_file: write_dict(self.language_file, self.t) return regex_backslash.sub( - lambda m: m.group(1).translate(ttab_in), str(mt)) + lambda m: m.group(1).translate(ttab_in), to_native(mt)) def params_substitution(self, message, symbols): """ diff --git a/gluon/serializers.py b/gluon/serializers.py index e578fd82..61fe88bb 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -83,6 +83,8 @@ def custom_json(o): return int(o) elif isinstance(o, decimal.Decimal): return str(o) + elif isinstance(o, (bytes, bytearray)): + return str(o) elif isinstance(o, lazyT): return str(o) elif isinstance(o, XmlComponent): diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 5cdb9147..84e25d3e 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -19,7 +19,7 @@ import urllib import re import os -from gluon._compat import StringIO, unichr, urllib_quote, iteritems +from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT from gluon.http import HTTP, redirect from gluon.html import XmlComponent, truncate_string from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG @@ -1141,7 +1141,7 @@ class SQLFORM(FORM): # try to retrieve the indicated record using its id # otherwise ignore it - if record and isinstance(record, (int, long, str, unicode)): + if record and isinstance(record, (int, long, str, unicodeT)): if not str(record).isdigit(): raise HTTP(404, "Object not found") record = table._db(table._id == record).select().first() @@ -1651,7 +1651,7 @@ class SQLFORM(FORM): original_filename = os.path.split(f)[1] elif hasattr(f, 'file'): (source_file, original_filename) = (f.file, f.filename) - elif isinstance(f, (str, unicode)): + elif isinstance(f, (str, unicodeT)): # do not know why this happens, it should not (source_file, original_filename) = \ (StringIO(f), 'file.txt') @@ -3426,7 +3426,7 @@ class ExportClass(object): """ if value is None: return '' - elif isinstance(value, unicode): + elif isinstance(value, unicodeT): return value.encode('utf8') elif isinstance(value, Reference): return int(value) diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index 59691e4d..f587b65e 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -14,14 +14,14 @@ from .test_contribs import * from .test_routes import * from .test_router import * from .test_validators import * - +from .test_tools import * if sys.version[:3] == '2.7': from .test_compileapp import * from .test_is_url import * from .test_languages import * from .test_serializers import * from .test_utils import * - from .test_tools import * + from .test_appadmin import * from .test_scheduler import * from .test_web import * diff --git a/gluon/tests/test_tools.py b/gluon/tests/test_tools.py index 374810c0..ed1b6483 100644 --- a/gluon/tests/test_tools.py +++ b/gluon/tests/test_tools.py @@ -547,7 +547,7 @@ class TestAuth(unittest.TestCase): def test_basic_blank_forms(self): for f in ['login', 'retrieve_password', 'retrieve_username', 'register']: html_form = getattr(self.auth, f)().xml() - self.assertTrue('name="_formkey"' in html_form) + self.assertTrue(b'name="_formkey"' in html_form) for f in ['logout', 'verify_email', 'reset_password', 'change_password', 'profile', 'groups']: self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f)) @@ -712,7 +712,7 @@ class TestAuth(unittest.TestCase): self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first()) # bypass login_bare() self.auth.settings.bulk_register_enabled = True bulk_register_form = self.auth.bulk_register(max_emails=10).xml() - self.assertTrue('name="_formkey"' in bulk_register_form) + self.assertTrue(b'name="_formkey"' in bulk_register_form) # TODO: def test_manage_tokens(self): # TODO: def test_reset_password(self): @@ -723,12 +723,12 @@ class TestAuth(unittest.TestCase): def test_change_password(self): self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first()) # bypass login_bare() change_password_form = getattr(self.auth, 'change_password')().xml() - self.assertTrue('name="_formkey"' in change_password_form) + self.assertTrue(b'name="_formkey"' in change_password_form) def test_profile(self): self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first()) # bypass login_bare() profile_form = getattr(self.auth, 'profile')().xml() - self.assertTrue('name="_formkey"' in profile_form) + self.assertTrue(b'name="_formkey"' in profile_form) # TODO: def test_run_login_onaccept(self): # TODO: def test_jwt(self): @@ -766,7 +766,7 @@ class TestAuth(unittest.TestCase): # basic impersonate() test that return a read form self.assertEqual(self.auth.impersonate().xml(), - '
') + b'
') # bart impersonate itself self.assertEqual(self.auth.impersonate(bart_id), None) self.assertFalse(self.auth.is_impersonating()) # User shouldn't impersonate itself? @@ -776,7 +776,7 @@ class TestAuth(unittest.TestCase): self.assertTrue(self.auth.is_impersonating()) self.assertEqual(self.auth.user_id, omer_id) # we make it really sure self.assertEqual(impersonate_form.xml(), - '
2
Omer
Simpson
omer@test.com
omer
') + b'
2
Omer
Simpson
omer@test.com
omer
') self.auth.logout_bare() # Failing impersonation # User lacking impersonate membership @@ -800,7 +800,7 @@ class TestAuth(unittest.TestCase): def test_groups(self): self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first()) # bypass login_bare() self.assertEqual(self.auth.groups().xml(), - '

user_1(1)

') + b'

user_1(1)

') def test_not_authorized(self): self.current.request.ajax = 'facke_ajax_request' diff --git a/gluon/tools.py b/gluon/tools.py index 8eb6e74e..8736b814 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -13,7 +13,8 @@ Auth, Mail, PluginManager and various utilities import base64 from functools import reduce from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, configparser, MIMEBase, MIMEMultipart, \ - MIMEText, Encoders, Charset, long, urllib_quote, iteritems + MIMEText, Encoders, Charset, long, urllib_quote, iteritems, to_bytes, to_native, add_charset, \ + charset_QP, basestring, unicodeT, to_unicode import datetime import logging import sys @@ -241,6 +242,7 @@ class Mail(object): if filename is None: raise Exception('Missing attachment name') payload = payload.read() + #FIXME PY3 can be used to_native? filename = filename.encode(encoding) if content_type is None: content_type = contenttype(filename) @@ -248,9 +250,9 @@ class Mail(object): self.my_payload = payload MIMEBase.__init__(self, *content_type.split('/', 1)) self.set_payload(payload) - self['Content-Disposition'] = 'attachment; filename="%s"' % filename + self['Content-Disposition'] = 'attachment; filename="%s"' % to_native(filename, encoding) if content_id is not None: - self['Content-Id'] = '<%s>' % content_id.encode(encoding) + self['Content-Id'] = '<%s>' % to_native(content_id, encoding) Encoders.encode_base64(self) def __init__(self, server=None, sender=None, login=None, tls=True): @@ -405,7 +407,7 @@ class Mail(object): """ # We don't want to use base64 encoding for unicode mail - Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8') + add_charset('utf-8', charset_QP, charset_QP, 'utf-8') def encode_header(key): if [c for c in key if 32 > ord(c) or ord(c) > 127]: @@ -428,12 +430,12 @@ class Mail(object): if not raw and attachments: # Use multipart/mixed if there is attachments - payload_in = MIMEMultipart.MIMEMultipart('mixed') + payload_in = MIMEMultipart('mixed') elif raw: # no encoding configuration for raw messages if not isinstance(message, basestring): message = message.read() - if isinstance(message, unicode): + if isinstance(message, unicodeT): text = message.encode('utf-8') elif not encoding == 'utf-8': text = message.decode(encoding).encode('utf-8') @@ -442,7 +444,7 @@ class Mail(object): # No charset passed to avoid transport encoding # NOTE: some unicode encoded strings will produce # unreadable mail contents. - payload_in = MIMEText.MIMEText(text) + payload_in = MIMEText(text) if to: if not isinstance(to, (list, tuple)): to = [to] @@ -471,14 +473,14 @@ class Mail(object): if text is not None: if not isinstance(text, basestring): text = text.read() - if isinstance(text, unicode): + if isinstance(text, unicodeT): text = text.encode('utf-8') elif not encoding == 'utf-8': text = text.decode(encoding).encode('utf-8') if html is not None: if not isinstance(html, basestring): html = html.read() - if isinstance(html, unicode): + if isinstance(html, unicodeT): html = html.encode('utf-8') elif not encoding == 'utf-8': html = html.decode(encoding).encode('utf-8') @@ -486,15 +488,13 @@ class Mail(object): # Construct mime part only if needed if text is not None and html: # We have text and html we need multipart/alternative - attachment = MIMEMultipart.MIMEMultipart('alternative') - attachment.attach(MIMEText.MIMEText(text, _charset='utf-8')) - attachment.attach( - MIMEText.MIMEText(html, 'html', _charset='utf-8')) + attachment = MIMEMultipart('alternative') + attachment.attach(MIMEText(text, _charset='utf-8')) + attachment.attach(MIMEText(html, 'html', _charset='utf-8')) elif text is not None: - attachment = MIMEText.MIMEText(text, _charset='utf-8') + attachment = MIMEText(text, _charset='utf-8') elif html: - attachment = \ - MIMEText.MIMEText(html, 'html', _charset='utf-8') + attachment = MIMEText(html, 'html', _charset='utf-8') if attachments: # If there is attachments put text and html into @@ -560,12 +560,11 @@ class Mail(object): c.op_sign(plain, sig, mode.DETACH) sig.seek(0, 0) # make it part of the email - payload = \ - MIMEMultipart.MIMEMultipart('signed', - boundary=None, - _subparts=None, - **dict(micalg="pgp-sha1", - protocol="application/pgp-signature")) + payload = MIMEMultipart('signed', + boundary=None, + _subparts=None, + **dict(micalg="pgp-sha1", + protocol="application/pgp-signature")) # insert the origin payload payload.attach(payload_in) # insert the detached signature @@ -605,10 +604,10 @@ class Mail(object): c.op_encrypt(recipients, 1, plain, cipher) cipher.seek(0, 0) # make it a part of the email - payload = MIMEMultipart.MIMEMultipart('encrypted', - boundary=None, - _subparts=None, - **dict(protocol="application/pgp-encrypted")) + payload = MIMEMultipart('encrypted', + boundary=None, + _subparts=None, + **dict(protocol="application/pgp-encrypted")) p = MIMEBase("application", 'pgp-encrypted') p.set_payload("Version: 1\r\n") payload.attach(p) @@ -729,29 +728,29 @@ class Mail(object): payload = payload_in if from_address: - payload['From'] = encoded_or_raw(from_address.decode(encoding)) + payload['From'] = encoded_or_raw(to_unicode(from_address, encoding)) else: - payload['From'] = encoded_or_raw(sender.decode(encoding)) + payload['From'] = encoded_or_raw(to_unicode(sender, encoding)) origTo = to[:] if to: - payload['To'] = encoded_or_raw(', '.join(to).decode(encoding)) + payload['To'] = encoded_or_raw(to_unicode(', '.join(to), encoding)) if reply_to: - payload['Reply-To'] = encoded_or_raw(reply_to.decode(encoding)) + payload['Reply-To'] = encoded_or_raw(to_unicode(reply_to, encoding)) if cc: - payload['Cc'] = encoded_or_raw(', '.join(cc).decode(encoding)) + payload['Cc'] = encoded_or_raw(to_unicode(', '.join(cc), encoding)) to.extend(cc) if bcc: to.extend(bcc) - payload['Subject'] = encoded_or_raw(subject.decode(encoding)) + payload['Subject'] = encoded_or_raw(to_unicode(subject, encoding)) payload['Date'] = email.utils.formatdate() for k, v in iteritems(headers): - payload[k] = encoded_or_raw(v.decode(encoding)) + payload[k] = encoded_or_raw(to_unicode(v, encoding)) result = {} try: if self.settings.server == 'logging': entry = 'email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' % \ ('-' * 40, sender, ', '.join(to), subject, text or html, '-' * 40) - logger.warn(entry) + logger.warning(entry) elif self.settings.server.startswith('logging:'): entry = 'email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' % \ ('-' * 40, sender, ', '.join(to), subject, text or html, '-' * 40) @@ -773,16 +772,16 @@ class Mail(object): if attachments: result = mail.send_mail( sender=sender, to=origTo, - subject=unicode(subject, encoding), body=unicode(text, encoding), html=html, + subject=to_unicode(subject, encoding), body=to_unicode(text, encoding), html=html, attachments=attachments, **xcc) elif html and (not raw): result = mail.send_mail( sender=sender, to=origTo, - subject=unicode(subject, encoding), body=unicode(text, encoding), html=html, **xcc) + subject=to_unicode(subject, encoding), body=to_unicode(text, encoding), html=html, **xcc) else: result = mail.send_mail( sender=sender, to=origTo, - subject=unicode(subject, encoding), body=unicode(text, encoding), **xcc) + subject=to_unicode(subject, encoding), body=to_unicode(text, encoding), **xcc) else: smtp_args = self.settings.server.split(':') kwargs = dict(timeout=self.settings.timeout) @@ -800,7 +799,7 @@ class Mail(object): sender, to, payload.as_string()) server.quit() except Exception as e: - logger.warn('Mail.send failure:%s' % e) + logger.warning('Mail.send failure:%s' % e) self.result = result self.error = e return False @@ -1250,8 +1249,7 @@ class AuthJWT(object): @staticmethod def jwt_b64e(string): - if isinstance(string, unicode): - string = string.encode('utf-8', 'strict') + string = to_bytes(string) return base64.urlsafe_b64encode(string).strip(b'=') @staticmethod @@ -1260,47 +1258,44 @@ class AuthJWT(object): called with a unicode string). The result is also a bytestring. """ - if isinstance(string, unicode): - string = string.encode('ascii', 'ignore') - return base64.urlsafe_b64decode(string + '=' * (-len(string) % 4)) + string = to_bytes(string, 'ascii', 'ignore') + return base64.urlsafe_b64decode(string + b'=' * (-len(string) % 4)) def generate_token(self, payload): - secret = self.secret_key + secret = to_bytes(self.secret_key) if self.salt: if callable(self.salt): secret = "%s$%s" % (secret, self.salt(payload)) else: secret = "%s$%s" % (secret, self.salt) - if isinstance(secret, unicode): + if isinstance(secret, unicodeT): secret = secret.encode('ascii', 'ignore') b64h = self.cached_b64h b64p = self.jwt_b64e(serializers.json(payload)) - jbody = b64h + '.' + b64p + jbody = b64h + b'.' + b64p mauth = hmac.new(key=secret, msg=jbody, digestmod=self.digestmod) jsign = self.jwt_b64e(mauth.digest()) - return jbody + '.' + jsign + return to_native(jbody + b'.' + jsign) def verify_signature(self, body, signature, secret): mauth = hmac.new(key=secret, msg=body, digestmod=self.digestmod) return compare(self.jwt_b64e(mauth.digest()), signature) def load_token(self, token): - if isinstance(token, unicode): - token = token.encode('utf-8', 'strict') - body, sig = token.rsplit('.', 1) - b64h, b64b = body.split('.', 1) + token = to_bytes(token, 'utf-8', 'strict') + body, sig = token.rsplit(b'.', 1) + b64h, b64b = body.split(b'.', 1) if b64h != self.cached_b64h: # header not the same raise HTTP(400, u'Invalid JWT Header') secret = self.secret_key - tokend = serializers.loads_json(self.jwt_b64d(b64b)) + tokend = serializers.loads_json(to_native(self.jwt_b64d(b64b))) if self.salt: if callable(self.salt): secret = "%s$%s" % (secret, self.salt(tokend)) else: secret = "%s$%s" % (secret, self.salt) - if isinstance(secret, unicode): - secret = secret.encode('ascii', 'ignore') + secret = to_bytes(secret, 'ascii', 'ignore') if not self.verify_signature(body, sig, secret): # signature verification failed raise HTTP(400, u'Token signature is invalid') @@ -3560,8 +3555,8 @@ class Auth(object): password = '' specials = r'!#$*' for i in range(0, 3): - password += random.choice(string.lowercase) - password += random.choice(string.uppercase) + password += random.choice(string.ascii_lowercase) + password += random.choice(string.ascii_uppercase) password += random.choice(string.digits) password += random.choice(specials) return ''.join(random.sample(password, len(password))) @@ -3994,7 +3989,7 @@ class Auth(object): requires = table_user[passfield].requires if not isinstance(requires, (list, tuple)): requires = [requires] - requires = filter(lambda t: isinstance(t, CRYPT), requires) + requires = list(filter(lambda t: isinstance(t, CRYPT), requires)) if requires: requires[0].min_length = 0 form = SQLFORM.factory( @@ -5621,7 +5616,7 @@ class Service(object): args = request.args def none_exception(value): - if isinstance(value, unicode): + if isinstance(value, unicodeT): return value.encode('utf8') if hasattr(value, 'isoformat'): return value.isoformat()[:19].replace('T', ' ') From ab2cdd595b47d71bfcd4c9ba4e54aadf42f0d94a Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 11 Jun 2016 15:26:15 +0200 Subject: [PATCH 2/7] fix utils --- gluon/tests/__init__.py | 4 ++-- gluon/tests/test_utils.py | 6 +++--- gluon/utils.py | 16 ++++++++++------ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index f587b65e..46a0f107 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -15,13 +15,13 @@ from .test_routes import * from .test_router import * from .test_validators import * from .test_tools import * +from .test_utils import * + if sys.version[:3] == '2.7': from .test_compileapp import * from .test_is_url import * from .test_languages import * from .test_serializers import * - from .test_utils import * - from .test_appadmin import * from .test_scheduler import * from .test_web import * diff --git a/gluon/tests/test_utils.py b/gluon/tests/test_utils.py index 01507cc5..21db1d88 100644 --- a/gluon/tests/test_utils.py +++ b/gluon/tests/test_utils.py @@ -12,7 +12,7 @@ from gluon.utils import md5_hash, compare, is_valid_ip_address, web2py_uuid import hashlib from hashlib import md5, sha1, sha224, sha256, sha384, sha512 -from gluon.utils import simple_hash, get_digest, secure_dumps, secure_loads +from gluon.utils import simple_hash, get_digest, secure_dumps, secure_loads, basestring class TestUtils(unittest.TestCase): @@ -94,8 +94,8 @@ class TestUtils(unittest.TestCase): secured = secure_dumps(testobj, testkey) original = secure_loads(secured, testkey) self.assertEqual(testobj, original) - self.assertTrue(isinstance(secured, basestring)) - self.assertTrue(':' in secured) + self.assertTrue(isinstance(secured, bytes)) + self.assertTrue(b':' in secured) large_testobj = [x for x in range(1000)] secured_comp = secure_dumps(large_testobj, testkey, compression_level=9) diff --git a/gluon/utils.py b/gluon/utils.py index 6933a72b..540f4e1f 100644 --- a/gluon/utils.py +++ b/gluon/utils.py @@ -85,7 +85,7 @@ def compare(a, b): def md5_hash(text): """ Generates a md5 hash with the given text """ - return md5(text).hexdigest() + return md5(to_bytes(text)).hexdigest() def simple_hash(text, key='', salt='', digest_alg='md5'): @@ -157,11 +157,12 @@ def get_callable_argspec(fn): return inspect.getargspec(inspectable) -def pad(s, n=32, padchar=' '): +def pad(s, n=32, padchar=b' '): return s + (32 - len(s) % 32) * padchar def secure_dumps(data, encryption_key, hash_key=None, compression_level=None): + encryption_key = to_bytes(encryption_key) if not hash_key: hash_key = sha1(encryption_key).hexdigest() dump = pickle.dumps(data, pickle.HIGHEST_PROTOCOL) @@ -170,17 +171,20 @@ def secure_dumps(data, encryption_key, hash_key=None, compression_level=None): key = pad(encryption_key)[:32] cipher, IV = AES_new(key) encrypted_data = base64.urlsafe_b64encode(IV + cipher.encrypt(pad(dump))) - signature = hmac.new(hash_key, encrypted_data).hexdigest() - return signature + ':' + encrypted_data + signature = to_bytes(hmac.new(to_bytes(hash_key), encrypted_data).hexdigest()) + return signature + b':' + encrypted_data def secure_loads(data, encryption_key, hash_key=None, compression_level=None): + encryption_key = to_bytes(encryption_key) + data = to_native(data) if ':' not in data: return None if not hash_key: hash_key = sha1(encryption_key).hexdigest() signature, encrypted_data = data.split(':', 1) - actual_signature = hmac.new(hash_key, encrypted_data).hexdigest() + encrypted_data = to_bytes(encrypted_data) + actual_signature = hmac.new(to_bytes(hash_key), encrypted_data).hexdigest() if not compare(signature, actual_signature): return None key = pad(encryption_key)[:32] @@ -189,7 +193,7 @@ def secure_loads(data, encryption_key, hash_key=None, compression_level=None): cipher, _ = AES_new(key, IV=IV) try: data = cipher.decrypt(encrypted_data) - data = data.rstrip(' ') + data = data.rstrip(b' ') if compression_level: data = zlib.decompress(data) return pickle.loads(data) From a27f6f88efdc1f830e94b422d58d3b29b66c8908 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 11 Jun 2016 15:45:27 +0200 Subject: [PATCH 3/7] fix serializers, websocket_messaging --- gluon/contrib/websocket_messaging.py | 34 ++++++++++++++++++++++------ gluon/custom_import.py | 4 ++-- gluon/serializers.py | 8 +++---- gluon/sqlhtml.py | 2 +- gluon/tests/__init__.py | 2 +- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/gluon/contrib/websocket_messaging.py b/gluon/contrib/websocket_messaging.py index 5531629a..84fe55cc 100644 --- a/gluon/contrib/websocket_messaging.py +++ b/gluon/contrib/websocket_messaging.py @@ -84,7 +84,6 @@ Tornado code inspired by http://thomas.pelletier.im/2010/08/websocket-tornado-re """ from __future__ import print_function - import tornado.httpserver import tornado.websocket import tornado.ioloop @@ -92,17 +91,38 @@ import tornado.web import hmac import sys import optparse -import urllib import time +import sys +if (sys.version_info[0] == 2): + from urllib import urlencode, urlopen + def to_bytes(obj, charset='utf-8', errors='strict'): + if obj is None: + return None + if isinstance(obj, (bytes, bytearray, buffer)): + return bytes(obj) + if isinstance(obj, unicode): + return obj.encode(charset, errors) + raise TypeError('Expected bytes') +else: + from urllib.request import urlopen + from urllib.parse import urlencode + def to_bytes(obj, charset='utf-8', errors='strict'): + if obj is None: + return None + if isinstance(obj, (bytes, bytearray, memoryview)): + return bytes(obj) + if isinstance(obj, str): + return obj.encode(charset, errors) + raise TypeError('Expected bytes') listeners, names, tokens = {}, {}, {} def websocket_send(url, message, hmac_key=None, group='default'): - sig = hmac_key and hmac.new(hmac_key, message).hexdigest() or '' - params = urllib.urlencode( + sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or '' + params = urlencode( {'message': message, 'signature': sig, 'group': group}) - f = urllib.urlopen(url, params) + f = urlopen(url, to_bytes(params)) data = f.read() f.close() return data @@ -121,7 +141,7 @@ class PostHandler(tornado.web.RequestHandler): print('%s:MESSAGE to %s:%s' % (time.time(), group, message)) if hmac_key: signature = self.request.arguments['signature'][0] - if not hmac.new(hmac_key, message).hexdigest() == signature: + if not to_bytes(hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest()) == signature: self.send_error(401) for client in listeners.get(group, []): client.write_message(message) @@ -140,7 +160,7 @@ class TokenHandler(tornado.web.RequestHandler): message = self.request.arguments['message'][0] if hmac_key: signature = self.request.arguments['signature'][0] - if not hmac.new(hmac_key, message).hexdigest() == signature: + if not to_bytes(hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest()) == signature: self.send_error(401) tokens[message] = None diff --git a/gluon/custom_import.py b/gluon/custom_import.py index 9dfb937c..4bd3ab5f 100644 --- a/gluon/custom_import.py +++ b/gluon/custom_import.py @@ -8,7 +8,7 @@ Support for smart import syntax for web2py applications ------------------------------------------------------- """ -from gluon._compat import builtin +from gluon._compat import builtin, unicodeT, PY2 import os import sys import threading @@ -47,7 +47,7 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1): If the import fails, it falls back on naive_importer """ - if isinstance(name, unicode): + if isinstance(name, unicodeT) and PY2: name = name.encode('utf8') globals = globals or {} diff --git a/gluon/serializers.py b/gluon/serializers.py index 61fe88bb..559eb725 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -10,7 +10,7 @@ from gluon.html import TAG, XmlComponent, xmlescape from gluon.languages import lazyT import gluon.contrib.rss2 as rss2 import json as json_parser -from gluon._compat import long +from gluon._compat import long, to_native, unicodeT have_yaml = True try: @@ -43,7 +43,7 @@ def cast_keys(o, cast=str, encoding="utf-8"): else: newobj = Storage() for k, v in o.items(): - if (cast == str) and isinstance(k, unicode): + if (cast == str) and isinstance(k, unicodeT): key = k.encode(encoding) else: key = cast(k) @@ -88,7 +88,7 @@ def custom_json(o): elif isinstance(o, lazyT): return str(o) elif isinstance(o, XmlComponent): - return str(o) + return to_native(o.xml()) elif isinstance(o, set): return list(o) elif hasattr(o, 'as_list') and callable(o.as_list): @@ -161,7 +161,7 @@ def ics(events, title=None, link=None, timeshift=0, calname=True, return s def safe_encode(text): - if not isinstance(text, (str, unicode)): + if not isinstance(text, (str, unicodeT)): text = str(text) try: text = text.encode('utf8','replace') diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 84e25d3e..6bf336b4 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -3487,7 +3487,7 @@ class ExporterTSV(ExportClass): import codecs final.write(codecs.BOM_UTF16) writer.writerow( - [unicode(col).encode("utf8") for col in self.rows.colnames]) + [to_unicode(col, "utf8") for col in self.rows.colnames]) data = out.getvalue().decode("utf8") data = data.encode("utf-16") data = data[2:] diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index 46a0f107..cb3f7c13 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -16,12 +16,12 @@ from .test_router import * from .test_validators import * from .test_tools import * from .test_utils import * +from .test_serializers import * if sys.version[:3] == '2.7': from .test_compileapp import * from .test_is_url import * from .test_languages import * - from .test_serializers import * from .test_appadmin import * from .test_scheduler import * from .test_web import * From 34f753be56f05f67e652c2d83e7f2424b3c19d55 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 11 Jun 2016 20:12:30 +0200 Subject: [PATCH 4/7] fix languages --- gluon/languages.py | 20 ++++++++------------ gluon/tests/__init__.py | 2 +- gluon/tests/test_languages.py | 7 +++++-- gluon/utf8.py | 26 +++++++++++++------------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/gluon/languages.py b/gluon/languages.py index 2d29a51a..b40cabd3 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -18,7 +18,8 @@ import pkgutil import logging from cgi import escape from threading import RLock -from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, _local_html_escape, to_native +from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, _local_html_escape, to_native, \ + pjoin from gluon.portalocker import read_locked, LockedFile from gluon.utf8 import Utf8 @@ -34,8 +35,6 @@ __all__ = ['translator', 'findT', 'update_all_languages'] ostat = os.stat oslistdir = os.listdir -pjoin = os.path.join -pexists = os.path.exists pdirname = os.path.dirname isdir = os.path.isdir @@ -167,7 +166,7 @@ def read_dict_aux(filename): lang_text = read_locked(filename).replace(b'\r\n', b'\n') clear_cache(filename) try: - return safe_eval(lang_text) or {} + return safe_eval(to_native(lang_text)) or {} except Exception: e = sys.exc_info()[1] status = 'Syntax error in %s (%s)' % (filename, e) @@ -623,7 +622,6 @@ class translator(object): of them matches possible_languages. """ pl_info = read_possible_languages(self.langpath) - def set_plural(language): """ initialize plural forms subsystem @@ -800,18 +798,16 @@ class translator(object): the ## notation is ignored in multiline strings and strings that start with ##. This is needed to allow markmin syntax to be translated """ - if isinstance(message, unicodeT): - message = message.encode('utf8') - if isinstance(prefix, unicodeT): - prefix = prefix.encode('utf8') + message = to_native(message, 'utf8') + prefix = to_native(prefix, 'utf8') key = prefix + message mt = self.t.get(key, None) if mt is not None: return mt # we did not find a translation - if message.find(to_bytes('##')) > 0: + if message.find('##') > 0: pass - if message.find(to_bytes('##')) > 0 and not '\n' in message: + if message.find('##') > 0 and not '\n' in message: # remove comments message = message.rsplit('##', 1)[0] # guess translation same as original @@ -966,7 +962,7 @@ def findT(path, language=DEFAULT_LANGUAGE): for filename in \ listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\ + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0): - data = read_locked(filename) + data = to_native(read_locked(filename)) items = regex_translate.findall(data) items += regex_translate_m.findall(data) for item in items: diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index cb3f7c13..5db0c683 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -17,11 +17,11 @@ from .test_validators import * from .test_tools import * from .test_utils import * from .test_serializers import * +from .test_languages import * if sys.version[:3] == '2.7': from .test_compileapp import * from .test_is_url import * - from .test_languages import * from .test_appadmin import * from .test_scheduler import * from .test_web import * diff --git a/gluon/tests/test_languages.py b/gluon/tests/test_languages.py index bed7ff42..0df6f21b 100644 --- a/gluon/tests/test_languages.py +++ b/gluon/tests/test_languages.py @@ -15,6 +15,7 @@ from .fix_path import fix_sys_path fix_sys_path(__file__) from gluon import languages +from gluon._compat import PY2 MP_WORKING = 0 try: @@ -105,8 +106,10 @@ class TestTranslations(unittest.TestCase): '1 quark') self.assertEqual(str(T('%s %%{quark[0]}', 2)), '2 quarks') - self.assertEqual(str(T.M('**Hello World**')), - 'Hello World') + if PY2: + # FIXME PY3 markmin is not supported yet + self.assertEqual(str(T.M('**Hello World**')), + 'Hello World') T.force('it') self.assertEqual(str(T('Hello World')), 'Salve Mondo') diff --git a/gluon/utf8.py b/gluon/utf8.py index 6fbab2dd..21fd12c4 100644 --- a/gluon/utf8.py +++ b/gluon/utf8.py @@ -11,7 +11,7 @@ Utilities and class for UTF8 strings managing ---------------------------------------------- """ from __future__ import print_function -from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode +from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native __all__ = ['Utf8'] @@ -51,10 +51,10 @@ def sort_key(s): from gluon.contrib.pyuca import unicode_collator unicode_sort_key = unicode_collator.sort_key sort_key = lambda s: unicode_sort_key( - unicode(s, 'utf-8') if isinstance(s, str) else s) + to_unicode(s, 'utf-8') if isinstance(s, str) else s) except: sort_key = lambda s: ( - unicode(s, 'utf-8') if isinstance(s, str) else s).lower() + to_unicode(s, 'utf-8') if isinstance(s, str) else s).lower() return sort_key(s) @@ -64,7 +64,7 @@ def ord(char): """ if isinstance(char, unicodeT): return __builtin__.ord(char) - return __builtin__.ord(unicode(char, 'utf-8')) + return __builtin__.ord(to_unicode(char, 'utf-8')) def chr(code): @@ -92,8 +92,8 @@ def truncate(string, length, dots='...'): Returns: (utf8-str): original or cutted string """ - text = unicode(string, 'utf-8') - dots = unicode(dots, 'utf-8') if isinstance(dots, str) else dots + text = to_unicode(string, 'utf-8') + dots = to_unicode(dots, 'utf-8') if isinstance(dots, str) else dots if len(text) > length: text = text[:length - len(dots)] + dots return str.__new__(Utf8, text.encode('utf-8')) @@ -120,11 +120,11 @@ class Utf8(str): """ def __new__(cls, content='', codepage='utf-8'): if isinstance(content, unicodeT): - return str.__new__(cls, unicode.encode(content, 'utf-8')) + return str.__new__(cls, to_native(content, 'utf-8')) elif codepage in ('utf-8', 'utf8') or isinstance(content, cls): return str.__new__(cls, content) else: - return str.__new__(cls, unicode(content, codepage).encode('utf-8')) + return str.__new__(cls, to_native(to_unicode(content, codepage), 'utf-8')) def __repr__(self): r''' # note that we use raw strings to avoid having to use double back slashes below @@ -156,9 +156,9 @@ class Utf8(str): True ''' if str.find(self, "'") >= 0 and str.find(self, '"') < 0: # only single quote exists - return '"' + unicode(self, 'utf-8').translate(repr_escape_tab).encode('utf-8') + '"' + return '"' + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab), 'utf-8') + '"' else: - return "'" + unicode(self, 'utf-8').translate(repr_escape_tab2).encode('utf-8') + "'" + return "'" + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab2), 'utf-8') + "'" def __size__(self): """ length of utf-8 string in bytes """ @@ -168,17 +168,17 @@ class Utf8(str): return str.__contains__(self, Utf8(other)) def __getitem__(self, index): - return str.__new__(Utf8, unicode(self, 'utf-8')[index].encode('utf-8')) + return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8')) def __getslice__(self, begin, end): - return str.__new__(Utf8, unicode(self, 'utf-8')[begin:end].encode('utf-8')) + return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8')) def __add__(self, other): return str.__new__(Utf8, str.__add__(self, unicode.encode(other, 'utf-8') if isinstance(other, unicode) else other)) def __len__(self): - return len(unicode(self, 'utf-8')) + return len(to_unicode(self, 'utf-8')) def __mul__(self, integer): return str.__new__(Utf8, str.__mul__(self, integer)) From 48209f5bdfff6f6a52c2980b5277354c5ee596b2 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sun, 12 Jun 2016 13:22:34 +0200 Subject: [PATCH 5/7] fix compileapp --- gluon/compileapp.py | 6 +++--- gluon/custom_import.py | 6 +++--- gluon/fileutils.py | 6 +++++- gluon/globals.py | 4 ++-- gluon/template.py | 7 +++---- gluon/tests/__init__.py | 2 +- gluon/tools.py | 4 ++-- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 679732c3..7c28fcbb 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -18,7 +18,7 @@ import fnmatch import os import copy import random -from gluon._compat import builtin, PY2 +from gluon._compat import builtin, PY2, unicodeT, to_native from gluon.storage import Storage, List from gluon.template import parse_template from gluon.restricted import restricted, compile2 @@ -650,8 +650,8 @@ def run_controller_in(controller, function, environment): vars = response._vars if response.postprocessing: vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars) - if isinstance(vars, unicode): - vars = vars.encode('utf8') + if isinstance(vars, unicodeT): + vars = to_native(vars) elif hasattr(vars, 'xml') and callable(vars.xml): vars = vars.xml() return vars diff --git a/gluon/custom_import.py b/gluon/custom_import.py index 4bd3ab5f..ad055153 100644 --- a/gluon/custom_import.py +++ b/gluon/custom_import.py @@ -8,7 +8,7 @@ Support for smart import syntax for web2py applications ------------------------------------------------------- """ -from gluon._compat import builtin, unicodeT, PY2 +from gluon._compat import builtin, unicodeT, PY2, to_native import os import sys import threading @@ -47,8 +47,8 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1): If the import fails, it falls back on naive_importer """ - if isinstance(name, unicodeT) and PY2: - name = name.encode('utf8') + if isinstance(name, unicodeT): + name = to_native(name) globals = globals or {} locals = locals or {} diff --git a/gluon/fileutils.py b/gluon/fileutils.py index 84e65365..e9d869b3 100644 --- a/gluon/fileutils.py +++ b/gluon/fileutils.py @@ -21,6 +21,7 @@ import logging from gluon.http import HTTP from gzip import open as gzopen from gluon.recfile import generate +from gluon._compat import PY2 __all__ = [ 'parse_version', @@ -100,7 +101,10 @@ def read_file(filename, mode='r'): """Returns content from filename, making sure to close the file explicitly on exit. """ - f = open(filename, mode) + if PY2 or 'b' in mode: + f = open(filename, mode) + else: + f = open(filename, mode, encoding="utf8") try: return f.read() finally: diff --git a/gluon/globals.py b/gluon/globals.py index 43079593..717d923b 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -13,7 +13,7 @@ Contains the classes for the global used variables: - Session """ -from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native +from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT from gluon.storage import Storage, List from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE from gluon.contenttype import contenttype @@ -569,7 +569,7 @@ class Response(Storage): if not request: request = current.request - if isinstance(stream, (str, unicode)): + if isinstance(stream, (str, unicodeT)): stream_file_or_304_or_206(stream, chunk_size=chunk_size, request=request, diff --git a/gluon/template.py b/gluon/template.py index 7d79c573..2afc29ab 100644 --- a/gluon/template.py +++ b/gluon/template.py @@ -263,7 +263,6 @@ class TemplateParser(object): # This will end up as # "%s(%s, escape=False)" % (self.writer, value) self.writer = writer - # Dictionary of custom name lexers to use. if isinstance(lexers, dict): self.lexers = lexers @@ -448,7 +447,7 @@ class TemplateParser(object): fileobj.close() except IOError: self._raise_error('Unable to open included view file: ' + filepath) - + text = to_native(text) return text def include(self, content, filename): @@ -788,7 +787,7 @@ def parse_template(filename, raise RestrictedError(filename, '', 'Unable to find the file') else: text = filename.read() - + text = to_native(text) # Use the file contents to get a parsed template and return it. return str(TemplateParser(text, context=context, path=path, lexers=lexers, delimiters=delimiters)) @@ -885,7 +884,7 @@ def render(content="hello world", """ # here to avoid circular Imports try: - from globals import Response + from gluon.globals import Response except ImportError: # Working standalone. Build a mock Response object. Response = DummyResponse diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index 5db0c683..8a43f01c 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -18,9 +18,9 @@ from .test_tools import * from .test_utils import * from .test_serializers import * from .test_languages import * +from .test_compileapp import * if sys.version[:3] == '2.7': - from .test_compileapp import * from .test_is_url import * from .test_appadmin import * from .test_scheduler import * diff --git a/gluon/tools.py b/gluon/tools.py index 8736b814..7bfa4b7f 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1457,7 +1457,7 @@ class AuthJWT(object): def f(*args, **kwargs): try: token = self.get_jwt_token_from_request(token_param=token_param) - except HTTP, e: + except HTTP as e: if required: raise e token = None @@ -4759,7 +4759,7 @@ class Auth(object): self._wiki.automenu() -class Crud(object): +class Crud(object): # pragma: no cover def url(self, f=None, args=None, vars=None): """ From 0f648eee561184c8da50a894f0b5308412a894b8 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Tue, 14 Jun 2016 18:01:12 +0200 Subject: [PATCH 6/7] enabled pg8000 pymysql --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 73433e70..a55f3cf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION" source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate" fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install --download-cache $HOME/.pip-cache pycrypto; fi; + - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install --download-cache $HOME/.pip-cache pycrypto pg8000 pymysql; fi; - if [[ $TRAVIS_PYTHON_VERSION != '3.5' ]]; then pip install -e .; fi; before_script: From 2aeb063890d6177de354d6f7c324a2f449c184e4 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Tue, 14 Jun 2016 20:31:41 +0200 Subject: [PATCH 7/7] enabled test_appadmin, fix markmin2html, fix main.py --- gluon/compileapp.py | 6 +++--- gluon/contrib/markmin/markmin2html.py | 18 ++++++------------ gluon/custom_import.py | 3 +++ gluon/html.py | 3 ++- gluon/languages.py | 8 +++----- gluon/main.py | 5 +++-- gluon/rocket.py | 4 ++-- gluon/streamer.py | 8 ++++++-- gluon/tests/__init__.py | 2 +- gluon/tests/test_languages.py | 6 ++---- 10 files changed, 31 insertions(+), 32 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 7c28fcbb..b6bbda79 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -18,7 +18,7 @@ import fnmatch import os import copy import random -from gluon._compat import builtin, PY2, unicodeT, to_native +from gluon._compat import builtin, PY2, unicodeT, to_native, basestring from gluon.storage import Storage, List from gluon.template import parse_template from gluon.restricted import restricted, compile2 @@ -151,7 +151,7 @@ def LOAD(c=None, f='index', args=None, vars=None, "infinity" or "continuous" are accepted to reload indefinitely the component """ - from html import TAG, DIV, URL, SCRIPT, XML + from gluon.html import TAG, DIV, URL, SCRIPT, XML if args is None: args = [] vars = Storage(vars or {}) @@ -430,7 +430,7 @@ def build_environment(request, response, session, store_current=True): __builtins__ = mybuiltin() elif is_pypy: # apply the same hack to pypy too __builtins__ = mybuiltin() - else: + elif PY2: __builtins__['__import__'] = builtin.__import__ # WHY? environment['request'] = request environment['response'] = response diff --git a/gluon/contrib/markmin/markmin2html.py b/gluon/contrib/markmin/markmin2html.py index cae5ff18..0752e063 100755 --- a/gluon/contrib/markmin/markmin2html.py +++ b/gluon/contrib/markmin/markmin2html.py @@ -6,15 +6,10 @@ from __future__ import print_function import re import urllib -from cgi import escape -from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes +from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes, to_native, _local_html_escape as escape +from ast import parse as ast_parse +import ast -try: - from ast import parse as ast_parse - import ast -except ImportError: # python 2.5 - from compiler import parse - import compiler.ast as ast """ TODO: next version should use MathJax @@ -950,12 +945,11 @@ def render(text, if protolinks == "default": protolinks = protolinks_simple pp = '\n' if pretty_print else '' - if isinstance(text, unicodeT): - text = text.encode('utf8') - text = str(text or '') + text = to_native(text) + if not (isinstance(text, str)): + text = str(text or '') text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text) text = text.replace('\x05', '').replace('\r\n', '\n') # concatenate strings separeted by \\n - if URL is not None: text = replace_at_urls(text, URL) diff --git a/gluon/custom_import.py b/gluon/custom_import.py index ad055153..8e848626 100644 --- a/gluon/custom_import.py +++ b/gluon/custom_import.py @@ -104,6 +104,9 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1): finally: if import_tb: import_tb = None + elif not(PY2) and level < 0: + # FIXME PY3 why level is < 0? + level = 0 return NATIVE_IMPORTER(name, globals, locals, fromlist, level) diff --git a/gluon/html.py b/gluon/html.py index 642f097c..f2fb7b9b 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -974,7 +974,8 @@ class DIV(XmlComponent): """ str(COMPONENT) returns COMPONENT.xml() """ - return self.xml() + # In PY3 __str__ cannot return bytes (TypeError: __str__ returned non-string (type bytes)) + return to_native(self.xml()) def flatten(self, render=None): """ diff --git a/gluon/languages.py b/gluon/languages.py index b40cabd3..6b4748d8 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -27,9 +27,7 @@ from gluon.utf8 import Utf8 from gluon.fileutils import listdir from gluon.cfs import getcfs from gluon.html import XML, xmlescape -if PY2: - # FIXME PY3 - from gluon.contrib.markmin.markmin2html import render, markmin_escape +from gluon.contrib.markmin.markmin2html import render, markmin_escape __all__ = ['translator', 'findT', 'update_all_languages'] @@ -761,10 +759,10 @@ class translator(object): symbols = (symbols,) symbols = tuple( value if isinstance(value, NUMBERS) - else xmlescape(value).translate(ttab_in) + else to_native(xmlescape(value)).translate(ttab_in) for value in symbols) message = self.params_substitution(message, symbols) - return XML(message.translate(ttab_out)) + return to_native(XML(message.translate(ttab_out)).xml()) def M(self, message, symbols={}, language=None, lazy=None, filter=None, ftag=None, ns=None): diff --git a/gluon/main.py b/gluon/main.py index d7a977c9..e438f4ba 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -298,6 +298,7 @@ def wsgibase(environ, responder): env.web2py_version = web2py_version #env.update(global_settings) static_file = False + http_response = None try: try: try: @@ -439,8 +440,8 @@ def wsgibase(environ, responder): gluon.debug.dbg.do_debug(mainpyfile=request.folder) serve_controller(request, response, session) - - except HTTP as http_response: + except HTTP as hr: + http_response = hr if static_file: return http_response.to(responder, env=env) diff --git a/gluon/rocket.py b/gluon/rocket.py index 51460532..199f1004 100644 --- a/gluon/rocket.py +++ b/gluon/rocket.py @@ -11,7 +11,7 @@ import errno import socket import logging import platform -from gluon._compat import iteritems +from gluon._compat import iteritems, to_bytes # Define Constants VERSION = '1.2.6' @@ -1772,7 +1772,7 @@ class WSGIWorker(Worker): if self.chunked: self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data))) else: - self.conn.sendall(data) + self.conn.sendall(to_bytes(data)) except socket.timeout: self.closeConnection = True except socket.error: diff --git a/gluon/streamer.py b/gluon/streamer.py index 1095b592..dd7b78b2 100644 --- a/gluon/streamer.py +++ b/gluon/streamer.py @@ -17,6 +17,7 @@ import re import errno from gluon.http import HTTP from gluon.contenttype import contenttype +from gluon._compat import PY2 regex_start_range = re.compile('\d+(?=\-)') @@ -53,8 +54,11 @@ def stream_file_or_304_or_206( # if error_message is None: # error_message = rewrite.THREAD_LOCAL.routes.error_message % 'invalid request' try: - open = file # this makes no sense but without it GAE cannot open files - fp = open(static_file,'rb') + if PY2: + open_f = file # this makes no sense but without it GAE cannot open files + else: + open_f = open + fp = open_f(static_file,'rb') except IOError as e: if e[0] == errno.EISDIR: raise HTTP(403, error_message, web2py_error='file is a directory') diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index 8a43f01c..4c879ad3 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -19,10 +19,10 @@ from .test_utils import * from .test_serializers import * from .test_languages import * from .test_compileapp import * +from .test_appadmin import * if sys.version[:3] == '2.7': from .test_is_url import * - from .test_appadmin import * from .test_scheduler import * from .test_web import * from .test_old_doctests import * diff --git a/gluon/tests/test_languages.py b/gluon/tests/test_languages.py index 0df6f21b..acc621ea 100644 --- a/gluon/tests/test_languages.py +++ b/gluon/tests/test_languages.py @@ -106,10 +106,8 @@ class TestTranslations(unittest.TestCase): '1 quark') self.assertEqual(str(T('%s %%{quark[0]}', 2)), '2 quarks') - if PY2: - # FIXME PY3 markmin is not supported yet - self.assertEqual(str(T.M('**Hello World**')), - 'Hello World') + self.assertEqual(str(T.M('**Hello World**')), + 'Hello World') T.force('it') self.assertEqual(str(T('Hello World')), 'Salve Mondo')