Merge pull request #1361 from ilvalle/py3_fixes_step2

few py3 fixes

.travis.yml

@@ -25,7 +25,7 @@ install:
         virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION"
         source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate"
       fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install --download-cache $HOME/.pip-cache pycrypto; fi;
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install --download-cache $HOME/.pip-cache pycrypto pg8000 pymysql; fi;
   - if [[ $TRAVIS_PYTHON_VERSION != '3.5' ]]; then pip install -e .; fi;
 
 before_script:

gluon/_compat.py

@@ -21,7 +21,10 @@ if PY2:
     import ConfigParser as configparser
     from email.MIMEBase import MIMEBase
     from email.Header import Header
-    from email import MIMEMultipart, MIMEText, Encoders, Charset
+    from email import Encoders, Charset
+    from email.MIMEMultipart import MIMEMultipart
+    from email.MIMEText import MIMEText
+    from email.Charset import add_charset, QP as charset_QP
     from urllib import FancyURLopener, urlencode, urlopen
     from urllib import quote as urllib_quote, unquote as urllib_unquote
     from string import maketrans
@@ -88,7 +91,7 @@ else:
     from email.mime.text import MIMEText
     from email import encoders as Encoders
     from email.header import Header
-    from email.charset import Charset
+    from email.charset import Charset, add_charset, QP as charset_QP
     from urllib.request import FancyURLopener, urlopen
     from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode
     import html

gluon/compileapp.py

@@ -18,7 +18,7 @@ import fnmatch
 import os
 import copy
 import random
-from gluon._compat import builtin, PY2
+from gluon._compat import builtin, PY2, unicodeT, to_native, basestring
 from gluon.storage import Storage, List
 from gluon.template import parse_template
 from gluon.restricted import restricted, compile2
@@ -151,7 +151,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
             "infinity" or "continuous" are accepted to reload indefinitely the
             component
     """
-    from html import TAG, DIV, URL, SCRIPT, XML
+    from gluon.html import TAG, DIV, URL, SCRIPT, XML
     if args is None:
         args = []
     vars = Storage(vars or {})
@@ -430,7 +430,7 @@ def build_environment(request, response, session, store_current=True):
         __builtins__ = mybuiltin()
     elif is_pypy:  # apply the same hack to pypy too
         __builtins__ = mybuiltin()
-    else:
+    elif PY2:
         __builtins__['__import__'] = builtin.__import__  # WHY?
     environment['request'] = request
     environment['response'] = response
@@ -650,8 +650,8 @@ def run_controller_in(controller, function, environment):
     vars = response._vars
     if response.postprocessing:
         vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
-    if isinstance(vars, unicode):
-        vars = vars.encode('utf8')
+    if isinstance(vars, unicodeT):
+        vars = to_native(vars)
     elif hasattr(vars, 'xml') and callable(vars.xml):
         vars = vars.xml()
     return vars

gluon/contenttype.py

@@ -19,6 +19,7 @@ Additions:
  - .pickle: application/python-pickle
  - .w2p': application/w2p
 """
+from gluon._compat import to_native
 
 __all__ = ['contenttype']
 
@@ -842,7 +843,7 @@ def contenttype(filename, default='text/plain'):
     """
     Returns the Content-Type string matching extension of the given filename.
     """
-
+    filename=to_native(filename)
     i = filename.rfind('.')
     if i >= 0:
         default = CONTENT_TYPE.get(filename[i:].lower(), default)

gluon/contrib/markmin/markmin2html.py

@@ -6,15 +6,10 @@
 from __future__ import print_function
 import re
 import urllib
-from cgi import escape
-from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes
+from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes, to_native, _local_html_escape as escape
+from ast import parse as ast_parse
+import ast
 
-try:
-    from ast import parse as ast_parse
-    import ast
-except ImportError:  # python 2.5
-    from compiler import parse
-    import compiler.ast as ast
 
 """
 TODO: next version should use MathJax
@@ -950,12 +945,11 @@ def render(text,
     if protolinks == "default":
         protolinks = protolinks_simple
     pp = '\n' if pretty_print else ''
-    if isinstance(text, unicodeT):
-        text = text.encode('utf8')
-    text = str(text or '')
+    text = to_native(text)
+    if not (isinstance(text, str)):
+        text = str(text or '')
     text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
     text = text.replace('\x05', '').replace('\r\n', '\n')  # concatenate strings separeted by \\n
-
     if URL is not None:
         text = replace_at_urls(text, URL)
 

gluon/contrib/websocket_messaging.py

@@ -84,7 +84,6 @@ Tornado code inspired by http://thomas.pelletier.im/2010/08/websocket-tornado-re
 
 """
 from __future__ import print_function
-
 import tornado.httpserver
 import tornado.websocket
 import tornado.ioloop
@@ -92,17 +91,38 @@ import tornado.web
 import hmac
 import sys
 import optparse
-import urllib
 import time
+import sys
+if (sys.version_info[0] == 2):
+    from urllib import urlencode, urlopen
+    def to_bytes(obj, charset='utf-8', errors='strict'):
+        if obj is None:
+            return None
+        if isinstance(obj, (bytes, bytearray, buffer)):
+            return bytes(obj)
+        if isinstance(obj, unicode):
+            return obj.encode(charset, errors)
+        raise TypeError('Expected bytes')
+else:
+    from urllib.request import urlopen
+    from urllib.parse import urlencode
+    def to_bytes(obj, charset='utf-8', errors='strict'):
+        if obj is None:
+            return None
+        if isinstance(obj, (bytes, bytearray, memoryview)):
+            return bytes(obj)
+        if isinstance(obj, str):
+            return obj.encode(charset, errors)
+        raise TypeError('Expected bytes')
 
 listeners, names, tokens = {}, {}, {}
 
 
 def websocket_send(url, message, hmac_key=None, group='default'):
-    sig = hmac_key and hmac.new(hmac_key, message).hexdigest() or ''
-    params = urllib.urlencode(
+    sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or ''
+    params = urlencode(
         {'message': message, 'signature': sig, 'group': group})
-    f = urllib.urlopen(url, params)
+    f = urlopen(url, to_bytes(params))
     data = f.read()
     f.close()
     return data
@@ -121,7 +141,7 @@ class PostHandler(tornado.web.RequestHandler):
             print('%s:MESSAGE to %s:%s' % (time.time(), group, message))
             if hmac_key:
                 signature = self.request.arguments['signature'][0]
-                if not hmac.new(hmac_key, message).hexdigest() == signature:
+                if not to_bytes(hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest()) == signature:
                     self.send_error(401)
             for client in listeners.get(group, []):
                 client.write_message(message)
@@ -140,7 +160,7 @@ class TokenHandler(tornado.web.RequestHandler):
             message = self.request.arguments['message'][0]
             if hmac_key:
                 signature = self.request.arguments['signature'][0]
-                if not hmac.new(hmac_key, message).hexdigest() == signature:
+                if not to_bytes(hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest()) == signature:
                     self.send_error(401)
             tokens[message] = None
 

gluon/custom_import.py

@@ -8,7 +8,7 @@
 Support for smart import syntax for web2py applications
 -------------------------------------------------------
 """
-from gluon._compat import builtin
+from gluon._compat import builtin, unicodeT, PY2, to_native
 import os
 import sys
 import threading
@@ -47,8 +47,8 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
     If the import fails, it falls back on naive_importer
     """
 
-    if isinstance(name, unicode):
-        name = name.encode('utf8')
+    if isinstance(name, unicodeT):
+        name = to_native(name)
 
     globals = globals or {}
     locals = locals or {}
@@ -104,6 +104,9 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
         finally:
             if import_tb:
                 import_tb = None
+    elif not(PY2) and level < 0:
+        # FIXME PY3 why level is < 0?
+        level = 0
 
     return NATIVE_IMPORTER(name, globals, locals, fromlist, level)
 

gluon/fileutils.py

@@ -21,6 +21,7 @@ import logging
 from gluon.http import HTTP
 from gzip import open as gzopen
 from gluon.recfile import generate
+from gluon._compat import PY2
 
 __all__ = [
     'parse_version',
@@ -100,7 +101,10 @@ def read_file(filename, mode='r'):
     """Returns content from filename, making sure to close the file explicitly
     on exit.
     """
-    f = open(filename, mode)
+    if PY2 or 'b' in mode:
+        f = open(filename, mode)
+    else:
+        f = open(filename, mode, encoding="utf8")
     try:
         return f.read()
     finally:

gluon/globals.py

@@ -13,7 +13,7 @@ Contains the classes for the global used variables:
 - Session
 
 """
-from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native
+from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -569,7 +569,7 @@ class Response(Storage):
 
         if not request:
             request = current.request
-        if isinstance(stream, (str, unicode)):
+        if isinstance(stream, (str, unicodeT)):
             stream_file_or_304_or_206(stream,
                                       chunk_size=chunk_size,
                                       request=request,

gluon/html.py

@@ -974,7 +974,8 @@ class DIV(XmlComponent):
         """
         str(COMPONENT) returns COMPONENT.xml()
         """
-        return self.xml()
+        # In PY3 __str__ cannot return bytes (TypeError: __str__ returned non-string (type bytes))
+        return to_native(self.xml())
         
     def flatten(self, render=None):
         """

gluon/languages.py

@@ -18,7 +18,8 @@ import pkgutil
 import logging
 from cgi import escape
 from threading import RLock
-from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems
+from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, _local_html_escape, to_native, \
+                          pjoin
 
 from gluon.portalocker import read_locked, LockedFile
 from gluon.utf8 import Utf8
@@ -26,16 +27,12 @@ from gluon.utf8 import Utf8
 from gluon.fileutils import listdir
 from gluon.cfs import getcfs
 from gluon.html import XML, xmlescape
-if PY2:
-    # FIXME PY3
-    from gluon.contrib.markmin.markmin2html import render, markmin_escape
+from gluon.contrib.markmin.markmin2html import render, markmin_escape
 
 __all__ = ['translator', 'findT', 'update_all_languages']
 
 ostat = os.stat
 oslistdir = os.listdir
-pjoin = os.path.join
-pexists = os.path.exists
 pdirname = os.path.dirname
 isdir = os.path.isdir
 
@@ -167,7 +164,7 @@ def read_dict_aux(filename):
     lang_text = read_locked(filename).replace(b'\r\n', b'\n')
     clear_cache(filename)
     try:
-        return safe_eval(lang_text) or {}
+        return safe_eval(to_native(lang_text)) or {}
     except Exception:
         e = sys.exc_info()[1]
         status = 'Syntax error in %s (%s)' % (filename, e)
@@ -426,7 +423,7 @@ class lazyT(object):
         return len(str(self))
 
     def xml(self):
-        return str(self) if self.M else escape(str(self))
+        return str(self) if self.M else _local_html_escape(str(self), quote=False)
 
     def encode(self, *a, **b):
         return str(self).encode(*a, **b)
@@ -623,7 +620,6 @@ class translator(object):
         of them matches possible_languages.
         """
         pl_info = read_possible_languages(self.langpath)
-
         def set_plural(language):
             """
             initialize plural forms subsystem
@@ -763,10 +759,10 @@ class translator(object):
                     symbols = (symbols,)
                 symbols = tuple(
                     value if isinstance(value, NUMBERS)
-                    else xmlescape(value).translate(ttab_in)
+                    else to_native(xmlescape(value)).translate(ttab_in)
                     for value in symbols)
             message = self.params_substitution(message, symbols)
-        return XML(message.translate(ttab_out))
+        return to_native(XML(message.translate(ttab_out)).xml())
 
     def M(self, message, symbols={}, language=None,
           lazy=None, filter=None, ftag=None, ns=None):
@@ -800,18 +796,16 @@ class translator(object):
         the ## notation is ignored in multiline strings and strings that
         start with ##. This is needed to allow markmin syntax to be translated
         """
-        if isinstance(message, unicodeT):
-            message = message.encode('utf8')
-        if isinstance(prefix, unicodeT):
-            prefix = prefix.encode('utf8')
+        message = to_native(message, 'utf8')
+        prefix = to_native(prefix, 'utf8')
         key = prefix + message
         mt = self.t.get(key, None)
         if mt is not None:
             return mt
         # we did not find a translation
-        if message.find(to_bytes('##')) > 0:
+        if message.find('##') > 0:
             pass
-        if message.find(to_bytes('##')) > 0 and not '\n' in message:
+        if message.find('##') > 0 and not '\n' in message:
             # remove comments
             message = message.rsplit('##', 1)[0]
         # guess translation same as original
@@ -821,7 +815,7 @@ class translator(object):
                 self.language_file != self.default_language_file:
             write_dict(self.language_file, self.t)
         return regex_backslash.sub(
-            lambda m: m.group(1).translate(ttab_in), str(mt))
+            lambda m: m.group(1).translate(ttab_in), to_native(mt))
 
     def params_substitution(self, message, symbols):
         """
@@ -966,7 +960,7 @@ def findT(path, language=DEFAULT_LANGUAGE):
     for filename in \
             listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\
             + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0):
-        data = read_locked(filename)
+        data = to_native(read_locked(filename))
         items = regex_translate.findall(data)
         items += regex_translate_m.findall(data)
         for item in items:

gluon/main.py

@@ -298,6 +298,7 @@ def wsgibase(environ, responder):
     env.web2py_version = web2py_version
     #env.update(global_settings)
     static_file = False
+    http_response = None
     try:
         try:
             try:
@@ -439,8 +440,8 @@ def wsgibase(environ, responder):
                     gluon.debug.dbg.do_debug(mainpyfile=request.folder)
 
                 serve_controller(request, response, session)
-
-            except HTTP as http_response:
+            except HTTP as hr:
+                http_response = hr
 
                 if static_file:
                     return http_response.to(responder, env=env)

gluon/rocket.py

@@ -11,7 +11,7 @@ import errno
 import socket
 import logging
 import platform
-from gluon._compat import iteritems
+from gluon._compat import iteritems, to_bytes
 
 # Define Constants
 VERSION = '1.2.6'
@@ -1772,7 +1772,7 @@ class WSGIWorker(Worker):
                 if self.chunked:
                     self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data)))
                 else:
-                    self.conn.sendall(data)
+                    self.conn.sendall(to_bytes(data))
             except socket.timeout:
                 self.closeConnection = True
             except socket.error:

gluon/serializers.py

@@ -10,7 +10,7 @@ from gluon.html import TAG, XmlComponent, xmlescape
 from gluon.languages import lazyT
 import gluon.contrib.rss2 as rss2
 import json as json_parser
-from gluon._compat import long
+from gluon._compat import long, to_native, unicodeT
 
 have_yaml = True
 try:
@@ -43,7 +43,7 @@ def cast_keys(o, cast=str, encoding="utf-8"):
         else:
             newobj = Storage()
         for k, v in o.items():
-            if (cast == str) and isinstance(k, unicode):
+            if (cast == str) and isinstance(k, unicodeT):
                 key = k.encode(encoding)
             else:
                 key = cast(k)
@@ -83,10 +83,12 @@ def custom_json(o):
         return int(o)
     elif isinstance(o, decimal.Decimal):
         return str(o)
+    elif isinstance(o, (bytes, bytearray)):
+        return str(o)
     elif isinstance(o, lazyT):
         return str(o)
     elif isinstance(o, XmlComponent):
-        return str(o)
+        return to_native(o.xml())
     elif isinstance(o, set):
         return list(o)
     elif hasattr(o, 'as_list') and callable(o.as_list):
@@ -159,7 +161,7 @@ def ics(events, title=None, link=None, timeshift=0, calname=True,
     return s
 
 def safe_encode(text):
-    if not isinstance(text, (str, unicode)):
+    if not isinstance(text, (str, unicodeT)):
         text = str(text)
     try:
         text = text.encode('utf8','replace')

gluon/sqlhtml.py

@@ -19,7 +19,7 @@ import urllib
 import re
 
 import os
-from gluon._compat import StringIO, unichr, urllib_quote, iteritems
+from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT
 from gluon.http import HTTP, redirect
 from gluon.html import XmlComponent, truncate_string
 from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG
@@ -1141,7 +1141,7 @@ class SQLFORM(FORM):
 
         # try to retrieve the indicated record using its id
         # otherwise ignore it
-        if record and isinstance(record, (int, long, str, unicode)):
+        if record and isinstance(record, (int, long, str, unicodeT)):
             if not str(record).isdigit():
                 raise HTTP(404, "Object not found")
             record = table._db(table._id == record).select().first()
@@ -1651,7 +1651,7 @@ class SQLFORM(FORM):
                         original_filename = os.path.split(f)[1]
                 elif hasattr(f, 'file'):
                     (source_file, original_filename) = (f.file, f.filename)
-                elif isinstance(f, (str, unicode)):
+                elif isinstance(f, (str, unicodeT)):
                     # do not know why this happens, it should not
                     (source_file, original_filename) = \
                         (StringIO(f), 'file.txt')
@@ -3426,7 +3426,7 @@ class ExportClass(object):
             """
             if value is None:
                 return '<NULL>'
-            elif isinstance(value, unicode):
+            elif isinstance(value, unicodeT):
                 return value.encode('utf8')
             elif isinstance(value, Reference):
                 return int(value)
@@ -3487,7 +3487,7 @@ class ExporterTSV(ExportClass):
             import codecs
             final.write(codecs.BOM_UTF16)
             writer.writerow(
-                [unicode(col).encode("utf8") for col in self.rows.colnames])
+                [to_unicode(col, "utf8") for col in self.rows.colnames])
             data = out.getvalue().decode("utf8")
             data = data.encode("utf-16")
             data = data[2:]

gluon/streamer.py

@@ -17,6 +17,7 @@ import re
 import errno
 from gluon.http import HTTP
 from gluon.contenttype import contenttype
+from gluon._compat import PY2
 
 
 regex_start_range = re.compile('\d+(?=\-)')
@@ -53,8 +54,11 @@ def stream_file_or_304_or_206(
     # if error_message is None:
     #     error_message = rewrite.THREAD_LOCAL.routes.error_message % 'invalid request'
     try:
-        open = file # this makes no sense but without it GAE cannot open files
-        fp = open(static_file,'rb')
+        if PY2:
+            open_f = file # this makes no sense but without it GAE cannot open files
+        else:
+            open_f = open
+        fp = open_f(static_file,'rb')
     except IOError as e:
         if e[0] == errno.EISDIR:
             raise HTTP(403, error_message, web2py_error='file is a directory')

gluon/template.py

@@ -263,7 +263,6 @@ class TemplateParser(object):
         # This will end up as
         # "%s(%s, escape=False)" % (self.writer, value)
         self.writer = writer
-
         # Dictionary of custom name lexers to use.
         if isinstance(lexers, dict):
             self.lexers = lexers
@@ -448,7 +447,7 @@ class TemplateParser(object):
             fileobj.close()
         except IOError:
             self._raise_error('Unable to open included view file: ' + filepath)
-
+        text = to_native(text)
         return text
 
     def include(self, content, filename):
@@ -788,7 +787,7 @@ def parse_template(filename,
             raise RestrictedError(filename, '', 'Unable to find the file')
     else:
         text = filename.read()
-
+    text = to_native(text)
     # Use the file contents to get a parsed template and return it.
     return str(TemplateParser(text, context=context, path=path, lexers=lexers, delimiters=delimiters))
 
@@ -885,7 +884,7 @@ def render(content="hello world",
     """
     # here to avoid circular Imports
     try:
-        from globals import Response
+        from gluon.globals import Response
     except ImportError:
         # Working standalone. Build a mock Response object.
         Response = DummyResponse

gluon/tests/__init__.py

@@ -14,15 +14,15 @@ from .test_contribs import *
 from .test_routes import *
 from .test_router import *
 from .test_validators import *
+from .test_tools import *
+from .test_utils import *
+from .test_serializers import *
+from .test_languages import *
+from .test_compileapp import *
+from .test_appadmin import *
 
 if sys.version[:3] == '2.7':
-    from .test_compileapp import *
     from .test_is_url import *
-    from .test_languages import *
-    from .test_serializers import *
-    from .test_utils import *
-    from .test_tools import *
-    from .test_appadmin import *
     from .test_scheduler import *
     from .test_web import *
     from .test_old_doctests import *

gluon/tests/test_languages.py

@@ -15,6 +15,7 @@ from .fix_path import fix_sys_path
 fix_sys_path(__file__)
 
 from gluon import languages
+from gluon._compat import PY2
 
 MP_WORKING = 0
 try:

gluon/tests/test_tools.py

@@ -547,7 +547,7 @@ class TestAuth(unittest.TestCase):
     def test_basic_blank_forms(self):
         for f in ['login', 'retrieve_password', 'retrieve_username', 'register']:
             html_form = getattr(self.auth, f)().xml()
-            self.assertTrue('name="_formkey"' in html_form)
+            self.assertTrue(b'name="_formkey"' in html_form)
 
         for f in ['logout', 'verify_email', 'reset_password', 'change_password', 'profile', 'groups']:
             self.assertRaisesRegexp(HTTP, "303*", getattr(self.auth, f))
@@ -712,7 +712,7 @@ class TestAuth(unittest.TestCase):
         self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         self.auth.settings.bulk_register_enabled = True
         bulk_register_form = self.auth.bulk_register(max_emails=10).xml()
-        self.assertTrue('name="_formkey"' in bulk_register_form)
+        self.assertTrue(b'name="_formkey"' in bulk_register_form)
 
     # TODO: def test_manage_tokens(self):
     # TODO: def test_reset_password(self):
@@ -723,12 +723,12 @@ class TestAuth(unittest.TestCase):
     def test_change_password(self):
         self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         change_password_form = getattr(self.auth, 'change_password')().xml()
-        self.assertTrue('name="_formkey"' in change_password_form)
+        self.assertTrue(b'name="_formkey"' in change_password_form)
 
     def test_profile(self):
         self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         profile_form = getattr(self.auth, 'profile')().xml()
-        self.assertTrue('name="_formkey"' in profile_form)
+        self.assertTrue(b'name="_formkey"' in profile_form)
 
     # TODO: def test_run_login_onaccept(self):
     # TODO: def test_jwt(self):
@@ -766,7 +766,7 @@ class TestAuth(unittest.TestCase):
 
         # basic impersonate() test that return a read form
         self.assertEqual(self.auth.impersonate().xml(),
-                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="no_table_user_id__row"><td class="w2p_fl"><label class="" for="no_table_user_id" id="no_table_user_id__label">User Id: </label></td><td class="w2p_fw"><input class="integer" id="no_table_user_id" name="user_id" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
+                         b'<form action="#" enctype="multipart/form-data" method="post"><table><tr id="no_table_user_id__row"><td class="w2p_fl"><label class="" for="no_table_user_id" id="no_table_user_id__label">User Id: </label></td><td class="w2p_fw"><input class="integer" id="no_table_user_id" name="user_id" type="text" value="" /></td><td class="w2p_fc"></td></tr><tr id="submit_record__row"><td class="w2p_fl"></td><td class="w2p_fw"><input type="submit" value="Submit" /></td><td class="w2p_fc"></td></tr></table></form>')
         # bart impersonate itself
         self.assertEqual(self.auth.impersonate(bart_id), None)
         self.assertFalse(self.auth.is_impersonating())  # User shouldn't impersonate itself?
@@ -776,7 +776,7 @@ class TestAuth(unittest.TestCase):
         self.assertTrue(self.auth.is_impersonating())
         self.assertEqual(self.auth.user_id, omer_id)  # we make it really sure
         self.assertEqual(impersonate_form.xml(),
-                         '<form action="#" enctype="multipart/form-data" method="post"><table><tr id="auth_user_id__row"><td class="w2p_fl"><label class="readonly" for="auth_user_id" id="auth_user_id__label">Id: </label></td><td class="w2p_fw"><span id="auth_user_id">2</span></td><td class="w2p_fc"></td></tr><tr id="auth_user_first_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_first_name" id="auth_user_first_name__label">First name: </label></td><td class="w2p_fw">Omer</td><td class="w2p_fc"></td></tr><tr id="auth_user_last_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_last_name" id="auth_user_last_name__label">Last name: </label></td><td class="w2p_fw">Simpson</td><td class="w2p_fc"></td></tr><tr id="auth_user_email__row"><td class="w2p_fl"><label class="readonly" for="auth_user_email" id="auth_user_email__label">E-mail: </label></td><td class="w2p_fw">omer@test.com</td><td class="w2p_fc"></td></tr><tr id="auth_user_username__row"><td class="w2p_fl"><label class="readonly" for="auth_user_username" id="auth_user_username__label">Username: </label></td><td class="w2p_fw">omer</td><td class="w2p_fc"></td></tr></table><div style="display:none;"><input name="id" type="hidden" value="2" /></div></form>')
+                         b'<form action="#" enctype="multipart/form-data" method="post"><table><tr id="auth_user_id__row"><td class="w2p_fl"><label class="readonly" for="auth_user_id" id="auth_user_id__label">Id: </label></td><td class="w2p_fw"><span id="auth_user_id">2</span></td><td class="w2p_fc"></td></tr><tr id="auth_user_first_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_first_name" id="auth_user_first_name__label">First name: </label></td><td class="w2p_fw">Omer</td><td class="w2p_fc"></td></tr><tr id="auth_user_last_name__row"><td class="w2p_fl"><label class="readonly" for="auth_user_last_name" id="auth_user_last_name__label">Last name: </label></td><td class="w2p_fw">Simpson</td><td class="w2p_fc"></td></tr><tr id="auth_user_email__row"><td class="w2p_fl"><label class="readonly" for="auth_user_email" id="auth_user_email__label">E-mail: </label></td><td class="w2p_fw">omer@test.com</td><td class="w2p_fc"></td></tr><tr id="auth_user_username__row"><td class="w2p_fl"><label class="readonly" for="auth_user_username" id="auth_user_username__label">Username: </label></td><td class="w2p_fw">omer</td><td class="w2p_fc"></td></tr></table><div style="display:none;"><input name="id" type="hidden" value="2" /></div></form>')
         self.auth.logout_bare()
         # Failing impersonation
         # User lacking impersonate membership
@@ -800,7 +800,7 @@ class TestAuth(unittest.TestCase):
     def test_groups(self):
         self.auth.login_user(self.db(self.db.auth_user.username == 'bart').select().first())  # bypass login_bare()
         self.assertEqual(self.auth.groups().xml(),
-                         '<table><tr><td><h3>user_1(1)</h3></td></tr><tr><td><p></p></td></tr></table>')
+                         b'<table><tr><td><h3>user_1(1)</h3></td></tr><tr><td><p></p></td></tr></table>')
 
     def test_not_authorized(self):
         self.current.request.ajax = 'facke_ajax_request'

gluon/tests/test_utils.py

@@ -12,7 +12,7 @@ from gluon.utils import md5_hash, compare, is_valid_ip_address, web2py_uuid
 
 import hashlib
 from hashlib import md5, sha1, sha224, sha256, sha384, sha512
-from gluon.utils import simple_hash, get_digest, secure_dumps, secure_loads
+from gluon.utils import simple_hash, get_digest, secure_dumps, secure_loads, basestring
 
 
 class TestUtils(unittest.TestCase):
@@ -94,8 +94,8 @@ class TestUtils(unittest.TestCase):
         secured = secure_dumps(testobj, testkey)
         original = secure_loads(secured, testkey)
         self.assertEqual(testobj, original)
-        self.assertTrue(isinstance(secured, basestring))
-        self.assertTrue(':' in secured)
+        self.assertTrue(isinstance(secured, bytes))
+        self.assertTrue(b':' in secured)
 
         large_testobj = [x for x in range(1000)]
         secured_comp = secure_dumps(large_testobj, testkey, compression_level=9)

gluon/tools.py

@@ -13,7 +13,8 @@ Auth, Mail, PluginManager and various utilities
 import base64
 from functools import reduce
 from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, configparser, MIMEBase, MIMEMultipart, \
-                          MIMEText, Encoders, Charset, long, urllib_quote, iteritems
+                          MIMEText, Encoders, Charset, long, urllib_quote, iteritems, to_bytes, to_native, add_charset, \
+                          charset_QP, basestring, unicodeT, to_unicode
 import datetime
 import logging
 import sys
@@ -241,6 +242,7 @@ class Mail(object):
                 if filename is None:
                     raise Exception('Missing attachment name')
                 payload = payload.read()
+            #FIXME PY3 can be used to_native?
             filename = filename.encode(encoding)
             if content_type is None:
                 content_type = contenttype(filename)
@@ -248,9 +250,9 @@ class Mail(object):
             self.my_payload = payload
             MIMEBase.__init__(self, *content_type.split('/', 1))
             self.set_payload(payload)
-            self['Content-Disposition'] = 'attachment; filename="%s"' % filename
+            self['Content-Disposition'] = 'attachment; filename="%s"' % to_native(filename, encoding)
             if content_id is not None:
-                self['Content-Id'] = '<%s>' % content_id.encode(encoding)
+                self['Content-Id'] = '<%s>' % to_native(content_id, encoding)
             Encoders.encode_base64(self)
 
     def __init__(self, server=None, sender=None, login=None, tls=True):
@@ -405,7 +407,7 @@ class Mail(object):
         """
 
         # We don't want to use base64 encoding for unicode mail
-        Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8')
+        add_charset('utf-8', charset_QP, charset_QP, 'utf-8')
 
         def encode_header(key):
             if [c for c in key if 32 > ord(c) or ord(c) > 127]:
@@ -428,12 +430,12 @@ class Mail(object):
 
         if not raw and attachments:
             # Use multipart/mixed if there is attachments
-            payload_in = MIMEMultipart.MIMEMultipart('mixed')
+            payload_in = MIMEMultipart('mixed')
         elif raw:
             # no encoding configuration for raw messages
             if not isinstance(message, basestring):
                 message = message.read()
-            if isinstance(message, unicode):
+            if isinstance(message, unicodeT):
                 text = message.encode('utf-8')
             elif not encoding == 'utf-8':
                 text = message.decode(encoding).encode('utf-8')
@@ -442,7 +444,7 @@ class Mail(object):
             # No charset passed to avoid transport encoding
             # NOTE: some unicode encoded strings will produce
             # unreadable mail contents.
-            payload_in = MIMEText.MIMEText(text)
+            payload_in = MIMEText(text)
         if to:
             if not isinstance(to, (list, tuple)):
                 to = [to]
@@ -471,14 +473,14 @@ class Mail(object):
             if text is not None:
                 if not isinstance(text, basestring):
                     text = text.read()
-                if isinstance(text, unicode):
+                if isinstance(text, unicodeT):
                     text = text.encode('utf-8')
                 elif not encoding == 'utf-8':
                     text = text.decode(encoding).encode('utf-8')
             if html is not None:
                 if not isinstance(html, basestring):
                     html = html.read()
-                if isinstance(html, unicode):
+                if isinstance(html, unicodeT):
                     html = html.encode('utf-8')
                 elif not encoding == 'utf-8':
                     html = html.decode(encoding).encode('utf-8')
@@ -486,15 +488,13 @@ class Mail(object):
             # Construct mime part only if needed
             if text is not None and html:
                 # We have text and html we need multipart/alternative
-                attachment = MIMEMultipart.MIMEMultipart('alternative')
-                attachment.attach(MIMEText.MIMEText(text, _charset='utf-8'))
-                attachment.attach(
-                    MIMEText.MIMEText(html, 'html', _charset='utf-8'))
+                attachment = MIMEMultipart('alternative')
+                attachment.attach(MIMEText(text, _charset='utf-8'))
+                attachment.attach(MIMEText(html, 'html', _charset='utf-8'))
             elif text is not None:
-                attachment = MIMEText.MIMEText(text, _charset='utf-8')
+                attachment = MIMEText(text, _charset='utf-8')
             elif html:
-                attachment = \
-                    MIMEText.MIMEText(html, 'html', _charset='utf-8')
+                attachment = MIMEText(html, 'html', _charset='utf-8')
 
             if attachments:
                 # If there is attachments put text and html into
@@ -560,12 +560,11 @@ class Mail(object):
                     c.op_sign(plain, sig, mode.DETACH)
                     sig.seek(0, 0)
                     # make it part of the email
-                    payload = \
-                        MIMEMultipart.MIMEMultipart('signed',
-                                                    boundary=None,
-                                                    _subparts=None,
-                                                    **dict(micalg="pgp-sha1",
-                                                           protocol="application/pgp-signature"))
+                    payload = MIMEMultipart('signed',
+                                            boundary=None,
+                                            _subparts=None,
+                                            **dict(micalg="pgp-sha1",
+                                                   protocol="application/pgp-signature"))
                     # insert the origin payload
                     payload.attach(payload_in)
                     # insert the detached signature
@@ -605,10 +604,10 @@ class Mail(object):
                     c.op_encrypt(recipients, 1, plain, cipher)
                     cipher.seek(0, 0)
                     # make it a part of the email
-                    payload = MIMEMultipart.MIMEMultipart('encrypted',
-                                                          boundary=None,
-                                                          _subparts=None,
-                                                          **dict(protocol="application/pgp-encrypted"))
+                    payload = MIMEMultipart('encrypted',
+                                            boundary=None,
+                                            _subparts=None,
+                                            **dict(protocol="application/pgp-encrypted"))
                     p = MIMEBase("application", 'pgp-encrypted')
                     p.set_payload("Version: 1\r\n")
                     payload.attach(p)
@@ -729,29 +728,29 @@ class Mail(object):
             payload = payload_in
 
         if from_address:
-            payload['From'] = encoded_or_raw(from_address.decode(encoding))
+            payload['From'] = encoded_or_raw(to_unicode(from_address, encoding))
         else:
-            payload['From'] = encoded_or_raw(sender.decode(encoding))
+            payload['From'] = encoded_or_raw(to_unicode(sender, encoding))
         origTo = to[:]
         if to:
-            payload['To'] = encoded_or_raw(', '.join(to).decode(encoding))
+            payload['To'] = encoded_or_raw(to_unicode(', '.join(to), encoding))
         if reply_to:
-            payload['Reply-To'] = encoded_or_raw(reply_to.decode(encoding))
+            payload['Reply-To'] = encoded_or_raw(to_unicode(reply_to, encoding))
         if cc:
-            payload['Cc'] = encoded_or_raw(', '.join(cc).decode(encoding))
+            payload['Cc'] = encoded_or_raw(to_unicode(', '.join(cc), encoding))
             to.extend(cc)
         if bcc:
             to.extend(bcc)
-        payload['Subject'] = encoded_or_raw(subject.decode(encoding))
+        payload['Subject'] = encoded_or_raw(to_unicode(subject, encoding))
         payload['Date'] = email.utils.formatdate()
         for k, v in iteritems(headers):
-            payload[k] = encoded_or_raw(v.decode(encoding))
+            payload[k] = encoded_or_raw(to_unicode(v, encoding))
         result = {}
         try:
             if self.settings.server == 'logging':
                 entry = 'email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' % \
                     ('-' * 40, sender, ', '.join(to), subject, text or html, '-' * 40)
-                logger.warn(entry)
+                logger.warning(entry)
             elif self.settings.server.startswith('logging:'):
                 entry = 'email not sent\n%s\nFrom: %s\nTo: %s\nSubject: %s\n\n%s\n%s\n' % \
                     ('-' * 40, sender, ', '.join(to), subject, text or html, '-' * 40)
@@ -773,16 +772,16 @@ class Mail(object):
                 if attachments:
                     result = mail.send_mail(
                         sender=sender, to=origTo,
-                        subject=unicode(subject, encoding), body=unicode(text, encoding), html=html,
+                        subject=to_unicode(subject, encoding), body=to_unicode(text, encoding), html=html,
                         attachments=attachments, **xcc)
                 elif html and (not raw):
                     result = mail.send_mail(
                         sender=sender, to=origTo,
-                        subject=unicode(subject, encoding), body=unicode(text, encoding), html=html, **xcc)
+                        subject=to_unicode(subject, encoding), body=to_unicode(text, encoding), html=html, **xcc)
                 else:
                     result = mail.send_mail(
                         sender=sender, to=origTo,
-                        subject=unicode(subject, encoding), body=unicode(text, encoding), **xcc)
+                        subject=to_unicode(subject, encoding), body=to_unicode(text, encoding), **xcc)
             else:
                 smtp_args = self.settings.server.split(':')
                 kwargs = dict(timeout=self.settings.timeout)
@@ -800,7 +799,7 @@ class Mail(object):
                     sender, to, payload.as_string())
                 server.quit()
         except Exception as e:
-            logger.warn('Mail.send failure:%s' % e)
+            logger.warning('Mail.send failure:%s' % e)
             self.result = result
             self.error = e
             return False
@@ -1250,8 +1249,7 @@ class AuthJWT(object):
         
     @staticmethod
     def jwt_b64e(string):
-        if isinstance(string, unicode):
-            string = string.encode('utf-8', 'strict')
+        string = to_bytes(string)
         return base64.urlsafe_b64encode(string).strip(b'=')
 
     @staticmethod
@@ -1260,47 +1258,44 @@ class AuthJWT(object):
         called with a unicode string).
         The result is also a bytestring.
         """
-        if isinstance(string, unicode):
-            string = string.encode('ascii', 'ignore')
-        return base64.urlsafe_b64decode(string + '=' * (-len(string) % 4))
+        string = to_bytes(string, 'ascii', 'ignore')
+        return base64.urlsafe_b64decode(string + b'=' * (-len(string) % 4))
 
     def generate_token(self, payload):
-        secret = self.secret_key
+        secret = to_bytes(self.secret_key)
         if self.salt:
             if callable(self.salt):
                 secret = "%s$%s" % (secret, self.salt(payload))
             else:
                 secret = "%s$%s" % (secret, self.salt)
-            if isinstance(secret, unicode):
+            if isinstance(secret, unicodeT):
                 secret = secret.encode('ascii', 'ignore')
         b64h = self.cached_b64h
         b64p = self.jwt_b64e(serializers.json(payload))
-        jbody = b64h + '.' + b64p
+        jbody = b64h + b'.' + b64p
         mauth = hmac.new(key=secret, msg=jbody, digestmod=self.digestmod)
         jsign = self.jwt_b64e(mauth.digest())
-        return jbody + '.' + jsign
+        return to_native(jbody + b'.' + jsign)
 
     def verify_signature(self, body, signature, secret):
         mauth = hmac.new(key=secret, msg=body, digestmod=self.digestmod)
         return compare(self.jwt_b64e(mauth.digest()), signature)
 
     def load_token(self, token):
-        if isinstance(token, unicode):
-            token = token.encode('utf-8', 'strict')
-        body, sig = token.rsplit('.', 1)
-        b64h, b64b = body.split('.', 1)
+        token = to_bytes(token, 'utf-8', 'strict')
+        body, sig = token.rsplit(b'.', 1)
+        b64h, b64b = body.split(b'.', 1)
         if b64h != self.cached_b64h:
             # header not the same
             raise HTTP(400, u'Invalid JWT Header')
         secret = self.secret_key
-        tokend = serializers.loads_json(self.jwt_b64d(b64b))
+        tokend = serializers.loads_json(to_native(self.jwt_b64d(b64b)))
         if self.salt:
             if callable(self.salt):
                 secret = "%s$%s" % (secret, self.salt(tokend))
             else:
                 secret = "%s$%s" % (secret, self.salt)
-            if isinstance(secret, unicode):
-                secret = secret.encode('ascii', 'ignore')
+        secret = to_bytes(secret, 'ascii', 'ignore')
         if not self.verify_signature(body, sig, secret):
             # signature verification failed
             raise HTTP(400, u'Token signature is invalid')
@@ -1462,7 +1457,7 @@ class AuthJWT(object):
             def f(*args, **kwargs):
                 try:
                     token = self.get_jwt_token_from_request(token_param=token_param)
-                except HTTP, e:
+                except HTTP as e:
                     if required:
                         raise e
                     token = None
@@ -3560,8 +3555,8 @@ class Auth(object):
         password = ''
         specials = r'!#$*'
         for i in range(0, 3):
-            password += random.choice(string.lowercase)
-            password += random.choice(string.uppercase)
+            password += random.choice(string.ascii_lowercase)
+            password += random.choice(string.ascii_uppercase)
             password += random.choice(string.digits)
             password += random.choice(specials)
         return ''.join(random.sample(password, len(password)))
@@ -3994,7 +3989,7 @@ class Auth(object):
         requires = table_user[passfield].requires
         if not isinstance(requires, (list, tuple)):
             requires = [requires]
-        requires = filter(lambda t: isinstance(t, CRYPT), requires)
+        requires = list(filter(lambda t: isinstance(t, CRYPT), requires))
         if requires:
             requires[0].min_length = 0
         form = SQLFORM.factory(
@@ -4764,7 +4759,7 @@ class Auth(object):
             self._wiki.automenu()
 
 
-class Crud(object):
+class Crud(object): # pragma: no cover
 
     def url(self, f=None, args=None, vars=None):
         """
@@ -5621,7 +5616,7 @@ class Service(object):
             args = request.args
 
         def none_exception(value):
-            if isinstance(value, unicode):
+            if isinstance(value, unicodeT):
                 return value.encode('utf8')
             if hasattr(value, 'isoformat'):
                 return value.isoformat()[:19].replace('T', ' ')

gluon/utf8.py

@@ -11,7 +11,7 @@ Utilities and class for UTF8 strings managing
 ----------------------------------------------
 """
 from __future__ import print_function
-from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode
+from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native
 
 __all__ = ['Utf8']
 
@@ -51,10 +51,10 @@ def sort_key(s):
         from gluon.contrib.pyuca import unicode_collator
         unicode_sort_key = unicode_collator.sort_key
         sort_key = lambda s: unicode_sort_key(
-            unicode(s, 'utf-8') if isinstance(s, str) else s)
+            to_unicode(s, 'utf-8') if isinstance(s, str) else s)
     except:
         sort_key = lambda s: (
-            unicode(s, 'utf-8') if isinstance(s, str) else s).lower()
+            to_unicode(s, 'utf-8') if isinstance(s, str) else s).lower()
     return sort_key(s)
 
 
@@ -64,7 +64,7 @@ def ord(char):
     """
     if isinstance(char, unicodeT):
         return __builtin__.ord(char)
-    return __builtin__.ord(unicode(char, 'utf-8'))
+    return __builtin__.ord(to_unicode(char, 'utf-8'))
 
 
 def chr(code):
@@ -92,8 +92,8 @@ def truncate(string, length, dots='...'):
     Returns:
         (utf8-str): original or cutted string
     """
-    text = unicode(string, 'utf-8')
-    dots = unicode(dots, 'utf-8') if isinstance(dots, str) else dots
+    text = to_unicode(string, 'utf-8')
+    dots = to_unicode(dots, 'utf-8') if isinstance(dots, str) else dots
     if len(text) > length:
         text = text[:length - len(dots)] + dots
     return str.__new__(Utf8, text.encode('utf-8'))
@@ -120,11 +120,11 @@ class Utf8(str):
     """
     def __new__(cls, content='', codepage='utf-8'):
         if isinstance(content, unicodeT):
-            return str.__new__(cls, unicode.encode(content, 'utf-8'))
+            return str.__new__(cls, to_native(content, 'utf-8'))
         elif codepage in ('utf-8', 'utf8') or isinstance(content, cls):
             return str.__new__(cls, content)
         else:
-            return str.__new__(cls, unicode(content, codepage).encode('utf-8'))
+            return str.__new__(cls, to_native(to_unicode(content, codepage), 'utf-8'))
 
     def __repr__(self):
         r''' # note that we use raw strings to avoid having to use double back slashes below
@@ -156,9 +156,9 @@ class Utf8(str):
             True
         '''
         if str.find(self, "'") >= 0 and str.find(self, '"') < 0:  # only single quote exists
-            return '"' + unicode(self, 'utf-8').translate(repr_escape_tab).encode('utf-8') + '"'
+            return '"' + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab), 'utf-8') + '"'
         else:
-            return "'" + unicode(self, 'utf-8').translate(repr_escape_tab2).encode('utf-8') + "'"
+            return "'" + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab2), 'utf-8') + "'"
 
     def __size__(self):
         """ length of utf-8 string in bytes """
@@ -168,17 +168,17 @@ class Utf8(str):
         return str.__contains__(self, Utf8(other))
 
     def __getitem__(self, index):
-        return str.__new__(Utf8, unicode(self, 'utf-8')[index].encode('utf-8'))
+        return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))
 
     def __getslice__(self, begin, end):
-        return str.__new__(Utf8, unicode(self, 'utf-8')[begin:end].encode('utf-8'))
+        return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
 
     def __add__(self, other):
         return str.__new__(Utf8, str.__add__(self, unicode.encode(other, 'utf-8')
                                              if isinstance(other, unicode) else other))
 
     def __len__(self):
-        return len(unicode(self, 'utf-8'))
+        return len(to_unicode(self, 'utf-8'))
 
     def __mul__(self, integer):
         return str.__new__(Utf8, str.__mul__(self, integer))

gluon/utils.py

@@ -85,7 +85,7 @@ def compare(a, b):
 
 def md5_hash(text):
     """ Generates a md5 hash with the given text """
-    return md5(text).hexdigest()
+    return md5(to_bytes(text)).hexdigest()
 
 
 def simple_hash(text, key='', salt='', digest_alg='md5'):
@@ -157,11 +157,12 @@ def get_callable_argspec(fn):
     return inspect.getargspec(inspectable)
 
 
-def pad(s, n=32, padchar=' '):
+def pad(s, n=32, padchar=b' '):
     return s + (32 - len(s) % 32) * padchar
 
 
 def secure_dumps(data, encryption_key, hash_key=None, compression_level=None):
+    encryption_key = to_bytes(encryption_key)
     if not hash_key:
         hash_key = sha1(encryption_key).hexdigest()
     dump = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
@@ -170,17 +171,20 @@ def secure_dumps(data, encryption_key, hash_key=None, compression_level=None):
     key = pad(encryption_key)[:32]
     cipher, IV = AES_new(key)
     encrypted_data = base64.urlsafe_b64encode(IV + cipher.encrypt(pad(dump)))
-    signature = hmac.new(hash_key, encrypted_data).hexdigest()
-    return signature + ':' + encrypted_data
+    signature = to_bytes(hmac.new(to_bytes(hash_key), encrypted_data).hexdigest())
+    return signature + b':' + encrypted_data
 
 
 def secure_loads(data, encryption_key, hash_key=None, compression_level=None):
+    encryption_key = to_bytes(encryption_key)
+    data = to_native(data)
     if ':' not in data:
         return None
     if not hash_key:
         hash_key = sha1(encryption_key).hexdigest()
     signature, encrypted_data = data.split(':', 1)
-    actual_signature = hmac.new(hash_key, encrypted_data).hexdigest()
+    encrypted_data = to_bytes(encrypted_data)
+    actual_signature = hmac.new(to_bytes(hash_key), encrypted_data).hexdigest()
     if not compare(signature, actual_signature):
         return None
     key = pad(encryption_key)[:32]
@@ -189,7 +193,7 @@ def secure_loads(data, encryption_key, hash_key=None, compression_level=None):
     cipher, _ = AES_new(key, IV=IV)
     try:
         data = cipher.decrypt(encrypted_data)
-        data = data.rstrip(' ')
+        data = data.rstrip(b' ')
         if compression_level:
             data = zlib.decompress(data)
         return pickle.loads(data)