From f434ebec8a0c399944e039d8860c2d33ff9d3a32 Mon Sep 17 00:00:00 2001 From: Vinyl Darkscratch Date: Sat, 9 Feb 2019 00:01:11 -0800 Subject: [PATCH 1/4] Update Python 3 compatibility --- gluon/admin.py | 2 +- gluon/authapi.py | 2 +- gluon/cache.py | 2 +- gluon/compileapp.py | 8 ++--- gluon/contrib/markdown/__init__.py | 3 +- gluon/decoder.py | 4 +-- gluon/highlight.py | 4 +-- gluon/html.py | 5 ++- gluon/languages.py | 4 +-- gluon/main.py | 13 ++++--- gluon/messageboxhandler.py | 18 ++++++---- gluon/newcron.py | 6 +--- gluon/rewrite.py | 1 - gluon/rocket.py | 47 +++++--------------------- gluon/rocket.py.footer | 1 - gluon/scheduler.py | 20 +++++------ gluon/serializers.py | 4 +-- gluon/shell.py | 3 +- gluon/sqlhtml.py | 16 ++++----- gluon/tests/test_appadmin.py | 2 +- gluon/tests/test_dal.py | 2 +- gluon/tests/test_globals.py | 4 +-- gluon/tests/test_router.py | 2 +- gluon/tests/test_scheduler.py | 4 +-- gluon/tests/test_web.py | 2 +- gluon/tools.py | 54 +++++++++++++++--------------- gluon/validators.py | 9 +++-- gluon/widget.py | 6 ++-- scripts/update_languages.py | 16 ++++----- 29 files changed, 112 insertions(+), 152 deletions(-) diff --git a/gluon/admin.py b/gluon/admin.py index ad5e18d7..cb756721 100644 --- a/gluon/admin.py +++ b/gluon/admin.py @@ -9,7 +9,7 @@ Utility functions for the Admin application ------------------------------------------- """ -from __future__ import print_function + import os import sys import traceback diff --git a/gluon/authapi.py b/gluon/authapi.py index 887ae724..220ba437 100644 --- a/gluon/authapi.py +++ b/gluon/authapi.py @@ -988,7 +988,7 @@ class AuthAPI(object): requires = table_user[passfield].requires if not isinstance(requires, (list, tuple)): requires = [requires] - requires = list(filter(lambda t: isinstance(t, CRYPT), requires)) + requires = [t for t in requires if isinstance(t, CRYPT)] if requires: requires[0] = CRYPT(**requires[0].__dict__) # Copy the existing CRYPT attributes requires[0].min_length = 0 # But do not enforce minimum length for the old password diff --git a/gluon/cache.py b/gluon/cache.py index 5050d213..90939b60 100644 --- a/gluon/cache.py +++ b/gluon/cache.py @@ -619,7 +619,7 @@ class Cache(object): if user_agent_ is True: cache_key.append("%(is_mobile)s_%(is_tablet)s" % current.request.user_agent()) else: - cache_key.append(str(user_agent_.items())) + cache_key.append(str(list(user_agent_.items()))) if vars_: cache_key.append(current.request.env.query_string) if lang_: diff --git a/gluon/compileapp.py b/gluon/compileapp.py index dd7aca00..87f2f6fa 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -18,7 +18,7 @@ import fnmatch import os, sys import copy import random -from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long, reload +from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, integer_types, basestring, reduce, xrange, long, reload from gluon.storage import Storage, List from gluon.template import parse_template from gluon.restricted import restricted, compile2 @@ -177,7 +177,7 @@ def LOAD(c=None, f='index', args=None, vars=None, else: raise TypeError("Unsupported times argument type %s" % type(times)) if timeout is not None: - if not isinstance(timeout, (int, long)): + if not isinstance(timeout, integer_types): raise ValueError("Timeout argument must be an integer or None") elif timeout <= 0: raise ValueError( @@ -261,7 +261,7 @@ class LoadFactory(object): if args is None: args = [] vars = Storage(vars or {}) - import globals + from . import globals target = target or 'c' + str(random.random())[2:] attr['_id'] = target request = current.request @@ -631,7 +631,7 @@ def run_controller_in(controller, function, environment): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) - environment['__symbols__'] = environment.keys() + environment['__symbols__'] = list(environment.keys()) code = read_file(filename) code += TEST_CODE ccode = compile2(code, filename) diff --git a/gluon/contrib/markdown/__init__.py b/gluon/contrib/markdown/__init__.py index 4378de21..358af631 100644 --- a/gluon/contrib/markdown/__init__.py +++ b/gluon/contrib/markdown/__init__.py @@ -1,5 +1,6 @@ from .markdown2 import * from gluon.html import XML +from gluon._compat import to_unicode def WIKI(text, encoding="utf8", safe_mode='escape', html4tags=False, **attributes): if not text: @@ -9,7 +10,7 @@ def WIKI(text, encoding="utf8", safe_mode='escape', html4tags=False, **attribute del attributes['extras'] else: extras=None - text = text.decode(encoding,'replace') + text = to_unicode(text, encoding, 'replace') return XML(markdown(text,extras=extras, safe_mode=safe_mode, html4tags=html4tags)\ diff --git a/gluon/decoder.py b/gluon/decoder.py index 57044bc6..f93a29c4 100644 --- a/gluon/decoder.py +++ b/gluon/decoder.py @@ -57,8 +57,8 @@ def autoDetectXMLEncoding(buffer): secret_decoder_ring = codecs.lookup(encoding)[1] (decoded, length) = secret_decoder_ring(buffer) first_line = decoded.split("\n")[0] - if first_line and first_line.startswith(u" | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) """ -from __future__ import print_function + from gluon._compat import xrange from gluon.utils import local_html_escape import re @@ -325,7 +325,7 @@ color: #A0A0A0; fa = ' '.join([key[1:].lower() for (key, value) in items if key[:1] == '_' and value is None] + ['%s="%s"' % (key[1:].lower(), str(value).replace('"', "'")) - for (key, value) in attributes.items() if key[:1] + for (key, value) in items if key[:1] == '_' and value]) if fa: fa = ' ' + fa diff --git a/gluon/html.py b/gluon/html.py index 4adf287c..6fde94d9 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -9,7 +9,6 @@ Template helpers -------------------------------------------- """ -from __future__ import print_function import cgi import os @@ -35,8 +34,8 @@ regex_crlf = re.compile('\r|\n') join = ''.join # name2codepoint is incomplete respect to xhtml (and xml): 'apos' is missing. -entitydefs = dict(map(lambda k_v: (k_v[0], unichr(k_v[1]).encode('utf-8')), iteritems(name2codepoint))) -entitydefs.setdefault('apos', u"'".encode('utf-8')) +entitydefs = dict([(k_v[0], to_bytes(unichr(k_v[1]))) for k_v in iteritems(name2codepoint)]) +entitydefs.setdefault('apos', to_bytes("'")) __all__ = [ diff --git a/gluon/languages.py b/gluon/languages.py index 2eac092e..c8c8ed1e 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -326,9 +326,7 @@ def write_plural_dict(filename, contents): def sort_function(x): - if sys.version_info.major == 3: # python 3 compatibility - unicode = str - return unicode(x, 'utf-8').lower() + return to_unicode(x, 'utf-8').lower() def write_dict(filename, contents): diff --git a/gluon/main.py b/gluon/main.py index e634de37..c138ed71 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -9,10 +9,9 @@ The gluon wsgi application --------------------------- """ -from __future__ import print_function if False: - import import_all # DO NOT REMOVE PART OF FREEZE PROCESS + from . import import_all # DO NOT REMOVE PART OF FREEZE PROCESS import gc import os @@ -249,7 +248,7 @@ class LazyWSGI(object): def app(environ, start_response): data = f() start_response(self.response.status, - self.response.headers.items()) + list(self.response.headers.items())) if isinstance(data, list): return data return [data] @@ -486,10 +485,10 @@ def wsgibase(environ, responder): if request.ajax: if response.flash: http_response.headers['web2py-component-flash'] = \ - urllib2.quote(xmlescape(response.flash).replace(b'\n', b'')) + urllib_quote(xmlescape(response.flash).replace(b'\n', b'')) if response.js: http_response.headers['web2py-component-command'] = \ - urllib2.quote(response.js.replace('\n', '')) + urllib_quote(response.js.replace('\n', '')) # ################################################## # store cookies in headers @@ -717,9 +716,9 @@ class HttpServer(object): if isinstance(interfaces, list): for i in interfaces: if not isinstance(i, tuple): - raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/" + raise AttributeError("Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/") else: - raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/" + raise AttributeError("Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/") if path: # if a path is specified change the global variables so that web2py diff --git a/gluon/messageboxhandler.py b/gluon/messageboxhandler.py index 50aa68cd..6808cb54 100644 --- a/gluon/messageboxhandler.py +++ b/gluon/messageboxhandler.py @@ -1,9 +1,13 @@ import logging import os +import sys try: - import Tkinter -except: + if sys.version_info[0] == 2: + import Tkinter as tkinter + else: + import tkinter +except ImportError: Tkinter = None @@ -12,15 +16,15 @@ class MessageBoxHandler(logging.Handler): logging.Handler.__init__(self) def emit(self, record): - if Tkinter: + if tkinter: msg = self.format(record) - root = Tkinter.Tk() + root = tkinter.Tk() root.wm_title("web2py logger message") - text = Tkinter.Text() + text = tkinter.Text() text["height"] = 12 text.insert(0.1, msg) text.pack() - button = Tkinter.Button(root, text="OK", command=root.destroy) + button = tkinter.Button(root, text="OK", command=root.destroy) button.pack() root.mainloop() @@ -30,6 +34,6 @@ class NotifySendHandler(logging.Handler): logging.Handler.__init__(self) def emit(self, record): - if Tkinter: + if tkinter: msg = self.format(record) os.system("notify-send '%s'" % msg) diff --git a/gluon/newcron.py b/gluon/newcron.py index d53ecbe5..c5a0119f 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -20,13 +20,9 @@ import re import datetime import platform from functools import reduce -try: - import cPickle as pickle -except: - import pickle from gluon.settings import global_settings from gluon import fileutils -from gluon._compat import to_bytes +from gluon._compat import to_bytes, pickle from pydal.contrib import portalocker logger = logging.getLogger("web2py.cron") diff --git a/gluon/rewrite.py b/gluon/rewrite.py index 31e1bd16..526e2a10 100644 --- a/gluon/rewrite.py +++ b/gluon/rewrite.py @@ -15,7 +15,6 @@ routes.py supports two styles of URL rewriting, depending on whether 'routers' i Refer to router.example.py and routes.example.py for additional documentation. """ -from __future__ import print_function import os import re diff --git a/gluon/rocket.py b/gluon/rocket.py index d76da94e..69192347 100644 --- a/gluon/rocket.py +++ b/gluon/rocket.py @@ -5,14 +5,14 @@ # Modified by Massimo Di Pierro # Import System Modules -from __future__ import print_function + import sys import errno import socket import logging import platform -from gluon._compat import iteritems, to_bytes, StringIO -from gluon._compat import urllib_unquote, to_native +from gluon._compat import iteritems, to_bytes, to_unicode, StringIO +from gluon._compat import urllib_unquote, to_native, PY2 # Define Constants VERSION = '1.2.6' @@ -32,7 +32,7 @@ DEFAULTS = dict(LISTEN_QUEUE_SIZE=DEFAULT_LISTEN_QUEUE_SIZE, MIN_THREADS=DEFAULT_MIN_THREADS, MAX_THREADS=DEFAULT_MAX_THREADS) -PY3K = sys.version_info[0] > 2 +PY3K = not PY2 class NullHandler(logging.Handler): @@ -40,39 +40,8 @@ class NullHandler(logging.Handler): def emit(self, record): pass -if PY3K: - def b(val): - """ Convert string/unicode/bytes literals into bytes. This allows for - the same code to run on Python 2.x and 3.x. """ - if isinstance(val, str): - return val.encode() - else: - return val - - def u(val, encoding="us-ascii"): - """ Convert bytes into string/unicode. This allows for the - same code to run on Python 2.x and 3.x. """ - if isinstance(val, bytes): - return val.decode(encoding) - else: - return val - -else: - def b(val): - """ Convert string/unicode/bytes literals into bytes. This allows for - the same code to run on Python 2.x and 3.x. """ - if isinstance(val, unicode): - return val.encode() - else: - return val - - def u(val, encoding="us-ascii"): - """ Convert bytes into string/unicode. This allows for the - same code to run on Python 2.x and 3.x. """ - if isinstance(val, str): - return val.decode(encoding) - else: - return val +b = to_bytes +u = to_unicode # Import Package Modules # package imports removed in monolithic build @@ -613,9 +582,9 @@ import socket import logging import traceback from threading import Lock -try: +if PY3K: from queue import Queue -except ImportError: +else: from Queue import Queue # Import Package Modules diff --git a/gluon/rocket.py.footer b/gluon/rocket.py.footer index 7d76ea4b..91ba75e5 100644 --- a/gluon/rocket.py.footer +++ b/gluon/rocket.py.footer @@ -1,4 +1,3 @@ -from __future__ import print_function # The following code is not part of Rocket but was added to # web2py for testing purposes. diff --git a/gluon/scheduler.py b/gluon/scheduler.py index 2343ffe6..fa7cdc0a 100644 --- a/gluon/scheduler.py +++ b/gluon/scheduler.py @@ -8,7 +8,6 @@ Background processes made simple --------------------------------- """ -from __future__ import print_function import os import re @@ -29,7 +28,7 @@ from json import loads, dumps from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB, IS_EMPTY_OR from gluon import IS_INT_IN_RANGE, IS_DATETIME, IS_IN_DB from gluon.utils import web2py_uuid -from gluon._compat import Queue, long, iteritems, PY2 +from gluon._compat import Queue, long, iteritems, PY2, to_bytes, string_types, integer_types from gluon.storage import Storage USAGE = """ @@ -417,8 +416,8 @@ def _decode_list(lst): return lst newlist = [] for i in lst: - if isinstance(i, unicode): - i = i.encode('utf-8') + if isinstance(i, string_types): + i = to_bytes(i) elif isinstance(i, list): i = _decode_list(i) newlist.append(i) @@ -430,10 +429,9 @@ def _decode_dict(dct): return dct newdict = {} for k, v in iteritems(dct): - if isinstance(k, unicode): - k = k.encode('utf-8') - if isinstance(v, unicode): - v = v.encode('utf-8') + k = to_bytes(k) + if isinstance(v, string_types): + v = to_bytes(v) elif isinstance(v, list): v = _decode_list(v) newdict[k] = v @@ -1572,7 +1570,7 @@ class Scheduler(MetaScheduler): """ from pydal.objects import Query sr, st = self.db.scheduler_run, self.db.scheduler_task - if isinstance(ref, (int, long)): + if isinstance(ref, integer_types): q = st.id == ref elif isinstance(ref, str): q = st.uuid == ref @@ -1623,7 +1621,7 @@ class Scheduler(MetaScheduler): Experimental """ st, sw = self.db.scheduler_task, self.db.scheduler_worker - if isinstance(ref, (int, long)): + if isinstance(ref, integer_types): q = st.id == ref elif isinstance(ref, str): q = st.uuid == ref @@ -1723,7 +1721,7 @@ def main(): sys.path.append(path) print('importing tasks...') tasks = __import__(filename, globals(), locals(), [], -1).tasks - print('tasks found: ' + ', '.join(tasks.keys())) + print('tasks found: ' + ', '.join(list(tasks.keys()))) else: tasks = {} group_names = [x.strip() for x in options.group_names.split(',')] diff --git a/gluon/serializers.py b/gluon/serializers.py index 032cff81..440b8657 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -10,7 +10,7 @@ from gluon.html import TAG, XmlComponent, xmlescape from gluon.languages import lazyT import gluon.contrib.rss2 as rss2 import json as json_parser -from gluon._compat import long, to_native, unicodeT +from gluon._compat import long, to_native, unicodeT, integer_types have_yaml = True try: @@ -79,7 +79,7 @@ def custom_json(o): datetime.datetime, datetime.time)): return o.isoformat()[:19].replace('T', ' ') - elif isinstance(o, (int, long)): + elif isinstance(o, integer_types): return int(o) elif isinstance(o, decimal.Decimal): return str(o) diff --git a/gluon/shell.py b/gluon/shell.py index eb4df8fe..c49e2cb4 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -10,6 +10,7 @@ Web2py environment in the shell -------------------------------- """ + from __future__ import print_function import os @@ -95,7 +96,7 @@ def exec_environment( if pyfile: pycfile = pyfile + 'c' if os.path.isfile(pycfile): - exec (read_pyc(pycfile), env) + exec(read_pyc(pycfile), env) else: execfile(pyfile, env) return Storage(env) diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 7dbff9fa..ed0a8cfc 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -717,7 +717,7 @@ class AutocompleteWidget(object): compact=table_rows.compact) elif settings and settings.global_settings.web2py_runtime_gae: rows = self.db(field.__ge__(kword) & - field.__lt__(kword + u'\ufffd') + field.__lt__(kword + '\ufffd') ).select(orderby=self.orderby, limitby=self.limitby, *(self.fields + self.help_fields)) @@ -1966,7 +1966,7 @@ class SQLFORM(FORM): AUTOTYPES = { type(''): ('string', None), - type(u''): ('string',None), + type(''): ('string',None), type(True): ('boolean', None), type(1): ('integer', IS_INT_IN_RANGE(-1e12, +1e12)), type(1.0): ('double', IS_FLOAT_IN_RANGE()), @@ -2462,8 +2462,8 @@ class SQLFORM(FORM): filter1 = lambda f: isinstance(f, Field) and (f.type!='blob' or showblobs) filter2 = lambda f: isinstance(f, Field) and f.readable and f.listable for table in tables: - fields += filter(filter1, table) - columns += filter(filter2, table) + fields += list(filter(filter1, table)) + columns += list(filter(filter2, table)) for k, f in iteritems(table): if not k.startswith('_'): if isinstance(f, Field.Virtual) and f.readable: @@ -2549,7 +2549,7 @@ class SQLFORM(FORM): table = db[request.args[-2]] record = table(request.args[-1]) or redirect(referrer) if represent_none is not None: - for field in record.iterkeys(): + for field in record.keys(): if record[field] is None: record[field] = represent_none sqlformargs = dict(upload=upload, ignore_rw=ignore_rw, @@ -2676,7 +2676,7 @@ class SQLFORM(FORM): # the query should be constructed using searchable # fields but not virtual fields is_searchable = lambda f: f.readable and not isinstance(f, Field.Virtual) and f.searchable - sfields = reduce(lambda a, b: a + b, [filter(is_searchable, t) for t in tables]) + sfields = reduce(lambda a, b: a + b, [list(filter(is_searchable, t)) for t in tables]) # use custom_query using searchable if callable(searchable): dbset = dbset(searchable(sfields, keywords)) @@ -2937,7 +2937,7 @@ class SQLFORM(FORM): paginator.append(LI(self_link('<<', 0))) if page > NPAGES: paginator.append(LI(self_link('<', page - 1))) - pages = range(max(0, page - NPAGES), min(page + NPAGES, npages)) + pages = list(range(max(0, page - NPAGES), min(page + NPAGES, npages))) for p in pages: if p == page: paginator.append(LI(A(p + 1, _onclick='return false'), @@ -3426,7 +3426,7 @@ class SQLTABLE(TABLE): if not sqlrows: return REGEX_TABLE_DOT_FIELD = sqlrows.db._adapter.REGEX_TABLE_DOT_FIELD - fieldmap = dict(zip(sqlrows.colnames, sqlrows.fields)) + fieldmap = dict(list(zip(sqlrows.colnames, sqlrows.fields))) tablemap = dict(((f.tablename, f.table) if isinstance(f, Field) else (f._table._tablename, f._table) for f in fieldmap.values())) for table in tablemap.values(): pref = table._tablename + '.' diff --git a/gluon/tests/test_appadmin.py b/gluon/tests/test_appadmin.py index 22b1d05a..6cd0c017 100644 --- a/gluon/tests/test_appadmin.py +++ b/gluon/tests/test_appadmin.py @@ -4,7 +4,7 @@ """ Unit tests for gluon.sqlhtml """ -from __future__ import print_function + import os import sys import unittest diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index 52701641..75ee08ce 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -56,7 +56,7 @@ def _prepare_exec_for_file(filename): elif os.path.split(filename)[1] == '__init__.py': filename = os.path.dirname(filename) else: - raise 'The file provided (%s) does is not a valid Python file.' + raise IOError('The file provided (%s) is not a valid Python file.') filename = os.path.realpath(filename) dirpath = filename while True: diff --git a/gluon/tests/test_globals.py b/gluon/tests/test_globals.py index bd78e4e1..3809a236 100644 --- a/gluon/tests/test_globals.py +++ b/gluon/tests/test_globals.py @@ -252,11 +252,11 @@ class testResponse(unittest.TestCase): def test_include_meta(self): response = Response() - response.meta[u'web2py'] = 'web2py' + response.meta['web2py'] = 'web2py' response.include_meta() self.assertEqual(response.body.getvalue(), '\n\n') response = Response() - response.meta[u'meta_dict'] = {u'tag_name':'tag_value'} + response.meta['meta_dict'] = {'tag_name':'tag_value'} response.include_meta() self.assertEqual(response.body.getvalue(), '\n\n') diff --git a/gluon/tests/test_router.py b/gluon/tests/test_router.py index 132f8a04..625673ad 100644 --- a/gluon/tests/test_router.py +++ b/gluon/tests/test_router.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """Unit tests for rewrite.py routers option""" -from __future__ import print_function + import os import unittest import tempfile diff --git a/gluon/tests/test_scheduler.py b/gluon/tests/test_scheduler.py index ee1270fb..f3349e34 100644 --- a/gluon/tests/test_scheduler.py +++ b/gluon/tests/test_scheduler.py @@ -551,12 +551,12 @@ class TestsForSchedulerAPIs(BaseTestScheduler): def isnotqueued(result): self.assertEqual(result.id, None) self.assertEqual(result.uuid, None) - self.assertEqual(len(result.errors.keys()) > 0, True) + self.assertEqual(len(list(result.errors.keys())) > 0, True) def isqueued(result): self.assertNotEqual(result.id, None) self.assertNotEqual(result.uuid, None) - self.assertEqual(len(result.errors.keys()), 0) + self.assertEqual(len(list(result.errors.keys())), 0) s = Scheduler(self.db) fname = 'foo' diff --git a/gluon/tests/test_web.py b/gluon/tests/test_web.py index a8d41a1d..864fc844 100644 --- a/gluon/tests/test_web.py +++ b/gluon/tests/test_web.py @@ -3,7 +3,7 @@ """ Unit tests for running web2py """ -from __future__ import print_function + import sys import os import unittest diff --git a/gluon/tools.py b/gluon/tools.py index de3223d6..bc3b8e35 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -15,8 +15,9 @@ from functools import reduce from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, urlencode from gluon._compat import configparser, MIMEBase, MIMEMultipart, MIMEText, Header from gluon._compat import Encoders, Charset, long, urllib_quote, iteritems -from gluon._compat import to_bytes, to_native, add_charset +from gluon._compat import to_bytes, to_native, add_charset, string_types from gluon._compat import charset_QP, basestring, unicodeT, to_unicode +from gluon._compat import urllib2, urlopen import datetime import logging import sys @@ -902,13 +903,13 @@ class Recaptcha2(DIV): 'secret': self.private_key, 'remoteip': remoteip, 'response': recaptcha_response_field, - }) + }).encode('utf-8') request = urllib2.Request( url=self.VERIFY_SERVER, data=to_bytes(params), headers={'Content-type': 'application/x-www-form-urlencoded', 'User-agent': 'reCAPTCHA Python'}) - httpresp = urllib2.urlopen(request) + httpresp = urlopen(request) content = httpresp.read() httpresp.close() try: @@ -1037,7 +1038,7 @@ class AuthJWT(object): Example: def mybefore_authorization(tokend): if not tokend['my_name_is'] == 'bond,james bond': - raise HTTP(400, u'Invalid JWT my_name_is claim') + raise HTTP(400, 'Invalid JWT my_name_is claim') - max_header_length: check max length to avoid load()ing unusually large tokens (could mean crafted, e.g. in a DDoS.) Basic Usage: @@ -1161,7 +1162,7 @@ class AuthJWT(object): b64h, b64b = body.split(b'.', 1) if b64h != self.cached_b64h: # header not the same - raise HTTP(400, u'Invalid JWT Header') + raise HTTP(400, 'Invalid JWT Header') secret = self.secret_key tokend = serializers.loads_json(to_native(self.jwt_b64d(b64b))) if self.salt: @@ -1172,11 +1173,11 @@ class AuthJWT(object): secret = to_bytes(secret, 'ascii', 'ignore') if not self.verify_signature(body, sig, secret): # signature verification failed - raise HTTP(400, u'Token signature is invalid') + raise HTTP(400, 'Token signature is invalid') if self.verify_expiration: now = time.mktime(datetime.datetime.utcnow().timetuple()) if tokend['exp'] + self.leeway < now: - raise HTTP(400, u'Token is expired') + raise HTTP(400, 'Token is expired') if callable(self.before_authorization): self.before_authorization(tokend) return tokend @@ -1209,11 +1210,11 @@ class AuthJWT(object): orig_exp = orig_payload['exp'] if orig_exp + self.leeway < now: # token already expired, can't be used for refresh - raise HTTP(400, u'Token already expired') + raise HTTP(400, 'Token already expired') orig_iat = orig_payload.get('orig_iat') or orig_payload['iat'] if orig_iat + self.refresh_expiration_delta < now: # refreshed too long ago - raise HTTP(400, u'Token issued too long ago') + raise HTTP(400, 'Token issued too long ago') expires = now + self.expiration orig_payload.update( orig_iat=orig_iat, @@ -1259,7 +1260,7 @@ class AuthJWT(object): pass if token: if not self.allow_refresh: - raise HTTP(403, u'Refreshing token is not allowed') + raise HTTP(403, 'Refreshing token is not allowed') tokend = self.load_token(token) # verification can fail here refreshed = self.refresh_token(tokend) @@ -1277,9 +1278,9 @@ class AuthJWT(object): ret = {'token': self.generate_token(payload)} elif ret is None: raise HTTP(401, - u'Not Authorized - need to be logged in, to pass a token ' - u'for refresh or username and password for login', - **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm}) + 'Not Authorized - need to be logged in, to pass a token ' + 'for refresh or username and password for login', + **{'WWW-Authenticate': 'JWT realm="%s"' % self.realm}) response.headers['Content-Type'] = 'application/json' return serializers.json(ret) @@ -1303,9 +1304,9 @@ class AuthJWT(object): if token_in_header: parts = token_in_header.split() if parts[0].lower() != self.header_prefix.lower(): - raise HTTP(400, u'Invalid JWT header') + raise HTTP(400, 'Invalid JWT header') elif len(parts) == 1: - raise HTTP(400, u'Invalid JWT header, missing token') + raise HTTP(400, 'Invalid JWT header, missing token') elif len(parts) > 2: raise HTTP(400, 'Invalid JWT header, token contains spaces') token = parts[1] @@ -2243,11 +2244,11 @@ class Auth(AuthAPI): if basic_auth_realm: if callable(basic_auth_realm): basic_auth_realm = basic_auth_realm() - elif isinstance(basic_auth_realm, (unicode, str)): - basic_realm = unicode(basic_auth_realm) # Warning python 3.5 does not have method unicod + elif isinstance(basic_auth_realm, string_types): + basic_realm = to_unicode(basic_auth_realm) elif basic_auth_realm is True: - basic_realm = u'' + current.request.application - http_401 = HTTP(401, u'Not Authorized', **{'WWW-Authenticate': u'Basic realm="' + basic_realm + '"'}) + basic_realm = '' + current.request.application + http_401 = HTTP(401, 'Not Authorized', **{'WWW-Authenticate': 'Basic realm="' + basic_realm + '"'}) if not basic or not basic[:6].lower() == 'basic ': if basic_auth_realm: raise http_401 @@ -3574,7 +3575,7 @@ class Auth(AuthAPI): requires = table_user[passfield].requires if not isinstance(requires, (list, tuple)): requires = [requires] - requires = list(filter(lambda t: isinstance(t, CRYPT), requires)) + requires = [t for t in requires if isinstance(t, CRYPT)] if requires: requires[0] = CRYPT(**requires[0].__dict__) # Copy the existing CRYPT attributes requires[0].min_length = 0 # But do not enforce minimum length for the old password @@ -4614,7 +4615,6 @@ class Crud(object): # pragma: no cover results = None return form, results - urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor())) @@ -4632,7 +4632,7 @@ def fetch(url, data=None, headers=None, from google.appengine.api import urlfetch except ImportError: req = urllib2.Request(url, data, headers) - html = urllib2.urlopen(req).read() + html = urlopen(req).read() else: method = ((data is None) and urlfetch.GET) or urlfetch.POST while url is not None: @@ -5000,7 +5000,7 @@ class Service(object): elif r and not isinstance(r, types.GeneratorType) and isinstance(r[0], (dict, Storage)): import csv writer = csv.writer(s) - writer.writerow(r[0].keys()) + writer.writerow(list(r[0].keys())) for line in r: writer.writerow([none_exception(v) for v in line.values()]) @@ -5217,7 +5217,7 @@ class Service(object): def serve_xmlrpc(self): request = current.request response = current.response - services = self.xmlrpc_procedures.values() + services = list(self.xmlrpc_procedures.values()) return response.xmlrpc(request, services) def serve_amfrpc(self, version=0): @@ -5572,7 +5572,7 @@ class PluginManager(object): return self.__dict__[key] def keys(self): - return self.__dict__.keys() + return list(self.__dict__.keys()) def __contains__(self, key): return key in self.__dict__ @@ -5818,7 +5818,7 @@ class Wiki(object): settings.templates = templates settings.controller = controller settings.function = function - settings.groups = auth.user_groups.values() \ + settings.groups = list(auth.user_groups.values()) \ if groups is None else groups db = auth.db @@ -5914,7 +5914,7 @@ class Wiki(object): if (auth.user and check_credentials(current.request, gae_login=False) and 'wiki_editor' not in auth.user_groups.values() and - self.settings.groups == auth.user_groups.values()): + self.settings.groups == list(auth.user_groups.values())): group = db.auth_group(role='wiki_editor') gid = group.id if group else db.auth_group.insert( role='wiki_editor') diff --git a/gluon/validators.py b/gluon/validators.py index 541dddd1..f3da9cc1 100644 --- a/gluon/validators.py +++ b/gluon/validators.py @@ -16,7 +16,6 @@ import datetime import time import cgi import json -import urllib import struct import decimal import unicodedata @@ -437,7 +436,7 @@ class IS_IN_SET(Validator): self.multiple = multiple if isinstance(theset, dict): self.theset = [str(item) for item in theset] - self.labels = theset.values() + self.labels = list(theset.values()) elif theset and isinstance(theset, (tuple, list)) \ and isinstance(theset[0], (tuple, list)) and len(theset[0]) == 2: self.theset = [str(item) for item, label in theset] @@ -575,7 +574,7 @@ class IS_IN_DB(Validator): else: fields = [table[k] for k in self.fieldnames] ignore = (FieldVirtual, FieldMethod) - fields = filter(lambda f: not isinstance(f, ignore), fields) + fields = [f for f in fields if not isinstance(f, ignore)] if self.dbset.db._dbname != 'gae': orderby = self.orderby or reduce(lambda a, b: a | b, fields) groupby = self.groupby @@ -649,7 +648,7 @@ class IS_IN_DB(Validator): return (values, None) else: def count(values, s=self.dbset, f=field): - return s(f.belongs(map(int, values))).count() + return s(f.belongs(list(map(int, values)))).count() if self.dbset.db._adapter.dbengine == "google:datastore": range_ids = range(0, len(values), 30) @@ -3533,7 +3532,7 @@ class IS_IPV4(Validator): if isinstance(value, str): temp.append(value.split('.')) elif isinstance(value, (list, tuple)): - if len(value) == len(list(filter(lambda item: isinstance(item, int), value))) == 4: + if len(value) == len([item for item in value if isinstance(item, int)]) == 4: temp.append(value) else: for item in value: diff --git a/gluon/widget.py b/gluon/widget.py index 756a95ef..65b37c1d 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -9,7 +9,6 @@ The widget is called from web2py ---------------------------------- """ -from __future__ import print_function import datetime import sys @@ -31,6 +30,9 @@ from gluon.settings import global_settings from gluon.shell import run, test from gluon.utils import is_valid_ip_address, is_loopback_ip_address, getipaddrinfo +if PY2: + input = raw_input + ProgramName = 'web2py Web Framework' ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str( @@ -952,7 +954,7 @@ def console(): if options.gae: if not os.path.exists('app.yaml'): - name = raw_input("Your GAE app name: ") + name = input("Your GAE app name: ") content = open(os.path.join('examples', 'app.example.yaml'), 'rb').read() open('app.yaml', 'wb').write(content.replace("yourappname", name)) else: diff --git a/scripts/update_languages.py b/scripts/update_languages.py index a918379b..a92e851a 100644 --- a/scripts/update_languages.py +++ b/scripts/update_languages.py @@ -14,15 +14,14 @@ parentdir = os.path.dirname(currentdir) sys.path.insert(0, parentdir) from gluon.cfs import getcfs -from gluon.utf8 import Utf8 from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin -from gluon.languages import findT +from gluon.languages import findT, sort_function # This script can be run with no arguments (which sets the application folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (application folder path and default language). # When run, it will update the default language, as well as strip all of the strings found in the non-default languages but not in the default language, and add the strings found in the default language to the non-default languages it is not, making sure translators don't do additional work that will never be used. def read_dict_aux(filename): - lang_text = open(filename, 'r').read().replace(b'\r\n', b'\n') + lang_text = open(filename, 'r').read().replace('\r\n', '\n') try: return safe_eval(to_native(lang_text)) or {} except Exception: @@ -41,14 +40,11 @@ def safe_eval(text): return eval(text, {}, {}) return None -def sort_function(x, y): - return cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower()) - def write_file(file, contents): file.write('# -*- coding: utf-8 -*-\n{\n') - for key in sorted(contents, sort_function): - file.write('%s: %s,\n' % (repr(Utf8(key)), - repr(Utf8(contents[key])))) + for key in sorted(contents, key = sort_function): + file.write('%s: %s,\n' % (repr(to_unicode(key)), + repr(to_unicode(contents[key])))) file.write('}\n') file.close() @@ -67,7 +63,7 @@ def update_languages(cwd, default_lang): if phrase in default: new_dict[phrase] = i18n[phrase] write_file(open(os.path.join(cwd, "languages", lang), 'w'), new_dict) - print lang + print(lang) if __name__ == "__main__": cwd = os.getcwd() From 405527672cb1a7e9f97766156b49ec9288c53fc5 Mon Sep 17 00:00:00 2001 From: Vinyl Darkscratch Date: Sat, 9 Feb 2019 09:41:54 -0800 Subject: [PATCH 2/4] Update Python version in setup scripts --- scripts/setup-web2py-centos7.sh | 4 ++-- scripts/setup-web2py-debian-sid.sh | 4 ++-- scripts/setup-web2py-fedora-ami.sh | 4 ++-- scripts/setup-web2py-fedora.sh | 4 ++-- scripts/setup-web2py-ubuntu.sh | 13 ++++++------- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/scripts/setup-web2py-centos7.sh b/scripts/setup-web2py-centos7.sh index 6b74400a..e8c65e96 100644 --- a/scripts/setup-web2py-centos7.sh +++ b/scripts/setup-web2py-centos7.sh @@ -1,6 +1,6 @@ echo "This script will: 1) Install modules needed to run web2py on Fedora and CentOS/RHEL -2) Install Python 2.6 to /opt and recompile wsgi if not provided +2) Install Python 3.7 to /opt and recompile wsgi if not provided 2) Install web2py in /opt/web-apps/ 3) Configure SELinux and iptables 5) Create a self signed ssl certificate @@ -56,7 +56,7 @@ echo yum update # Install required packages -yum install httpd mod_ssl mod_wsgi wget python unzip +yum install httpd mod_ssl mod_wsgi wget python3 unzip ### ### Phase 2 - Install web2py diff --git a/scripts/setup-web2py-debian-sid.sh b/scripts/setup-web2py-debian-sid.sh index cf9b1a92..88b8431f 100644 --- a/scripts/setup-web2py-debian-sid.sh +++ b/scripts/setup-web2py-debian-sid.sh @@ -38,8 +38,8 @@ apt-get -y install libapache2-mod-wsgi apt-get -y install python-psycopg2 apt-get -y install postfix apt-get -y install wget -apt-get -y install python-matplotlib -apt-get -y install python-reportlab +apt-get -y install python3-matplotlib +apt-get -y install python3-reportlab apt-get -y install mercurial /etc/init.d/postgresql restart diff --git a/scripts/setup-web2py-fedora-ami.sh b/scripts/setup-web2py-fedora-ami.sh index 5ef2acaf..d726796b 100755 --- a/scripts/setup-web2py-fedora-ami.sh +++ b/scripts/setup-web2py-fedora-ami.sh @@ -1,6 +1,6 @@ echo "This script will: 1) Install modules needed to run web2py on Fedora and CentOS/RHEL -2) Install Python 2.6 to /opt and recompile wsgi if not provided +2) Install Python 3.7 to /opt and recompile wsgi if not provided 2) Install web2py in /opt/web-apps/ 3) Configure SELinux and iptables 5) Create a self signed ssl certificate @@ -54,7 +54,7 @@ echo yum update # Install required packages -yum install httpd mod_ssl mod_wsgi wget python +yum install httpd mod_ssl mod_wsgi wget python3 # Verify we have at least Python 2.5 typeset -i version_major diff --git a/scripts/setup-web2py-fedora.sh b/scripts/setup-web2py-fedora.sh index d7d6d01c..f0f3b0ed 100644 --- a/scripts/setup-web2py-fedora.sh +++ b/scripts/setup-web2py-fedora.sh @@ -1,7 +1,7 @@ #!/bin/bash echo "This script will: 1) Install modules needed to run web2py on Fedora and CentOS/RHEL -2) Install Python 2.6 to /opt and recompile wsgi if not provided +2) Install Python 3.7 to /opt and recompile wsgi if not provided 2) Install web2py in /opt/web-apps/ 3) Configure SELinux and iptables 5) Create a self signed ssl certificate @@ -55,7 +55,7 @@ echo yum update # Install required packages -yum install httpd mod_ssl mod_wsgi wget python +yum install httpd mod_ssl mod_wsgi wget python3 # Verify we have at least Python 2.5 typeset -i version_major diff --git a/scripts/setup-web2py-ubuntu.sh b/scripts/setup-web2py-ubuntu.sh index 8de3050a..2864f281 100644 --- a/scripts/setup-web2py-ubuntu.sh +++ b/scripts/setup-web2py-ubuntu.sh @@ -31,18 +31,17 @@ apt-get -y install zip unzip apt-get -y install tar apt-get -y install openssh-server apt-get -y install build-essential -apt-get -y install python -#apt-get -y install python2.5 -apt-get -y install ipython -apt-get -y install python-dev +apt-get -y install python3 +apt-get -y install ipython3 +apt-get -y install python3-dev apt-get -y install postgresql apt-get -y install apache2 apt-get -y install libapache2-mod-wsgi -apt-get -y install python2.5-psycopg2 +apt-get -y install python3-psycopg2 apt-get -y install postfix apt-get -y install wget -apt-get -y install python-matplotlib -apt-get -y install python-reportlab +apt-get -y install python3-matplotlib +apt-get -y install python3-reportlab apt-get -y install mercurial /etc/init.d/postgresql restart From c856c2b0e28d2f092adfa4a1e7a0899788c1eb5f Mon Sep 17 00:00:00 2001 From: Vinyl Darkscratch Date: Sat, 9 Feb 2019 10:03:41 -0800 Subject: [PATCH 3/4] Fix tabs + indents mix --- gluon/contrib/login_methods/saml2_auth.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gluon/contrib/login_methods/saml2_auth.py b/gluon/contrib/login_methods/saml2_auth.py index ea650463..7bec2612 100644 --- a/gluon/contrib/login_methods/saml2_auth.py +++ b/gluon/contrib/login_methods/saml2_auth.py @@ -108,8 +108,8 @@ def saml2_handler(session, request, config_filename = None, entityid = None): config_filename = config_filename or os.path.join(request.folder,'private','sp_conf') client = Saml2Client(config_file = config_filename) if not entityid: - idps = client.metadata.with_descriptor("idpsso") - entityid = idps.keys()[0] + idps = client.metadata.with_descriptor("idpsso") + entityid = idps.keys()[0] bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST] binding, destination = client.pick_binding( "single_sign_on_service", bindings, "idpsso", entity_id=entityid) @@ -120,7 +120,7 @@ def saml2_handler(session, request, config_filename = None, entityid = None): if not request.vars.SAMLResponse: req_id, req = client.create_authn_request(destination, binding=binding) relay_state = web2py_uuid().replace('-','') - session.saml_outstanding_queries = {req_id: request.url} + session.saml_outstanding_queries = {req_id: request.url} session.saml_req_id = req_id http_args = client.apply_binding(binding, str(req), destination, relay_state=relay_state) @@ -150,14 +150,14 @@ class Saml2Auth(object): self.config_file = config_file self.maps = maps - # URL for redirecting users to when they sign out + # URL for redirecting users to when they sign out self.saml_logout_url = logout_url # URL to let users change their password in the IDP system self.saml_change_password_url = change_password_url - - # URL to specify an IDP if using federation metadata or an MDQ - self.entityid = entityid + + # URL to specify an IDP if using federation metadata or an MDQ + self.entityid = entityid def login_url(self, next="/"): d = saml2_handler(current.session, current.request, entityid=self.entityid) From f4ffac58ae8846474d850bc039d684099a48efb8 Mon Sep 17 00:00:00 2001 From: Vinyl Darkscratch Date: Sat, 9 Feb 2019 10:03:49 -0800 Subject: [PATCH 4/4] Convert to print function --- gluon/contrib/pbkdf2.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gluon/contrib/pbkdf2.py b/gluon/contrib/pbkdf2.py index b7a7dd42..a19f1159 100644 --- a/gluon/contrib/pbkdf2.py +++ b/gluon/contrib/pbkdf2.py @@ -82,14 +82,14 @@ def test(): def check(data, salt, iterations, keylen, expected): rv = pbkdf2_hex(data, salt, iterations, keylen) if rv != expected: - print 'Test failed:' - print ' Expected: %s' % expected - print ' Got: %s' % rv - print ' Parameters:' - print ' data=%s' % data - print ' salt=%s' % salt - print ' iterations=%d' % iterations - print + print('Test failed:') + print(' Expected: %s' % expected) + print(' Got: %s' % rv) + print(' Parameters:') + print(' data=%s' % data) + print(' salt=%s' % salt) + print(' iterations=%d' % iterations) + print() failed.append(1) # From RFC 6070