From 8aecaf451436c014ac32dfe2535c96a7e6f4221f Mon Sep 17 00:00:00 2001 From: ilvalle Date: Fri, 24 Jun 2016 15:02:28 +0200 Subject: [PATCH] PY3 fixes and added tests for gluon/admin.py --- .travis.yml | 8 +++--- applications/admin/controllers/debug.py | 7 +++-- applications/admin/controllers/default.py | 24 ++++++++--------- applications/admin/controllers/webservices.py | 2 +- applications/admin/controllers/wizard.py | 2 +- .../admin/views/default.mobile/ticket.html | 2 +- .../admin/views/default.mobile/ticket.load | 2 +- applications/admin/views/default/about.html | 2 +- applications/admin/views/default/design.html | 8 +++--- .../admin/views/default/manage_students.html | 4 +-- gluon/admin.py | 8 +++--- gluon/compileapp.py | 5 +++- gluon/contrib/markmin/markmin2html.py | 2 +- gluon/fileutils.py | 5 +++- gluon/html.py | 3 ++- gluon/rocket.py | 26 +++--------------- gluon/tests/test_compileapp.py | 27 ++++++++++++++++++- gluon/widget.py | 6 ++--- web2py.py | 1 + 19 files changed, 75 insertions(+), 69 deletions(-) diff --git a/.travis.yml b/.travis.yml index a55f3cf9..be3d0356 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,12 +25,12 @@ install: virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION" source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate" fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install --download-cache $HOME/.pip-cache pycrypto pg8000 pymysql; fi; + - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install pycrypto pg8000 pymysql; fi; - if [[ $TRAVIS_PYTHON_VERSION != '3.5' ]]; then pip install -e .; fi; before_script: - - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache coverage; fi; - - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache codecov; fi + - pip install --download-cache $HOME/.pip-cache coverage + - pip install --download-cache $HOME/.pip-cache codecov - mysql -e 'create database pydal;' - psql -c 'create database pydal;' -U postgres - psql -c 'create extension postgis;' -U postgres -d pydal; @@ -41,7 +41,7 @@ script: export COVERAGE_PROCESS_START=gluon/tests/coverage.ini; ./web2py.py --ru after_success: - coverage combine; - - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then codecov; fi + - codecov notifications: email: true diff --git a/applications/admin/controllers/debug.py b/applications/admin/controllers/debug.py index 7176e09c..06ba3b83 100644 --- a/applications/admin/controllers/debug.py +++ b/applications/admin/controllers/debug.py @@ -1,6 +1,5 @@ import os import sys -import cStringIO import gluon.contrib.shell import gluon.dal import gluon.html @@ -122,7 +121,7 @@ def execute(): output = web_debugger.do_exec(command) if output is None: output = "" - except Exception, e: + except Exception as e: output = T("Exception %s") % str(e) k = len(session['debug_commands:' + app]) - 1 return '[%i] %s%s\n' % (k + 1, command, output) @@ -212,7 +211,7 @@ def toggle_breakpoint(): ok = True else: response.flash = T("Unable to determine the line number!") - except Exception, e: + except Exception as e: session.flash = str(e) return response.json({'ok': ok, 'lineno': lineno}) @@ -233,6 +232,6 @@ def list_breakpoints(): if filename == bp_filename: breakpoints.append(bp_lineno) ok = True - except Exception, e: + except Exception as e: session.flash = str(e) return response.json({'ok': ok, 'breakpoints': breakpoints}) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index b28c7626..359a5bc8 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -15,7 +15,7 @@ from gluon.utils import web2py_uuid from gluon.tools import Config from gluon.compileapp import find_exposed_functions from glob import glob -from gluon._compat import iteritems, PY2 +from gluon._compat import iteritems, PY2, pickle, xrange, urlopen, to_bytes import shutil import platform @@ -127,7 +127,6 @@ def index(): redirect(send) elif failed_login_count() >= allowed_number_of_attempts: time.sleep(2 ** allowed_number_of_attempts) - print('4033') raise HTTP(403) elif request.vars.password: if verify_password(request.vars.password[:1024]): @@ -274,7 +273,7 @@ def site(): elif form_update.vars.url: # fetch an application via URL or file upload try: - f = urllib.urlopen(form_update.vars.url) + f = urlopen(form_update.vars.url) if f.code == 404: raise Exception("404 file not found") except Exception as e: @@ -387,7 +386,7 @@ def pack_exe(app, base, filenames=None): # Download latest web2py_win and open it with zipfile download_url = 'http://www.web2py.com/examples/static/web2py_win.zip' out = StringIO() - out.write(urllib.urlopen(download_url).read()) + out.write(urlopen(download_url).read()) web2py_win = zipfile.ZipFile(out, mode='a') # Write routes.py with the application as default routes = u'# -*- coding: utf-8 -*-\nrouters = dict(BASE=dict(default_application="%s"))' % app @@ -858,7 +857,7 @@ def todolist(): for f in listfiles(app, d): matches = [] filename = apath(os.path.join(app, d, f), r=request) - with open(filename, 'r') as f_s: + with safe_open(filename, 'r') as f_s: src = f_s.read() for m in regex.finditer(src): start = m.start() @@ -1575,7 +1574,6 @@ def errors(): """ Error handler """ import operator import os - import pickle import hashlib app = get_app() @@ -1605,7 +1603,7 @@ def errors(): if not os.path.isfile(fullpath): continue try: - fullpath_file = open(fullpath, 'r') + fullpath_file = safe_open(fullpath, 'rb') try: error = pickle.load(fullpath_file) finally: @@ -1615,7 +1613,7 @@ def errors(): except EOFError: continue - hash = hashlib.md5(error['traceback']).hexdigest() + hash = hashlib.md5(to_bytes(error['traceback'])).hexdigest() if hash in delete_hashes: os.unlink(fullpath) @@ -1710,7 +1708,7 @@ def get_ticket_storage(app): private_folder = apath('%s/private' % app, r=request) ticket_file = os.path.join(private_folder, 'ticket_storage.txt') if os.path.exists(ticket_file): - db_string = open(ticket_file).read() + db_string = safe_open(ticket_file).read() db_string = db_string.strip().replace('\r', '').replace('\n', '') elif is_gae: # use Datastore as fallback if there is no ticket_file @@ -1965,9 +1963,9 @@ def plugins(): from serializers import loads_json if not session.plugins: try: - rawlist = urllib.urlopen("http://www.web2pyslices.com/" + - "public/api.json/action/list/content/Package?package" + - "_type=plugin&search_index=false").read() + rawlist = urlopen("http://www.web2pyslices.com/" + + "public/api.json/action/list/content/Package?package" + + "_type=plugin&search_index=false").read() session.plugins = loads_json(rawlist) except: response.flash = T('Unable to download the list of plugins') @@ -1993,7 +1991,7 @@ def install_plugin(): source.split("web2py.plugin.")[-1].split(".w2p")[0] else: filename = "web2py.plugin.%s.w2p" % cleanpath(plugin) - if plugin_install(app, urllib.urlopen(source), + if plugin_install(app, urlopen(source), request, filename): session.flash = T('New plugin installed: %s', filename) else: diff --git a/applications/admin/controllers/webservices.py b/applications/admin/controllers/webservices.py index a043aab5..eb760df2 100644 --- a/applications/admin/controllers/webservices.py +++ b/applications/admin/controllers/webservices.py @@ -101,7 +101,7 @@ def attach_debugger(host='localhost', port=6000, authkey='secret password'): gluon.debug.qdb_debugger = qdb.Qdb(gluon.debug.qdb_connection) gluon.debug.dbg = gluon.debug.qdb_debugger # welcome message (this should be displayed on the frontend) - print 'debugger connected to', gluon.debug.qdb_listener.last_accepted + print('debugger connected to', gluon.debug.qdb_listener.last_accepted) return True # connection successful! diff --git a/applications/admin/controllers/wizard.py b/applications/admin/controllers/wizard.py index 065729ff..f4511b96 100644 --- a/applications/admin/controllers/wizard.py +++ b/applications/admin/controllers/wizard.py @@ -516,7 +516,7 @@ def create(options): plugin_name = 'web2py.plugin.' + plugin + '.w2p' stream = urllib.urlopen(PLUGINS_APP + '/static/' + plugin_name) plugin_install(app, stream, request, plugin_name) - except Exception, e: + except Exception as e: session.flash = T("unable to download plugin: %s" % plugin) ### write configuration file into newapp/models/0.py diff --git a/applications/admin/views/default.mobile/ticket.html b/applications/admin/views/default.mobile/ticket.html index f9e08ed3..44484537 100644 --- a/applications/admin/views/default.mobile/ticket.html +++ b/applications/admin/views/default.mobile/ticket.html @@ -118,7 +118,7 @@
session
{{=BEAUTIFY(snapshot['session'])}}
response
{{=BEAUTIFY(snapshot['response'])}}
-{{except Exception, e:}} +{{except Exception as e:}} {{import traceback;tb=traceback.format_exc().replace("\n","\\n") }} diff --git a/applications/admin/views/default.mobile/ticket.load b/applications/admin/views/default.mobile/ticket.load index 381eee1d..9a42bba3 100644 --- a/applications/admin/views/default.mobile/ticket.load +++ b/applications/admin/views/default.mobile/ticket.load @@ -117,7 +117,7 @@
session
{{=BEAUTIFY(snapshot['session'])}}
response
{{=BEAUTIFY(snapshot['response'])}}
-{{except Exception, e:}} +{{except Exception as e:}} {{import traceback;tb=traceback.format_exc().replace("\n","\\n") }} diff --git a/applications/admin/views/default/about.html b/applications/admin/views/default/about.html index 1fe839e7..1e15b821 100644 --- a/applications/admin/views/default/about.html +++ b/applications/admin/views/default/about.html @@ -23,4 +23,4 @@ jQuery(document).ready(function() { jQuery.plot(jQuery("#placeholder"), [ {{=progress}} ]); }) - \ No newline at end of file + diff --git a/applications/admin/views/default/design.html b/applications/admin/views/default/design.html index 173f0053..82ad7bd4 100644 --- a/applications/admin/views/default/design.html +++ b/applications/admin/views/default/design.html @@ -67,7 +67,6 @@ def deletefile(arglist, vars={}): **{'_data-placement':'right', '_data-original-title':T('Delete this file (you will be asked to confirm deletion)')}) }} - {{block sectionclass}}design{{end}} @@ -120,7 +119,7 @@ def deletefile(arglist, vars={}): {{=peekfile('models',m, dict(id=id))}} - {{if len(defines[m]):}}{{=T("defines tables")}} {{pass}}{{=XML(', '.join([B(table).xml() for table in defines[m]]))}} + {{if len(defines[m]):}}{{=T("defines tables")}} {{pass}}{{=XML(b', '.join([B(table).xml() for table in defines[m]]))}} {{pass}} @@ -133,7 +132,6 @@ def deletefile(arglist, vars={}): - {{ controller_functions=[] @@ -168,7 +166,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi {{=peekfile('controllers',c, dict(id=id))}} - {{if functions[c]:}}{{=T("exposes")}}{{pass}} {{=XML(', '.join([A(f,_href=URL(a=app,c=c[:-3],f=f)).xml() for f in functions[c]]))}} + {{if functions[c]:}}{{=T("exposes")}}{{pass}} {{=XML(b', '.join([A(f,_href=URL(a=app,c=c[:-3],f=f)).xml() for f in functions[c]]))}} {{pass}} @@ -208,7 +206,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi {{if c in extend:}}{{=T("extends")}} {{=extend[c]}} {{pass}} - {{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(', '.join([B(f).xml() for f in include[c]]))}} + {{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(b', '.join([B(f).xml() for f in include[c]]))}} {{pass}} diff --git a/applications/admin/views/default/manage_students.html b/applications/admin/views/default/manage_students.html index d4e827d7..0885ff76 100644 --- a/applications/admin/views/default/manage_students.html +++ b/applications/admin/views/default/manage_students.html @@ -150,7 +150,7 @@ elif request.args(0)=='edit': input = tr[1][0] help = tr[2][0] new_form_flds.append(label) - print label + print(label) new_form_flds.append(input) new_form_flds.append(help) pass @@ -160,4 +160,4 @@ pass }}

{{=T('Manage Admin Users/Students')}}

-{{=grid}} \ No newline at end of file +{{=grid}} diff --git a/gluon/admin.py b/gluon/admin.py index 61aa56e8..8a2f9277 100644 --- a/gluon/admin.py +++ b/gluon/admin.py @@ -14,7 +14,6 @@ import os import sys import traceback import zipfile -import urllib from shutil import rmtree from gluon.utils import web2py_uuid from gluon.fileutils import w2p_pack, w2p_unpack, w2p_pack_plugin, w2p_unpack_plugin @@ -23,7 +22,7 @@ from gluon.fileutils import read_file, write_file, parse_version from gluon.restricted import RestrictedError from gluon.settings import global_settings from gluon.cache import CacheOnDisk - +from gluon._compat import urlopen, to_native if not global_settings.web2py_runtime_gae: import site @@ -338,8 +337,7 @@ def check_new_version(myversion, version_url): """ try: - from urllib import urlopen - version = urlopen(version_url).read() + version = to_native(urlopen(version_url).read()) pversion = parse_version(version) pmyversion = parse_version(myversion) except IOError: @@ -423,7 +421,7 @@ def upgrade(request, url='http://web2py.com'): full_url = url + '/examples/static/web2py_%s.zip' % version_type filename = abspath('web2py_%s_downloaded.zip' % version_type) try: - write_file(filename, urllib.urlopen(full_url).read(), 'wb') + write_file(filename, urlopen(full_url).read(), 'wb') except Exception as e: return False, e try: diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 1a55c3bc..5bedc04e 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -18,7 +18,7 @@ import fnmatch import os import copy import random -from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring +from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long from gluon.storage import Storage, List from gluon.template import parse_template from gluon.restricted import restricted, compile2 @@ -400,6 +400,9 @@ _base_environment_['PY2'] = PY2 _base_environment_['to_native'] = to_native _base_environment_['to_bytes'] = to_bytes _base_environment_['iteritems'] = iteritems +_base_environment_['reduce'] = reduce +_base_environment_['xrange'] = xrange + def build_environment(request, response, session, store_current=True): """ diff --git a/gluon/contrib/markmin/markmin2html.py b/gluon/contrib/markmin/markmin2html.py index 0752e063..fe9b374d 100755 --- a/gluon/contrib/markmin/markmin2html.py +++ b/gluon/contrib/markmin/markmin2html.py @@ -6,7 +6,7 @@ from __future__ import print_function import re import urllib -from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes, to_native, _local_html_escape as escape +from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes, to_native, _local_html_escape as escape, xrange from ast import parse as ast_parse import ast diff --git a/gluon/fileutils.py b/gluon/fileutils.py index e9d869b3..5ccdfaac 100644 --- a/gluon/fileutils.py +++ b/gluon/fileutils.py @@ -115,7 +115,10 @@ def write_file(filename, value, mode='w'): """Writes to filename, making sure to close the file explicitly on exit. """ - f = open(filename, mode) + if PY2: + f = open(filename, mode) + else: + f = open(filename, mode, encoding="utf8") try: return f.write(value) finally: diff --git a/gluon/html.py b/gluon/html.py index f2fb7b9b..5ddf5dd4 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -2826,7 +2826,8 @@ class MARKMIN(XmlComponent): return to_bytes(html) if not self.kwargs else to_bytes(DIV(XML(html), **self.kwargs).xml()) def __str__(self): - return self.xml() + # In PY3 __str__ cannot return bytes (TypeError: __str__ returned non-string (type bytes)) + return to_native(self.xml()) def ASSIGNJS(**kargs): diff --git a/gluon/rocket.py b/gluon/rocket.py index 199f1004..83261a36 100644 --- a/gluon/rocket.py +++ b/gluon/rocket.py @@ -11,7 +11,7 @@ import errno import socket import logging import platform -from gluon._compat import iteritems, to_bytes +from gluon._compat import iteritems, to_bytes, StringIO, urllib_unquote # Define Constants VERSION = '1.2.6' @@ -182,13 +182,6 @@ class Connection(object): # Import System Modules import socket -try: - from io import StringIO -except ImportError: - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO # Import Package Modules # package imports removed in monolithic build @@ -1179,19 +1172,6 @@ from wsgiref.headers import Headers from threading import Thread from datetime import datetime -try: - from urllib import unquote -except ImportError: - from urllib.parse import unquote - -try: - from io import StringIO -except ImportError: - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - try: from ssl import SSLError except ImportError: @@ -1436,7 +1416,7 @@ class Worker(Thread): req[k] = "" if k == 'path': req['path'] = r'%2F'.join( - [unquote(x) for x in re_SLASH.split(v)]) + [urllib_unquote(x) for x in re_SLASH.split(v)]) self.protocol = req['protocol'] return req @@ -1471,7 +1451,7 @@ class Worker(Thread): if '?' in path: path, query_string = path.split('?', 1) - path = r'%2F'.join([unquote(x) for x in re_SLASH.split(path)]) + path = r'%2F'.join([urllib_unquote(x) for x in re_SLASH.split(path)]) req.update(path=path, query_string=query_string, diff --git a/gluon/tests/test_compileapp.py b/gluon/tests/test_compileapp.py index 2e740e06..956c28a4 100644 --- a/gluon/tests/test_compileapp.py +++ b/gluon/tests/test_compileapp.py @@ -10,8 +10,12 @@ fix_sys_path(__file__) from gluon.compileapp import compile_application, remove_compiled_application from gluon.fileutils import w2p_pack, w2p_unpack -import os +from gluon.globals import Request +from gluon.admin import app_compile, app_create, app_cleanup, check_new_version, app_uninstall +from gluon.main import global_settings +import os, shutil +WEB2PY_VERSION_URL = "http://web2py.com/examples/default/version" class TestPack(unittest.TestCase): """ Tests the compileapp.py module """ @@ -30,6 +34,27 @@ class TestPack(unittest.TestCase): w2p_unpack(test_path, unpack_path) return + def test_admin_compile(self): + #apps = ['welcome', 'admin', 'examples'] + request = Request(env={}) + request.application = 'a' + request.controller = 'c' + request.function = 'f' + request.folder = 'applications/admin' + apps = ['welcome'] + for appname in apps: + appname_path = os.path.join(os.getcwd(), 'applications', appname) + self.assertEqual(app_compile(appname_path, request), None) + # remove any existing test_app + new_app = 'test_app_%s' % (appname) + if(os.path.exists('applications/%s' % (new_app))): + shutil.rmtree('applications/%s' % (new_app)) + self.assertEqual(app_create(new_app, request), True) + self.assertEqual(os.path.exists('applications/test_app_%s/controllers/default.py' % (appname)), True) + self.assertEqual(app_cleanup(new_app, request), True) + self.assertEqual(app_uninstall(new_app, request), True) + self.assertNotEqual(check_new_version(global_settings.web2py_version, WEB2PY_VERSION_URL), -1) + return if __name__ == '__main__': unittest.main() diff --git a/gluon/widget.py b/gluon/widget.py index b641bb50..1bbdb830 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -13,7 +13,7 @@ from __future__ import print_function import datetime import sys -from gluon._compat import StringIO, thread +from gluon._compat import StringIO, thread, xrange import time import threading import os @@ -87,7 +87,7 @@ class IO(object): def __init__(self): """ """ - self.buffer = cStringIO.StringIO() + self.buffer = StringIO() def write(self, data): """ """ @@ -968,7 +968,7 @@ def console(): run_system_tests(options) if options.quiet: - capture = cStringIO.StringIO() + capture = StringIO() sys.stdout = capture logger.setLevel(logging.CRITICAL + 1) else: diff --git a/web2py.py b/web2py.py index 516427f7..ced8ff22 100755 --- a/web2py.py +++ b/web2py.py @@ -28,5 +28,6 @@ if __name__ == '__main__': import coverage coverage.process_startup() except: + print('Coverage is not available') pass gluon.widget.start(cron=True)