PY3 fixes and added tests for gluon/admin.py

2016-06-24 15:02:28 +02:00
parent 48350664f0
commit 8aecaf4514
19 changed files with 75 additions and 69 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,12 +25,12 @@ install:
        virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION"
        source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate"
      fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install --download-cache $HOME/.pip-cache pycrypto pg8000 pymysql; fi;
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install pycrypto pg8000 pymysql; fi;
  - if [[ $TRAVIS_PYTHON_VERSION != '3.5' ]]; then pip install -e .; fi;

 before_script:
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache coverage; fi;
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --download-cache $HOME/.pip-cache codecov; fi
+  - pip install --download-cache $HOME/.pip-cache coverage
+  - pip install --download-cache $HOME/.pip-cache codecov
  - mysql -e 'create database pydal;'
  - psql -c 'create database pydal;' -U postgres
  - psql -c 'create extension postgis;' -U postgres -d pydal;
@@ -41,7 +41,7 @@ script: export COVERAGE_PROCESS_START=gluon/tests/coverage.ini; ./web2py.py --ru

 after_success:
  - coverage combine;
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then codecov; fi
+  - codecov

 notifications:
  email: true
--- a/applications/admin/controllers/debug.py
+++ b/applications/admin/controllers/debug.py
@@ -1,6 +1,5 @@
 import os
 import sys
-import cStringIO
 import gluon.contrib.shell
 import gluon.dal
 import gluon.html
@@ -122,7 +121,7 @@ def execute():
        output = web_debugger.do_exec(command)
        if output is None:
            output = ""
-    except Exception, e:
+    except Exception as e:
        output = T("Exception %s") % str(e)
    k = len(session['debug_commands:' + app]) - 1
    return '[%i] %s%s\n' % (k + 1, command, output)
@@ -212,7 +211,7 @@ def toggle_breakpoint():
                ok = True
        else:
            response.flash = T("Unable to determine the line number!")
-    except Exception, e:
+    except Exception as e:
        session.flash = str(e)
    return response.json({'ok': ok, 'lineno': lineno})

@@ -233,6 +232,6 @@ def list_breakpoints():
            if filename == bp_filename:
                breakpoints.append(bp_lineno)
        ok = True
-    except Exception, e:
+    except Exception as e:
        session.flash = str(e)
    return response.json({'ok': ok, 'breakpoints': breakpoints})
--- a/applications/admin/controllers/default.py
+++ b/applications/admin/controllers/default.py
@@ -15,7 +15,7 @@ from gluon.utils import web2py_uuid
 from gluon.tools import Config
 from gluon.compileapp import find_exposed_functions
 from glob import glob
-from gluon._compat import iteritems, PY2
+from gluon._compat import iteritems, PY2, pickle, xrange, urlopen, to_bytes
 import shutil
 import platform

@@ -127,7 +127,6 @@ def index():
        redirect(send)
    elif failed_login_count() >= allowed_number_of_attempts:
        time.sleep(2 ** allowed_number_of_attempts)
-        print('4033')
        raise HTTP(403)
    elif request.vars.password:
        if verify_password(request.vars.password[:1024]):
@@ -274,7 +273,7 @@ def site():
        elif form_update.vars.url:
            # fetch an application via URL or file upload
            try:
-                f = urllib.urlopen(form_update.vars.url)
+                f = urlopen(form_update.vars.url)
                if f.code == 404:
                    raise Exception("404 file not found")
            except Exception as e:
@@ -387,7 +386,7 @@ def pack_exe(app, base, filenames=None):
    # Download latest web2py_win and open it with zipfile
    download_url = 'http://www.web2py.com/examples/static/web2py_win.zip'
    out = StringIO()
-    out.write(urllib.urlopen(download_url).read())
+    out.write(urlopen(download_url).read())
    web2py_win = zipfile.ZipFile(out, mode='a')
    # Write routes.py with the application as default
    routes = u'# -*- coding: utf-8 -*-\nrouters = dict(BASE=dict(default_application="%s"))' % app
@@ -858,7 +857,7 @@ def todolist():
        for f in listfiles(app, d):
            matches = []
            filename = apath(os.path.join(app, d, f), r=request)
-            with open(filename, 'r') as f_s:
+            with safe_open(filename, 'r') as f_s:
                src = f_s.read()
                for m in regex.finditer(src):
                    start = m.start()
@@ -1575,7 +1574,6 @@ def errors():
    """ Error handler """
    import operator
    import os
-    import pickle
    import hashlib

    app = get_app()
@@ -1605,7 +1603,7 @@ def errors():
            if not os.path.isfile(fullpath):
                continue
            try:
-                fullpath_file = open(fullpath, 'r')
+                fullpath_file = safe_open(fullpath, 'rb')
                try:
                    error = pickle.load(fullpath_file)
                finally:
@@ -1615,7 +1613,7 @@ def errors():
            except EOFError:
                continue

-            hash = hashlib.md5(error['traceback']).hexdigest()
+            hash = hashlib.md5(to_bytes(error['traceback'])).hexdigest()

            if hash in delete_hashes:
                os.unlink(fullpath)
@@ -1710,7 +1708,7 @@ def get_ticket_storage(app):
    private_folder = apath('%s/private' % app, r=request)
    ticket_file = os.path.join(private_folder, 'ticket_storage.txt')
    if os.path.exists(ticket_file):
-        db_string = open(ticket_file).read()
+        db_string = safe_open(ticket_file).read()
        db_string = db_string.strip().replace('\r', '').replace('\n', '')
    elif is_gae:
        # use Datastore as fallback if there is no ticket_file
@@ -1965,9 +1963,9 @@ def plugins():
    from serializers import loads_json
    if not session.plugins:
        try:
-            rawlist = urllib.urlopen("http://www.web2pyslices.com/" +
-                                     "public/api.json/action/list/content/Package?package" +
-                                     "_type=plugin&search_index=false").read()
+            rawlist = urlopen("http://www.web2pyslices.com/" +
+                              "public/api.json/action/list/content/Package?package" +
+                              "_type=plugin&search_index=false").read()
            session.plugins = loads_json(rawlist)
        except:
            response.flash = T('Unable to download the list of plugins')
@@ -1993,7 +1991,7 @@ def install_plugin():
                source.split("web2py.plugin.")[-1].split(".w2p")[0]
        else:
            filename = "web2py.plugin.%s.w2p" % cleanpath(plugin)
-        if plugin_install(app, urllib.urlopen(source),
+        if plugin_install(app, urlopen(source),
                          request, filename):
            session.flash = T('New plugin installed: %s', filename)
        else:
--- a/applications/admin/controllers/webservices.py
+++ b/applications/admin/controllers/webservices.py
@@ -101,7 +101,7 @@ def attach_debugger(host='localhost', port=6000, authkey='secret password'):
        gluon.debug.qdb_debugger = qdb.Qdb(gluon.debug.qdb_connection)
        gluon.debug.dbg = gluon.debug.qdb_debugger
        # welcome message (this should be displayed on the frontend)
-        print 'debugger connected to', gluon.debug.qdb_listener.last_accepted
+        print('debugger connected to', gluon.debug.qdb_listener.last_accepted)
    return True     # connection successful!


--- a/applications/admin/controllers/wizard.py
+++ b/applications/admin/controllers/wizard.py
@@ -516,7 +516,7 @@ def create(options):
            plugin_name = 'web2py.plugin.' + plugin + '.w2p'
            stream = urllib.urlopen(PLUGINS_APP + '/static/' + plugin_name)
            plugin_install(app, stream, request, plugin_name)
-        except Exception, e:
+        except Exception as e:
            session.flash = T("unable to download plugin: %s" % plugin)

    ### write configuration file into newapp/models/0.py
--- a/applications/admin/views/default.mobile/ticket.html
+++ b/applications/admin/views/default.mobile/ticket.html
@@ -118,7 +118,7 @@
        <div class="hide inspect" id="session"><h6>session</h6>{{=BEAUTIFY(snapshot['session'])}}</div>
        <div class="hide inspect" id="response"><h6>response</h6>{{=BEAUTIFY(snapshot['response'])}}</div>
    </div>
-{{except Exception, e:}}
+{{except Exception as e:}}
    <!-- this should not happen, just in case... (cannot output normal hmtl as we don't know current open tags) -->
    {{import traceback;tb=traceback.format_exc().replace("\n","\\n") }}
    <script language='javascript'>alert("Exception during snapshot rendering: {{=tb}} ");</script>
--- a/applications/admin/views/default.mobile/ticket.load
+++ b/applications/admin/views/default.mobile/ticket.load
@@ -117,7 +117,7 @@
        <div class="hide inspect" id="session"><h6>session</h6>{{=BEAUTIFY(snapshot['session'])}}</div>
        <div class="hide inspect" id="response"><h6>response</h6>{{=BEAUTIFY(snapshot['response'])}}</div>
    </div>
-{{except Exception, e:}}
+{{except Exception as e:}}
    <!-- this should not happen, just in case... (cannot output normal hmtl as we don't know current open tags) -->
    {{import traceback;tb=traceback.format_exc().replace("\n","\\n") }}
    <script language='javascript'>alert("Exception during snapshot rendering: {{=tb}} ");</script>
--- a/applications/admin/views/default/about.html
+++ b/applications/admin/views/default/about.html
@@ -23,4 +23,4 @@ jQuery(document).ready(function() {
 	jQuery.plot(jQuery("#placeholder"), [ {{=progress}} ]);
 })
 </script>
-<!-- end "about" block -->
+<!-- end "about" block -->
--- a/applications/admin/views/default/design.html
+++ b/applications/admin/views/default/design.html
@@ -67,7 +67,6 @@ def deletefile(arglist, vars={}):
                        **{'_data-placement':'right',
                            '_data-original-title':T('Delete this file (you will be asked to confirm deletion)')})
 }}
-
 {{block sectionclass}}design{{end}}
 <!-- begin "design" block -->

@@ -120,7 +119,7 @@ def deletefile(arglist, vars={}):
                {{=peekfile('models',m, dict(id=id))}}
            </span>
            <span class="extras">
-                {{if len(defines[m]):}}{{=T("defines tables")}} {{pass}}{{=XML(', '.join([B(table).xml() for table in defines[m]]))}}
+                {{if len(defines[m]):}}{{=T("defines tables")}} {{pass}}{{=XML(b', '.join([B(table).xml() for table in defines[m]]))}}
            </span>
        </li>
        {{pass}}
@@ -133,7 +132,6 @@ def deletefile(arglist, vars={}):
      </div>
    </div>
 </div>
-
 <!-- FIND CONTROLLER FUNCTIONS -->
 {{
 controller_functions=[]
@@ -168,7 +166,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
        {{=peekfile('controllers',c, dict(id=id))}}
        </span>
        <span class="extras celled">
-            {{if functions[c]:}}{{=T("exposes")}}{{pass}} {{=XML(', '.join([A(f,_href=URL(a=app,c=c[:-3],f=f)).xml() for f in functions[c]]))}}
+            {{if functions[c]:}}{{=T("exposes")}}{{pass}} {{=XML(b', '.join([A(f,_href=URL(a=app,c=c[:-3],f=f)).xml() for f in functions[c]]))}}
        </span>
        </li>
        {{pass}}
@@ -208,7 +206,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
            </span>
            <span class="extras celled celled-one">
                {{if c in extend:}}{{=T("extends")}} <b>{{=extend[c]}}</b> {{pass}}
-                {{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(', '.join([B(f).xml() for f in include[c]]))}}
+                {{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(b', '.join([B(f).xml() for f in include[c]]))}}
            </span>
        </li>
        {{pass}}
--- a/applications/admin/views/default/manage_students.html
+++ b/applications/admin/views/default/manage_students.html
@@ -150,7 +150,7 @@ elif request.args(0)=='edit':
 		input = tr[1][0]
 		help  = tr[2][0]
 		new_form_flds.append(label)
-		print label
+		print(label)
 		new_form_flds.append(input)
 		new_form_flds.append(help)
 	pass
@@ -160,4 +160,4 @@ pass
 }}

 <h2>{{=T('Manage Admin Users/Students')}}</h2>
-{{=grid}}
+{{=grid}}
--- a/gluon/admin.py
+++ b/gluon/admin.py
@@ -14,7 +14,6 @@ import os
 import sys
 import traceback
 import zipfile
-import urllib
 from shutil import rmtree
 from gluon.utils import web2py_uuid
 from gluon.fileutils import w2p_pack, w2p_unpack, w2p_pack_plugin, w2p_unpack_plugin
@@ -23,7 +22,7 @@ from gluon.fileutils import read_file, write_file, parse_version
 from gluon.restricted import RestrictedError
 from gluon.settings import global_settings
 from gluon.cache import CacheOnDisk
-
+from gluon._compat import urlopen, to_native

 if not global_settings.web2py_runtime_gae:
    import site
@@ -338,8 +337,7 @@ def check_new_version(myversion, version_url):

    """
    try:
-        from urllib import urlopen
-        version = urlopen(version_url).read()
+        version = to_native(urlopen(version_url).read())
        pversion = parse_version(version)
        pmyversion = parse_version(myversion)
    except IOError:
@@ -423,7 +421,7 @@ def upgrade(request, url='http://web2py.com'):
    full_url = url + '/examples/static/web2py_%s.zip' % version_type
    filename = abspath('web2py_%s_downloaded.zip' % version_type)
    try:
-        write_file(filename, urllib.urlopen(full_url).read(), 'wb')
+        write_file(filename, urlopen(full_url).read(), 'wb')
    except Exception as e:
        return False, e
    try:
--- a/gluon/compileapp.py
+++ b/gluon/compileapp.py
@@ -18,7 +18,7 @@ import fnmatch
 import os
 import copy
 import random
-from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring
+from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long
 from gluon.storage import Storage, List
 from gluon.template import parse_template
 from gluon.restricted import restricted, compile2
@@ -400,6 +400,9 @@ _base_environment_['PY2'] = PY2
 _base_environment_['to_native'] = to_native
 _base_environment_['to_bytes'] = to_bytes
 _base_environment_['iteritems'] = iteritems
+_base_environment_['reduce'] = reduce
+_base_environment_['xrange'] = xrange
+

 def build_environment(request, response, session, store_current=True):
    """
--- a/gluon/contrib/markmin/markmin2html.py
+++ b/gluon/contrib/markmin/markmin2html.py
@@ -6,7 +6,7 @@
 from __future__ import print_function
 import re
 import urllib
-from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes, to_native, _local_html_escape as escape
+from gluon._compat import maketrans, urllib_quote, unicodeT, _local_html_escape, to_bytes, to_native, _local_html_escape as escape, xrange
 from ast import parse as ast_parse
 import ast

--- a/gluon/fileutils.py
+++ b/gluon/fileutils.py
@@ -115,7 +115,10 @@ def write_file(filename, value, mode='w'):
    """Writes <value> to filename, making sure to close the file
    explicitly on exit.
    """
-    f = open(filename, mode)
+    if PY2:
+        f = open(filename, mode)
+    else:
+        f = open(filename, mode, encoding="utf8")
    try:
        return f.write(value)
    finally:
--- a/gluon/html.py
+++ b/gluon/html.py
@@ -2826,7 +2826,8 @@ class MARKMIN(XmlComponent):
        return to_bytes(html) if not self.kwargs else to_bytes(DIV(XML(html), **self.kwargs).xml())

    def __str__(self):
-        return self.xml()
+        # In PY3 __str__ cannot return bytes (TypeError: __str__ returned non-string (type bytes))
+        return to_native(self.xml())


 def ASSIGNJS(**kargs):
--- a/gluon/rocket.py
+++ b/gluon/rocket.py
@@ -11,7 +11,7 @@ import errno
 import socket
 import logging
 import platform
-from gluon._compat import iteritems, to_bytes
+from gluon._compat import iteritems, to_bytes, StringIO, urllib_unquote

 # Define Constants
 VERSION = '1.2.6'
@@ -182,13 +182,6 @@ class Connection(object):

 # Import System Modules
 import socket
-try:
-    from io import StringIO
-except ImportError:
-    try:
-        from cStringIO import StringIO
-    except ImportError:
-        from StringIO import StringIO
 # Import Package Modules
 # package imports removed in monolithic build

@@ -1179,19 +1172,6 @@ from wsgiref.headers import Headers
 from threading import Thread
 from datetime import datetime

-try:
-    from urllib import unquote
-except ImportError:
-    from urllib.parse import unquote
-
-try:
-    from io import StringIO
-except ImportError:
-    try:
-        from cStringIO import StringIO
-    except ImportError:
-        from StringIO import StringIO
-
 try:
    from ssl import SSLError
 except ImportError:
@@ -1436,7 +1416,7 @@ class Worker(Thread):
                req[k] = ""
            if k == 'path':
                req['path'] = r'%2F'.join(
-                    [unquote(x) for x in re_SLASH.split(v)])
+                    [urllib_unquote(x) for x in re_SLASH.split(v)])

        self.protocol = req['protocol']
        return req
@@ -1471,7 +1451,7 @@ class Worker(Thread):
        if '?' in path:
            path, query_string = path.split('?', 1)

-        path = r'%2F'.join([unquote(x) for x in re_SLASH.split(path)])
+        path = r'%2F'.join([urllib_unquote(x) for x in re_SLASH.split(path)])

        req.update(path=path,
                   query_string=query_string,
--- a/gluon/tests/test_compileapp.py
+++ b/gluon/tests/test_compileapp.py
@@ -10,8 +10,12 @@ fix_sys_path(__file__)

 from gluon.compileapp import compile_application, remove_compiled_application
 from gluon.fileutils import w2p_pack, w2p_unpack
-import os
+from gluon.globals import Request
+from gluon.admin import app_compile, app_create, app_cleanup, check_new_version, app_uninstall
+from gluon.main import global_settings
+import os, shutil

+WEB2PY_VERSION_URL = "http://web2py.com/examples/default/version"

 class TestPack(unittest.TestCase):
    """ Tests the compileapp.py module """
@@ -30,6 +34,27 @@ class TestPack(unittest.TestCase):
            w2p_unpack(test_path, unpack_path)
        return

+    def test_admin_compile(self):
+        #apps = ['welcome', 'admin', 'examples']
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        apps = ['welcome']
+        for appname in apps:
+            appname_path = os.path.join(os.getcwd(), 'applications', appname)
+            self.assertEqual(app_compile(appname_path, request), None)
+            # remove any existing test_app
+            new_app = 'test_app_%s' % (appname)
+            if(os.path.exists('applications/%s' % (new_app))):
+                shutil.rmtree('applications/%s' % (new_app))
+            self.assertEqual(app_create(new_app, request), True)
+            self.assertEqual(os.path.exists('applications/test_app_%s/controllers/default.py' % (appname)), True)
+            self.assertEqual(app_cleanup(new_app, request), True)
+            self.assertEqual(app_uninstall(new_app, request), True)
+        self.assertNotEqual(check_new_version(global_settings.web2py_version, WEB2PY_VERSION_URL), -1)
+        return

 if __name__ == '__main__':
    unittest.main()
--- a/gluon/widget.py
+++ b/gluon/widget.py
@@ -13,7 +13,7 @@ from __future__ import print_function

 import datetime
 import sys
-from gluon._compat import StringIO, thread
+from gluon._compat import StringIO, thread, xrange
 import time
 import threading
 import os
@@ -87,7 +87,7 @@ class IO(object):
    def __init__(self):
        """   """

-        self.buffer = cStringIO.StringIO()
+        self.buffer = StringIO()

    def write(self, data):
        """   """
@@ -968,7 +968,7 @@ def console():
        run_system_tests(options)

    if options.quiet:
-        capture = cStringIO.StringIO()
+        capture = StringIO()
        sys.stdout = capture
        logger.setLevel(logging.CRITICAL + 1)
    else:
--- a/web2py.py
+++ b/web2py.py
@@ -28,5 +28,6 @@ if __name__ == '__main__':
            import coverage
            coverage.process_startup()
        except:
+            print('Coverage is not available')
            pass
    gluon.widget.start(cron=True)