diff --git a/applications/admin/controllers/pythonanywhere.py b/applications/admin/controllers/pythonanywhere.py index e61b06ab..d9989cee 100644 --- a/applications/admin/controllers/pythonanywhere.py +++ b/applications/admin/controllers/pythonanywhere.py @@ -5,7 +5,7 @@ import re import gzip import tarfile from gluon.contrib.simplejsonrpc import ServerProxy -from gluon._compat import StringIO, ProtocolError +from gluon._compat import StringIO, ProtocolError, urlencode, urllib2 def deploy(): response.title = T('Deploy to pythonanywhere') @@ -26,9 +26,8 @@ def create_account(): except ProtocolError as error: pass - import urllib, urllib2 url = 'https://www.pythonanywhere.com/api/web2py/create_account' - data = urllib.urlencode(request.vars) + data = urlencode(request.vars) req = urllib2.Request(url, data) try: diff --git a/applications/admin/static/js/web2py.js b/applications/admin/static/js/web2py.js index 7131727d..22b9f253 100644 --- a/applications/admin/static/js/web2py.js +++ b/applications/admin/static/js/web2py.js @@ -617,7 +617,7 @@ flash: function (message, status) { var flash = $('.w2p_flash'); web2py.hide_flash(); - flash.html(message).addClass(status); + flash.text(message).addClass(status); if (flash.html()) flash.append(' × ')[animateIn](); }, hide_flash: function () { diff --git a/applications/examples/static/js/web2py.js b/applications/examples/static/js/web2py.js index 7131727d..22b9f253 100644 --- a/applications/examples/static/js/web2py.js +++ b/applications/examples/static/js/web2py.js @@ -617,7 +617,7 @@ flash: function (message, status) { var flash = $('.w2p_flash'); web2py.hide_flash(); - flash.html(message).addClass(status); + flash.text(message).addClass(status); if (flash.html()) flash.append(' × ')[animateIn](); }, hide_flash: function () { diff --git a/applications/welcome/controllers/appadmin.py b/applications/welcome/controllers/appadmin.py index 74240e59..0188ade9 100644 --- a/applications/welcome/controllers/appadmin.py +++ b/applications/welcome/controllers/appadmin.py @@ -392,12 +392,11 @@ def ccache(): cache.disk.clear() session.flash += T("Disk Cleared") redirect(URL(r=request)) - + try: - from guppy import hpy - hp = hpy() + from pympler.asizeof import asizeof except ImportError: - hp = False + asizeof = False import shelve import os @@ -451,9 +450,9 @@ def ccache(): ram['ratio'] = 0 for key, value in iteritems(cache.ram.storage): - if hp: - ram['bytes'] += hp.iso(value[1]).size - ram['objects'] += hp.iso(value[1]).count + if asizeof: + ram['bytes'] += asizeof(value[1]) + ram['objects'] += 1 ram['entries'] += 1 if value[0] < ram['oldest']: ram['oldest'] = value[0] @@ -469,9 +468,9 @@ def ccache(): except (KeyError, ZeroDivisionError): disk['ratio'] = 0 else: - if hp: - disk['bytes'] += hp.iso(value[1]).size - disk['objects'] += hp.iso(value[1]).count + if asizeof: + disk['bytes'] += asizeof(value[1]) + disk['objects'] += 1 disk['entries'] += 1 if value[0] < disk['oldest']: disk['oldest'] = value[0] @@ -511,7 +510,7 @@ def ccache(): total['keys'] = key_table(total['keys']) return dict(form=form, total=total, - ram=ram, disk=disk, object_stats=hp != False) + ram=ram, disk=disk, object_stats=asizeof != False) def table_template(table): diff --git a/applications/welcome/static/js/web2py.js b/applications/welcome/static/js/web2py.js index 7131727d..22b9f253 100644 --- a/applications/welcome/static/js/web2py.js +++ b/applications/welcome/static/js/web2py.js @@ -617,7 +617,7 @@ flash: function (message, status) { var flash = $('.w2p_flash'); web2py.hide_flash(); - flash.html(message).addClass(status); + flash.text(message).addClass(status); if (flash.html()) flash.append(' × ')[animateIn](); }, hide_flash: function () { diff --git a/applications/welcome/views/appadmin.html b/applications/welcome/views/appadmin.html index 59b85563..8a9d3400 100644 --- a/applications/welcome/views/appadmin.html +++ b/applications/welcome/views/appadmin.html @@ -148,7 +148,7 @@ {{=T.M("(**%.0d MB**)", total['bytes'] / 1048576)}} {{pass}} {{else:}} - {{=T.M("**not available** (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}} + {{=T.M("**not available** (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}} {{pass}}

@@ -176,7 +176,7 @@ {{=T.M("(**%.0d MB**)", ram['bytes'] / 10485576)}} {{pass}} {{else:}} - {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}} + {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}} {{pass}}

@@ -205,7 +205,7 @@ {{=T.M("(**%.0d MB**)", disk['bytes'] / 1048576)}} {{pass}} {{else:}} - {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}} + {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}} {{pass}}

diff --git a/applications/welcome/views/layout.html b/applications/welcome/views/layout.html index ab477415..54dcf8d2 100644 --- a/applications/welcome/views/layout.html +++ b/applications/welcome/views/layout.html @@ -39,7 +39,7 @@ web2py -

+ diff --git a/gluon/contrib/login_methods/rpx_account.py b/gluon/contrib/login_methods/rpx_account.py index dcfdc114..e79d17df 100644 --- a/gluon/contrib/login_methods/rpx_account.py +++ b/gluon/contrib/login_methods/rpx_account.py @@ -13,12 +13,11 @@ import os import re -import urllib from gluon import * from gluon.tools import fetch from gluon.storage import Storage import json - +from gluon._compat import urlencode class RPXAccount(object): @@ -83,7 +82,7 @@ class RPXAccount(object): token = request.post_vars.token or request.get_vars.token if token: user = Storage() - data = urllib.urlencode( + data = urlencode( dict(apiKey=self.api_key, token=token)) auth_info_json = fetch(self.auth_url + '?' + data) auth_info = json.loads(auth_info_json) diff --git a/gluon/contrib/login_methods/saml2_auth.py b/gluon/contrib/login_methods/saml2_auth.py index 462d9845..ea650463 100644 --- a/gluon/contrib/login_methods/saml2_auth.py +++ b/gluon/contrib/login_methods/saml2_auth.py @@ -104,11 +104,12 @@ def obj2dict(obj, processed=None): types.BuiltinFunctionType, types.BuiltinMethodType)) -def saml2_handler(session, request, config_filename = None): +def saml2_handler(session, request, config_filename = None, entityid = None): config_filename = config_filename or os.path.join(request.folder,'private','sp_conf') client = Saml2Client(config_file = config_filename) - idps = client.metadata.with_descriptor("idpsso") - entityid = idps.keys()[0] + if not entityid: + idps = client.metadata.with_descriptor("idpsso") + entityid = idps.keys()[0] bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST] binding, destination = client.pick_binding( "single_sign_on_service", bindings, "idpsso", entity_id=entityid) @@ -119,7 +120,7 @@ def saml2_handler(session, request, config_filename = None): if not request.vars.SAMLResponse: req_id, req = client.create_authn_request(destination, binding=binding) relay_state = web2py_uuid().replace('-','') - session.saml_outstanding_queries = {req_id: request.url} + session.saml_outstanding_queries = {req_id: request.url} session.saml_req_id = req_id http_args = client.apply_binding(binding, str(req), destination, relay_state=relay_state) @@ -145,7 +146,7 @@ class Saml2Auth(object): username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], - ), logout_url=None, change_password_url=None): + ), logout_url=None, change_password_url=None, entityid=None): self.config_file = config_file self.maps = maps @@ -154,9 +155,12 @@ class Saml2Auth(object): # URL to let users change their password in the IDP system self.saml_change_password_url = change_password_url + + # URL to specify an IDP if using federation metadata or an MDQ + self.entityid = entityid def login_url(self, next="/"): - d = saml2_handler(current.session, current.request) + d = saml2_handler(current.session, current.request, entityid=self.entityid) if 'url' in d: redirect(d['url']) elif 'error' in d: diff --git a/gluon/html.py b/gluon/html.py index 73eae9b8..4adf287c 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -1419,20 +1419,21 @@ class LINK(DIV): class SCRIPT(DIV): - tag = 'script' + tag = b'script' + tagname = to_bytes(tag) def xml(self): (fa, co) = self._xml() - fa = to_native(fa) + fa = to_bytes(fa) # no escaping of subcomponents - co = '\n'.join([str(component) for component in + co = b'\n'.join([to_bytes(component) for component in self.components]) if co: # # return '<%s%s>' % (self.tag, fa, co, self.tag) - return '<%s%s>' % (self.tag, fa, co, self.tag) + return b'<%s%s>' % (self.tagname, fa, co, self.tagname) else: return DIV.xml(self) @@ -1440,18 +1441,19 @@ class SCRIPT(DIV): class STYLE(DIV): tag = 'style' + tagname = to_bytes(tag) def xml(self): (fa, co) = self._xml() - fa = to_native(fa) + fa = to_bytes(fa) # no escaping of subcomponents - co = '\n'.join([str(component) for component in + co = b'\n'.join([to_bytes(component) for component in self.components]) if co: # - return '<%s%s>' % (self.tag, fa, co, self.tag) + return b'<%s%s>' % (self.tagname, fa, co, self.tagname) else: return DIV.xml(self) diff --git a/gluon/languages.py b/gluon/languages.py index 0d8bd4a2..293ecac9 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -431,10 +431,16 @@ class lazyT(object): return str(self) if self.M else local_html_escape(str(self), quote=False) def encode(self, *a, **b): - return str(self).encode(*a, **b) + if PY2 and a[0] != 'utf8': + return to_unicode(str(self)).encode(*a, **b) + else: + return str(self) def decode(self, *a, **b): - return str(self).decode(*a, **b) + if PY2: + return str(self).decode(*a, **b) + else: + return str(self) def read(self): return str(self) diff --git a/gluon/rocket.py b/gluon/rocket.py index e3faf0bd..d76da94e 100644 --- a/gluon/rocket.py +++ b/gluon/rocket.py @@ -1750,7 +1750,7 @@ class WSGIWorker(Worker): if self.request_method != 'HEAD': try: if self.chunked: - self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data))) + self.conn.sendall(b'%x\r\n%s\r\n' % (len(data), to_bytes(data, 'ISO-8859-1'))) else: self.conn.sendall(to_bytes(data)) except socket.timeout: diff --git a/gluon/scheduler.py b/gluon/scheduler.py index 710815db..79d22c0f 100644 --- a/gluon/scheduler.py +++ b/gluon/scheduler.py @@ -525,7 +525,7 @@ class MetaScheduler(threading.Thread): self.have_heartbeat = True # set to False to kill self.empty_runs = 0 - def async(self, task): + def local_async(self, task): """Start the background process. Args: @@ -913,7 +913,7 @@ class Scheduler(MetaScheduler): self.w_stats.empty_runs = 0 self.w_stats.status = RUNNING self.w_stats.total += 1 - self.wrapped_report_task(task, self.async(task)) + self.wrapped_report_task(task, self.local_async(task)) if not self.w_stats.status == DISABLED: self.w_stats.status = ACTIVE else: diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 41637bd8..2354983b 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -15,12 +15,11 @@ Holds: """ import datetime -import urllib import re import copy import os -from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode +from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode, urlencode from gluon.http import HTTP, redirect from gluon.html import XmlComponent, truncate_string from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG @@ -3550,7 +3549,7 @@ class SQLTABLE(TABLE): if ref.find('.') >= 0: tref, fref = ref.split('.') if hasattr(sqlrows.db[tref], '_primarykey'): - href = '%s/%s?%s' % (linkto, tref, urllib.urlencode({fref: r})) + href = '%s/%s?%s' % (linkto, tref, urlencode({fref: r})) r = A(represent(field, r, record), _href=str(href)) elif field.represent: if field not in repr_cache: @@ -3561,7 +3560,7 @@ class SQLTABLE(TABLE): elif linkto and hasattr(field._table, '_primarykey')\ and fieldname in field._table._primarykey: # have to test this with multi-key tables - key = urllib.urlencode(dict([ + key = urlencode(dict([ ((tablename in record and isinstance(record, Row) and isinstance(record[tablename], Row)) and diff --git a/gluon/tests/test_html.py b/gluon/tests/test_html.py index 26693efb..4cca19e2 100644 --- a/gluon/tests/test_html.py +++ b/gluon/tests/test_html.py @@ -340,18 +340,20 @@ class TestBareHelpers(unittest.TestCase): def test_SCRIPT(self): self.assertEqual(SCRIPT('<>', _a='1', _b='2').xml(), - '''''') self.assertEqual(SCRIPT('<>').xml(), - '''''') self.assertEqual(SCRIPT().xml(), b'') + self.assertEqual(SCRIPT(';').xml() + DIV().xml(), + b'
') def test_STYLE(self): self.assertEqual(STYLE('<>', _a='1', _b='2').xml(), - '') + b'') # Try to hit : return DIV.xml(self) self.assertEqual(STYLE().xml(), b'') diff --git a/gluon/tests/test_languages.py b/gluon/tests/test_languages.py index cb2fb6bc..c996bd6a 100644 --- a/gluon/tests/test_languages.py +++ b/gluon/tests/test_languages.py @@ -12,7 +12,9 @@ import tempfile import unittest from gluon import languages -from gluon._compat import PY2 +from gluon._compat import PY2, to_unicode, to_bytes +from gluon.storage import Messages +from gluon.html import SPAN MP_WORKING = 0 try: @@ -108,6 +110,8 @@ class TestTranslations(unittest.TestCase): T.force('it') self.assertEqual(str(T('Hello World')), 'Salve Mondo') + self.assertEqual(to_unicode(T('Hello World')), + 'Salve Mondo') class TestDummyApp(unittest.TestCase): @@ -179,3 +183,43 @@ def index(): for key in ['hello', 'world', '%s %%{shop}', 'ahoy']: self.assertTrue(key in en_dict) self.assertTrue(key in pt_dict) + +class TestMessages(unittest.TestCase): + + def setUp(self): + if os.path.isdir('gluon'): + self.langpath = 'applications/welcome/languages' + else: + self.langpath = os.path.realpath( + '../../applications/welcome/languages') + self.http_accept_language = 'en' + + def tearDown(self): + pass + + def test_decode(self): + T = languages.translator(self.langpath, self.http_accept_language) + messages = Messages(T) + messages.update({'email_sent':'Email sent', 'test': "ä"}) + self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent') + +class TestHTMLTag(unittest.TestCase): + + def setUp(self): + if os.path.isdir('gluon'): + self.langpath = 'applications/welcome/languages' + else: + self.langpath = os.path.realpath( + '../../applications/welcome/languages') + self.http_accept_language = 'en' + + def tearDown(self): + pass + + def test_decode(self): + T = languages.translator(self.langpath, self.http_accept_language) + elem = SPAN(T("Complete")) + self.assertEqual(elem.flatten(), "Complete") + elem = SPAN(T("Cannot be empty", language="ru")) + self.assertEqual(elem.xml(), to_bytes('Пустое значение недопустимо')) + self.assertEqual(elem.flatten(), 'Пустое значение недопустимо') diff --git a/gluon/tools.py b/gluon/tools.py index 716dafcd..c138242d 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -12,7 +12,7 @@ Auth, Mail, PluginManager and various utilities import base64 from functools import reduce -from gluon._compat import pickle, thread, urllib2, Cookie, StringIO +from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, urlencode from gluon._compat import configparser, MIMEBase, MIMEMultipart, MIMEText, Header from gluon._compat import Encoders, Charset, long, urllib_quote, iteritems from gluon._compat import to_bytes, to_native, add_charset @@ -27,7 +27,6 @@ import time import fnmatch import traceback import smtplib -import urllib import email.utils import random import hmac @@ -873,7 +872,7 @@ class Recaptcha(DIV): and len(recaptcha_challenge_field)): self.errors['captcha'] = self.error_message return False - params = urllib.urlencode({ + params = urlencode({ 'privatekey': private_key, 'remoteip': remoteip, 'challenge': recaptcha_challenge_field, @@ -1026,7 +1025,7 @@ class Recaptcha2(DIV): if not recaptcha_response_field: self.errors['captcha'] = self.error_message return False - params = urllib.urlencode({ + params = urlencode({ 'secret': self.private_key, 'remoteip': remoteip, 'response': recaptcha_response_field, @@ -4738,7 +4737,7 @@ def fetch(url, data=None, headers=None, user_agent='Mozilla/5.0'): headers = headers or {} if data is not None: - data = urllib.urlencode(data) + data = urlencode(data) if user_agent: headers['User-agent'] = user_agent headers['Cookie'] = ' '.join( diff --git a/gluon/utf8.py b/gluon/utf8.py index 21fd12c4..34bfa87f 100644 --- a/gluon/utf8.py +++ b/gluon/utf8.py @@ -11,7 +11,7 @@ Utilities and class for UTF8 strings managing ---------------------------------------------- """ from __future__ import print_function -from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native +from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload __all__ = ['Utf8'] diff --git a/scripts/access.wsgi b/scripts/access.wsgi index 86285070..8cc6b40b 100644 --- a/scripts/access.wsgi +++ b/scripts/access.wsgi @@ -35,11 +35,17 @@ # URL_CHECK_ACCESS = 'http://127.0.0.1:8002/%(app)s/default/check_access' +PY2 = sys.version_info[0] == 2 def allow_access(environ,host): + if PY2: + import urllib2 + from urllib import urlencode + else: + from urllib import request as urllib2 + from urllib.parse import urlencode + import os - import urllib - import urllib2 import datetime header = '%s @ %s ' % (datetime.datetime.now(),host) + '='*20 pprint = '\n'.join('%s:%s' % item for item in environ.items()) @@ -56,7 +62,7 @@ def allow_access(environ,host): if key.startswith('HTTP_'): headers[key[5:]] = environ[key] # this passes the cookies through! try: - data = urllib.urlencode({'request_uri':environ['REQUEST_URI']}) + data = urlencode({'request_uri':environ['REQUEST_URI']}) request = urllib2.Request(URL_CHECK_ACCESS % dict(app=app),data,headers) response = urllib2.urlopen(request).read().strip().lower() if response.startswith('true'): return True diff --git a/scripts/tickets2db.py b/scripts/tickets2db.py index ecb1b4cd..da499483 100755 --- a/scripts/tickets2db.py +++ b/scripts/tickets2db.py @@ -31,7 +31,7 @@ hashes = {} while 1: if request.tickets_db: - print "You're storing tickets yet in database" + print("You're storing tickets yet in database") sys.exit(1) for file in os.listdir(errors_path): diff --git a/scripts/tickets2slack.py b/scripts/tickets2slack.py index 692dce2b..c9d181bb 100755 --- a/scripts/tickets2slack.py +++ b/scripts/tickets2slack.py @@ -22,7 +22,7 @@ import json try: import requests except ImportError as e: - print "missing module 'Requests', aborting." + print("missing module 'Requests', aborting.") sys.exit(1) from gluon import URL diff --git a/scripts/zip_static_files.py b/scripts/zip_static_files.py index 57054e5b..170ef12a 100644 --- a/scripts/zip_static_files.py +++ b/scripts/zip_static_files.py @@ -14,7 +14,7 @@ def zip_static(filelist=[]): extension = os.path.splitext(fi) extension = len(extension) > 1 and extension[1] or None if not extension or extension not in ALLOWED_EXTS: - print 'skipping %s' % os.path.basename(fi) + print('skipping %s' % os.path.basename(fi)) continue fstats = os.stat(fi) atime, mtime = fstats.st_atime, fstats.st_mtime @@ -23,10 +23,10 @@ def zip_static(filelist=[]): zstats = os.stat(gfi) zatime, zmtime = zstats.st_atime, zstats.st_mtime if zatime == atime and zmtime == mtime: - print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi) + print('skipping %s, already gzipped to the latest version' % os.path.basename(fi)) continue - print 'gzipping %s to %s' % ( - os.path.basename(fi), os.path.basename(gfi)) + print('gzipping %s to %s' % ( + os.path.basename(fi), os.path.basename(gfi))) f_in = open(fi, 'rb') f_out = gzip.open(gfi, 'wb') f_out.writelines(f_in) @@ -36,7 +36,7 @@ def zip_static(filelist=[]): saved = fstats.st_size - os.stat(gfi).st_size tsave += saved - print 'saved %s KB' % (int(tsave) / 1000.0) + print('saved %s KB' % (int(tsave) / 1000.0)) if __name__ == '__main__': ALLOWED_EXTS = ['.css', '.js']