Merge branch 'master' of github.com:web2py/web2py

applications/admin/controllers/pythonanywhere.py

@@ -5,7 +5,7 @@ import re
 import gzip
 import tarfile
 from gluon.contrib.simplejsonrpc import ServerProxy
-from gluon._compat import StringIO, ProtocolError
+from gluon._compat import StringIO, ProtocolError, urlencode, urllib2
 
 def deploy():
     response.title = T('Deploy to pythonanywhere')
@@ -26,9 +26,8 @@ def create_account():
         except ProtocolError as error:
             pass
 
-    import urllib, urllib2
     url = 'https://www.pythonanywhere.com/api/web2py/create_account'
-    data = urllib.urlencode(request.vars)
+    data = urlencode(request.vars)
     req = urllib2.Request(url, data)
     
     try:

applications/admin/static/js/web2py.js

@@ -617,7 +617,7 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
             if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {

applications/examples/static/js/web2py.js

@@ -617,7 +617,7 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
             if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {

applications/welcome/controllers/appadmin.py

@@ -392,12 +392,11 @@ def ccache():
                 cache.disk.clear()
                 session.flash += T("Disk Cleared")
         redirect(URL(r=request))
-
+    
     try:
-        from guppy import hpy
+        from pympler.asizeof import asizeof
-        hp = hpy()
     except ImportError:
-        hp = False
+        asizeof = False
 
     import shelve
     import os
@@ -451,9 +450,9 @@ def ccache():
             ram['ratio'] = 0
 
         for key, value in iteritems(cache.ram.storage):
-            if hp:
+            if asizeof:
-                ram['bytes'] += hp.iso(value[1]).size
+                ram['bytes'] += asizeof(value[1])
-                ram['objects'] += hp.iso(value[1]).count
+                ram['objects'] += 1
             ram['entries'] += 1
             if value[0] < ram['oldest']:
                 ram['oldest'] = value[0]
@@ -469,9 +468,9 @@ def ccache():
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:
-                if hp:
+                if asizeof:
-                    disk['bytes'] += hp.iso(value[1]).size
+                    disk['bytes'] += asizeof(value[1])
-                    disk['objects'] += hp.iso(value[1]).count
+                    disk['objects'] += 1
                 disk['entries'] += 1
                 if value[0] < disk['oldest']:
                     disk['oldest'] = value[0]
@@ -511,7 +510,7 @@ def ccache():
         total['keys'] = key_table(total['keys'])
 
     return dict(form=form, total=total,
-                ram=ram, disk=disk, object_stats=hp != False)
+                ram=ram, disk=disk, object_stats=asizeof != False)
 
 
 def table_template(table):

applications/welcome/static/js/web2py.js

@@ -617,7 +617,7 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
             if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {

applications/welcome/views/appadmin.html

@@ -148,7 +148,7 @@
           {{=T.M("(**%.0d MB**)", total['bytes'] / 1048576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("**not available** (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("**not available** (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
     </p>
     <p>
@@ -176,7 +176,7 @@
           {{=T.M("(**%.0d MB**)", ram['bytes'] / 10485576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
     </p>
     <p>
@@ -205,7 +205,7 @@
           {{=T.M("(**%.0d MB**)", disk['bytes'] / 1048576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
       </p>
       <p>

applications/welcome/views/layout.html

@@ -39,7 +39,7 @@
         <span class="navbar-toggler-icon"></span>
       </button>
       <a class="navbar-brand" href="http://web2py.com">web2py</a>
-      <div class="collapse navbar-collapse">
+      <div id="navbarNavDropdown" class="collapse navbar-collapse">
         <ul class="navbar-nav">
           {{for _item in response.menu or []:}}
           {{if len(_item)<4 or not _item[3]:}}
@@ -75,7 +75,7 @@
               {{else:}}
               <a class="dropdown-item" href="{{=URL('default','user/login')}}">{{=T('Login')}}</a>
               <a class="dropdown-item" href="{{=URL('default','user/register')}}">{{=T('Sign up')}}</a>
-              <a class="dropdown-item" href="{{=URL('default','user/request_password')}}">{{=T('Lost Password')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/retrieve_password')}}">{{=T('Lost Password')}}</a>
               {{pass}}
             </div>
           </li>

gluon/contrib/login_methods/rpx_account.py

@@ -13,12 +13,11 @@
 
 import os
 import re
-import urllib
 from gluon import *
 from gluon.tools import fetch
 from gluon.storage import Storage
 import json
-
+from gluon._compat import urlencode
 
 class RPXAccount(object):
 
@@ -83,7 +82,7 @@ class RPXAccount(object):
         token = request.post_vars.token or request.get_vars.token
         if token:
             user = Storage()
-            data = urllib.urlencode(
+            data = urlencode(
                 dict(apiKey=self.api_key, token=token))
             auth_info_json = fetch(self.auth_url + '?' + data)
             auth_info = json.loads(auth_info_json)

gluon/contrib/login_methods/saml2_auth.py

@@ -104,11 +104,12 @@ def obj2dict(obj, processed=None):
                                     types.BuiltinFunctionType,
                                     types.BuiltinMethodType))
 
-def saml2_handler(session, request, config_filename = None):
+def saml2_handler(session, request, config_filename = None, entityid = None):
     config_filename = config_filename or os.path.join(request.folder,'private','sp_conf')
     client = Saml2Client(config_file = config_filename)
-    idps = client.metadata.with_descriptor("idpsso")
+    if not entityid:
-    entityid = idps.keys()[0]
+    	idps = client.metadata.with_descriptor("idpsso")
+    	entityid = idps.keys()[0]
     bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST]
     binding, destination = client.pick_binding(
         "single_sign_on_service", bindings, "idpsso", entity_id=entityid)
@@ -119,7 +120,7 @@ def saml2_handler(session, request, config_filename = None):
     if not request.vars.SAMLResponse:
         req_id, req = client.create_authn_request(destination, binding=binding)
         relay_state = web2py_uuid().replace('-','')
-        session.saml_outstanding_queries = {req_id: request.url}
+        session.saml_outstanding_queries = {req_id: request.url}	
         session.saml_req_id = req_id
         http_args = client.apply_binding(binding, str(req), destination,
                                          relay_state=relay_state)
@@ -145,7 +146,7 @@ class Saml2Auth(object):
             username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
-            ), logout_url=None, change_password_url=None):
+            ), logout_url=None, change_password_url=None, entityid=None):
         self.config_file = config_file
         self.maps = maps
 
@@ -154,9 +155,12 @@ class Saml2Auth(object):
 
         # URL to let users change their password in the IDP system
         self.saml_change_password_url = change_password_url
+	
+	# URL to specify an IDP if using federation metadata or an MDQ
+	self.entityid = entityid
 
     def login_url(self, next="/"):
-        d = saml2_handler(current.session, current.request)
+        d = saml2_handler(current.session, current.request, entityid=self.entityid)
         if 'url' in d:
             redirect(d['url'])
         elif 'error' in d:

gluon/html.py

@@ -1419,20 +1419,21 @@ class LINK(DIV):
 
 class SCRIPT(DIV):
 
-    tag = 'script'
+    tag = b'script'
+    tagname = to_bytes(tag)
 
     def xml(self):
         (fa, co) = self._xml()
-        fa = to_native(fa)
+        fa = to_bytes(fa)
         # no escaping of subcomponents
-        co = '\n'.join([str(component) for component in
+        co = b'\n'.join([to_bytes(component) for component in
                        self.components])
         if co:
             # <script [attributes]><!--//--><![CDATA[//><!--
             # script body
             # //--><!]]></script>
             # return '<%s%s><!--//--><![CDATA[//><!--\n%s\n//--><!]]></%s>' % (self.tag, fa, co, self.tag)
-            return '<%s%s><!--\n%s\n//--></%s>' % (self.tag, fa, co, self.tag)
+            return b'<%s%s><!--\n%s\n//--></%s>' % (self.tagname, fa, co, self.tagname)
         else:
             return DIV.xml(self)
 
@@ -1440,18 +1441,19 @@ class SCRIPT(DIV):
 class STYLE(DIV):
 
     tag = 'style'
+    tagname = to_bytes(tag)
 
     def xml(self):
         (fa, co) = self._xml()
-        fa = to_native(fa)
+        fa = to_bytes(fa)
         # no escaping of subcomponents
-        co = '\n'.join([str(component) for component in
+        co = b'\n'.join([to_bytes(component) for component in
                        self.components])
         if co:
             # <style [attributes]><!--/*--><![CDATA[/*><!--*/
             # style body
             # /*]]>*/--></style>
-            return '<%s%s><!--/*--><![CDATA[/*><!--*/\n%s\n/*]]>*/--></%s>' % (self.tag, fa, co, self.tag)
+            return b'<%s%s><!--/*--><![CDATA[/*><!--*/\n%s\n/*]]>*/--></%s>' % (self.tagname, fa, co, self.tagname)
         else:
             return DIV.xml(self)
 

gluon/languages.py

@@ -431,10 +431,16 @@ class lazyT(object):
         return str(self) if self.M else local_html_escape(str(self), quote=False)
 
     def encode(self, *a, **b):
-        return str(self).encode(*a, **b)
+        if PY2 and a[0] != 'utf8':
+            return to_unicode(str(self)).encode(*a, **b)
+        else:
+            return str(self)
 
     def decode(self, *a, **b):
-        return str(self).decode(*a, **b)
+        if PY2:
+            return str(self).decode(*a, **b)
+        else:
+            return str(self)
 
     def read(self):
         return str(self)

gluon/rocket.py

@@ -1750,7 +1750,7 @@ class WSGIWorker(Worker):
         if self.request_method != 'HEAD':
             try:
                 if self.chunked:
-                    self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data)))
+                    self.conn.sendall(b'%x\r\n%s\r\n' % (len(data), to_bytes(data, 'ISO-8859-1')))
                 else:
                     self.conn.sendall(to_bytes(data))
             except socket.timeout:

gluon/scheduler.py

@@ -525,7 +525,7 @@ class MetaScheduler(threading.Thread):
         self.have_heartbeat = True   # set to False to kill
         self.empty_runs = 0
 
-    def async(self, task):
+    def local_async(self, task):
         """Start the background process.
 
         Args:
@@ -913,7 +913,7 @@ class Scheduler(MetaScheduler):
                     self.w_stats.empty_runs = 0
                     self.w_stats.status = RUNNING
                     self.w_stats.total += 1
-                    self.wrapped_report_task(task, self.async(task))
+                    self.wrapped_report_task(task, self.local_async(task))
                     if not self.w_stats.status == DISABLED:
                         self.w_stats.status = ACTIVE
                 else:

gluon/sqlhtml.py

@@ -15,12 +15,11 @@ Holds:
 """
 
 import datetime
-import urllib
 import re
 import copy
 
 import os
-from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode
+from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode, urlencode
 from gluon.http import HTTP, redirect
 from gluon.html import XmlComponent, truncate_string
 from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG
@@ -3550,7 +3549,7 @@ class SQLTABLE(TABLE):
                             if ref.find('.') >= 0:
                                 tref, fref = ref.split('.')
                                 if hasattr(sqlrows.db[tref], '_primarykey'):
-                                    href = '%s/%s?%s' % (linkto, tref, urllib.urlencode({fref: r}))
+                                    href = '%s/%s?%s' % (linkto, tref, urlencode({fref: r}))
                         r = A(represent(field, r, record), _href=str(href))
                     elif field.represent:
                         if field not in repr_cache:
@@ -3561,7 +3560,7 @@ class SQLTABLE(TABLE):
                 elif linkto and hasattr(field._table, '_primarykey')\
                         and fieldname in field._table._primarykey:
                     # have to test this with multi-key tables
-                    key = urllib.urlencode(dict([
+                    key = urlencode(dict([
                                 ((tablename in record
                                       and isinstance(record, Row)
                                       and isinstance(record[tablename], Row)) and

gluon/tests/test_html.py

@@ -340,18 +340,20 @@ class TestBareHelpers(unittest.TestCase):
 
     def test_SCRIPT(self):
         self.assertEqual(SCRIPT('<>', _a='1', _b='2').xml(),
-                         '''<script a="1" b="2"><!--
+                         b'''<script a="1" b="2"><!--
 <>
 //--></script>''')
         self.assertEqual(SCRIPT('<>').xml(),
-                         '''<script><!--
+                         b'''<script><!--
 <>
 //--></script>''')
         self.assertEqual(SCRIPT().xml(), b'<script></script>')
+        self.assertEqual(SCRIPT(';').xml() + DIV().xml(), 
+                         b'<script><!--\n;\n//--></script><div></div>')
 
     def test_STYLE(self):
         self.assertEqual(STYLE('<>', _a='1', _b='2').xml(),
-                         '<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
+                         b'<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
         # Try to hit : return DIV.xml(self)
         self.assertEqual(STYLE().xml(), b'<style></style>')
 

gluon/tests/test_languages.py

@@ -12,7 +12,9 @@ import tempfile
 import unittest
 
 from gluon import languages
-from gluon._compat import PY2
+from gluon._compat import PY2, to_unicode, to_bytes
+from gluon.storage import Messages
+from gluon.html import SPAN
 
 MP_WORKING = 0
 try:
@@ -108,6 +110,8 @@ class TestTranslations(unittest.TestCase):
         T.force('it')
         self.assertEqual(str(T('Hello World')),
                          'Salve Mondo')
+        self.assertEqual(to_unicode(T('Hello World')),
+                         'Salve Mondo')
 
 class TestDummyApp(unittest.TestCase):
 
@@ -179,3 +183,43 @@ def index():
         for key in ['hello', 'world', '%s %%{shop}', 'ahoy']:
             self.assertTrue(key in en_dict)
             self.assertTrue(key in pt_dict)
+
+class TestMessages(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        messages = Messages(T)
+        messages.update({'email_sent':'Email sent', 'test': "ä"})
+        self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
+
+class TestHTMLTag(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        elem = SPAN(T("Complete"))
+        self.assertEqual(elem.flatten(), "Complete")
+        elem = SPAN(T("Cannot be empty", language="ru"))
+        self.assertEqual(elem.xml(), to_bytes('<span>Пустое значение недопустимо</span>'))
+        self.assertEqual(elem.flatten(), 'Пустое значение недопустимо')

gluon/tools.py

@@ -12,7 +12,7 @@ Auth, Mail, PluginManager and various utilities
 
 import base64
 from functools import reduce
-from gluon._compat import pickle, thread, urllib2, Cookie, StringIO
+from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, urlencode
 from gluon._compat import configparser, MIMEBase, MIMEMultipart, MIMEText, Header
 from gluon._compat import Encoders, Charset, long, urllib_quote, iteritems
 from gluon._compat import to_bytes, to_native, add_charset
@@ -27,7 +27,6 @@ import time
 import fnmatch
 import traceback
 import smtplib
-import urllib
 import email.utils
 import random
 import hmac
@@ -873,7 +872,7 @@ class Recaptcha(DIV):
                 and len(recaptcha_challenge_field)):
             self.errors['captcha'] = self.error_message
             return False
-        params = urllib.urlencode({
+        params = urlencode({
             'privatekey': private_key,
             'remoteip': remoteip,
             'challenge': recaptcha_challenge_field,
@@ -1026,7 +1025,7 @@ class Recaptcha2(DIV):
         if not recaptcha_response_field:
             self.errors['captcha'] = self.error_message
             return False
-        params = urllib.urlencode({
+        params = urlencode({
             'secret': self.private_key,
             'remoteip': remoteip,
             'response': recaptcha_response_field,
@@ -4738,7 +4737,7 @@ def fetch(url, data=None, headers=None,
           user_agent='Mozilla/5.0'):
     headers = headers or {}
     if data is not None:
-        data = urllib.urlencode(data)
+        data = urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
     headers['Cookie'] = ' '.join(

gluon/utf8.py

@@ -11,7 +11,7 @@ Utilities and class for UTF8 strings managing
 ----------------------------------------------
 """
 from __future__ import print_function
-from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native
+from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload
 
 __all__ = ['Utf8']
 

scripts/access.wsgi

@@ -35,11 +35,17 @@
 #
 
 URL_CHECK_ACCESS = 'http://127.0.0.1:8002/%(app)s/default/check_access'
+PY2 = sys.version_info[0] == 2
 
 def allow_access(environ,host):
+    if PY2:
+        import urllib2
+        from urllib import urlencode
+    else:
+        from urllib import request as urllib2
+        from urllib.parse import urlencode
+
     import os
-    import urllib
-    import urllib2
     import datetime
     header = '%s @ %s ' % (datetime.datetime.now(),host) + '='*20
     pprint = '\n'.join('%s:%s' % item for item in environ.items())
@@ -56,7 +62,7 @@ def allow_access(environ,host):
         if key.startswith('HTTP_'):
             headers[key[5:]] = environ[key] # this passes the cookies through!
     try:
-        data = urllib.urlencode({'request_uri':environ['REQUEST_URI']})
+        data = urlencode({'request_uri':environ['REQUEST_URI']})
         request = urllib2.Request(URL_CHECK_ACCESS % dict(app=app),data,headers)
         response = urllib2.urlopen(request).read().strip().lower()
         if response.startswith('true'): return True

scripts/tickets2db.py

@@ -31,7 +31,7 @@ hashes = {}
 
 while 1:
     if request.tickets_db:
-        print "You're storing tickets yet in database"
+        print("You're storing tickets yet in database")
         sys.exit(1)
 
     for file in os.listdir(errors_path):

scripts/tickets2slack.py

@@ -22,7 +22,7 @@ import json
 try:
     import requests
 except ImportError as e:
-    print "missing module 'Requests', aborting."
+    print("missing module 'Requests', aborting.")
     sys.exit(1)
 
 from gluon import URL

scripts/zip_static_files.py

@@ -14,7 +14,7 @@ def zip_static(filelist=[]):
         extension = os.path.splitext(fi)
         extension = len(extension) > 1 and extension[1] or None
         if not extension or extension not in ALLOWED_EXTS:
-            print 'skipping %s' % os.path.basename(fi)
+            print('skipping %s' % os.path.basename(fi))
             continue
         fstats = os.stat(fi)
         atime, mtime = fstats.st_atime, fstats.st_mtime
@@ -23,10 +23,10 @@ def zip_static(filelist=[]):
             zstats = os.stat(gfi)
             zatime, zmtime = zstats.st_atime, zstats.st_mtime
             if zatime == atime and zmtime == mtime:
-                print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
+                print('skipping %s, already gzipped to the latest version' % os.path.basename(fi))
                 continue
-        print 'gzipping %s to %s' % (
+        print('gzipping %s to %s' % (
-            os.path.basename(fi), os.path.basename(gfi))
+            os.path.basename(fi), os.path.basename(gfi)))
         f_in = open(fi, 'rb')
         f_out = gzip.open(gfi, 'wb')
         f_out.writelines(f_in)
@@ -36,7 +36,7 @@ def zip_static(filelist=[]):
         saved = fstats.st_size - os.stat(gfi).st_size
         tsave += saved
 
-    print 'saved %s KB' % (int(tsave) / 1000.0)
+    print('saved %s KB' % (int(tsave) / 1000.0))
 
 if __name__ == '__main__':
     ALLOWED_EXTS = ['.css', '.js']