testing revert in pydal

Added row class to form-group so form becomes inline in Bootstrap v4

.travis.yml

@@ -10,12 +10,16 @@ python:
   - '2.7'
   - '3.5'
   - '3.6'
+  - '3.6-dev'
+  - '3.7-dev'
   - 'pypy-5.3.1'
   - 'pypy3.5-5.7.1-beta'
 
 matrix:
   allow_failures:
     - python: 'pypy3.5-5.7.1-beta'
+    - python: '3.6-dev'
+    - python: '3.7-dev'
 
 install:
   - pip install -e .

applications/admin/controllers/default.py

@@ -963,8 +963,7 @@ def edit_language():
         form = SPAN(strings['__corrupted__'], _class='error')
         return dict(filename=filename, form=form)
 
-    keys = sorted(strings.keys(), lambda x, y: cmp(
-        unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower()))
+    keys = sorted(strings.keys(), key=lambda x: to_native(x).lower())
     rows = []
     rows.append(H2(T('Original/Translation')))
 

applications/admin/controllers/pythonanywhere.py

@@ -5,7 +5,7 @@ import re
 import gzip
 import tarfile
 from gluon.contrib.simplejsonrpc import ServerProxy
-from gluon._compat import StringIO, ProtocolError
+from gluon._compat import StringIO, ProtocolError, urlencode, urllib2
 
 def deploy():
     response.title = T('Deploy to pythonanywhere')
@@ -26,9 +26,8 @@ def create_account():
         except ProtocolError as error:
             pass
 
-    import urllib, urllib2
     url = 'https://www.pythonanywhere.com/api/web2py/create_account'
-    data = urllib.urlencode(request.vars)
+    data = urlencode(request.vars)
     req = urllib2.Request(url, data)
     
     try:

applications/admin/static/js/web2py.js

@@ -617,7 +617,7 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
             if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {

applications/examples/static/js/web2py.js

@@ -617,7 +617,7 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
             if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {

applications/welcome/controllers/appadmin.py

@@ -392,12 +392,11 @@ def ccache():
                 cache.disk.clear()
                 session.flash += T("Disk Cleared")
         redirect(URL(r=request))
-
+    
     try:
-        from guppy import hpy
-        hp = hpy()
+        from pympler.asizeof import asizeof
     except ImportError:
-        hp = False
+        asizeof = False
 
     import shelve
     import os
@@ -451,9 +450,9 @@ def ccache():
             ram['ratio'] = 0
 
         for key, value in iteritems(cache.ram.storage):
-            if hp:
-                ram['bytes'] += hp.iso(value[1]).size
-                ram['objects'] += hp.iso(value[1]).count
+            if asizeof:
+                ram['bytes'] += asizeof(value[1])
+                ram['objects'] += 1
             ram['entries'] += 1
             if value[0] < ram['oldest']:
                 ram['oldest'] = value[0]
@@ -469,9 +468,9 @@ def ccache():
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:
-                if hp:
-                    disk['bytes'] += hp.iso(value[1]).size
-                    disk['objects'] += hp.iso(value[1]).count
+                if asizeof:
+                    disk['bytes'] += asizeof(value[1])
+                    disk['objects'] += 1
                 disk['entries'] += 1
                 if value[0] < disk['oldest']:
                     disk['oldest'] = value[0]
@@ -511,7 +510,7 @@ def ccache():
         total['keys'] = key_table(total['keys'])
 
     return dict(form=form, total=total,
-                ram=ram, disk=disk, object_stats=hp != False)
+                ram=ram, disk=disk, object_stats=asizeof != False)
 
 
 def table_template(table):

applications/welcome/languages/es.py

@@ -8,8 +8,8 @@
 '%s %%{row} deleted': '%s %%{fila} %%{eliminada}',
 '%s %%{row} updated': '%s %%{fila} %%{actualizada}',
 '%s selected': '%s %%{seleccionado}',
-'%Y-%m-%d': '%d/%m/%A',
-'%Y-%m-%d %H:%M:%S': '%d/%m/%A %H:%M:%S',
+'%Y-%m-%d': '%d/%m/%Y',
+'%Y-%m-%d %H:%M:%S': '%d/%m/%Y %H:%M:%S',
 '(**%.0d MB**)': '(**%.0d MB**)',
 '(something like "it-it")': '(algo como "it-it")',
 '**%(items)s** %%{item(items)}, **%(bytes)s** %%{byte(bytes)}': '**%(items)s** %%{item(items)}, **%(bytes)s** %%{byte(bytes)}',

applications/welcome/models/db.py

@@ -130,7 +130,7 @@ response.google_analytics_id = configuration.get('google.analytics_id')
 # -------------------------------------------------------------------------
 if configuration.get('scheduler.enabled'):
     from gluon.scheduler import Scheduler
-    scheduler = Scheduler(db, heartbeat=configure.get('heartbeat'))
+    scheduler = Scheduler(db, heartbeat=configuration.get('heartbeat'))
 
 # -------------------------------------------------------------------------
 # Define your tables below (or better in another model file) for example

applications/welcome/static/css/web2py-bootstrap4.css

@@ -1,3 +1,11 @@
+ul.navbar-nav.navbar-right {
+    position: absolute;
+    right: 100px;
+}
+label, th {
+    font-weigth: bold;
+    white-space: nowrap;
+}
 div.w2p_flash {
     background-image: none;
     border-radius: 4px;
@@ -313,3 +321,20 @@ td.w2p_fc,
 input[type=checkbox], input[type=radio] {
     margin: 4px 4px 0 0;
 }
+
+/* for backward compatbility with pre-font-awesome */
+.icon.plus,.icon.arrowleft,.icon.download,.icon.trash,.icon.pen,.icon.arrowright,.icon.magnifier {
+    display: inline-block;
+    font: normal normal normal 14px/1 FontAwesome;
+    font-size: inherit;
+    text-rendering: auto;
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+}
+.icon.plus:before { content: "\f067";}
+.icon.arrowleft:before { content: "\f060";}
+.icon.download:before { content: "\f019";}
+.icon.trash:before { content: "\f1f8";}
+.icon.pen:before { content: "\f040";}
+.icon.arrowright:before { content: "\f061";}
+.icon.magnifier:before { content: "\f002";}

applications/welcome/static/js/share.js

@@ -1,44 +0,0 @@
-/**
-
-   Created and copyrighted by Massimo Di Pierro <massimo.dipierro@gmail.com>
-   (MIT license)  
-
-   Example:
-
-   <script src="share.js"></script>
-
-**/
-
-jQuery(function(){
-	var script_source = jQuery('script[src*="share.js"]').attr('src');
-        var params = function(name,default_value) {
-            var match = RegExp('[?&]' + name + '=([^&]*)').exec(script_source);
-            return match && decodeURIComponent(match[1].replace(/\+/g, ' '))||default_value;
-        }
-	var path = params('static','social');
-	var url = encodeURIComponent(window.location.href);
-	var host =  window.location.hostname;
-	var title = escape(jQuery('title').text());
-	var twit = 'http://twitter.com/home?status='+title+'%20'+url;
-	var facebook = 'http://www.facebook.com/sharer.php?u='+url;
-	var gplus = 'https://plus.google.com/share?url='+url;
-	var tbar = '<div id="socialdrawer"><span>Share<br/></span><div id="sicons"><a href="'+twit+'" id="twit" title="Share on twitter"><img src="'+path+'/twitter.png"  alt="Share on Twitter" width="32" height="32" /></a><a href="'+facebook+'" id="facebook" title="Share on Facebook"><img src="'+path+'/facebook.png"  alt="Share on facebook" width="32" height="32" /></a><a href="'+gplus+'" id="gplus" title="Share on Google Plus"><img src="'+path+'/gplus-32.png"  alt="Share on Google Plus" width="32" height="32" /></a></div></div>';	
-	// Add the share tool bar.
-	jQuery('body').append(tbar); 
-	var st = jQuery('#socialdrawer');
-	st.css({'opacity':'.7','z-index':'3000','background':'#FFF','border':'solid 1px #666','border-width':' 1px 0 0 1px','height':'20px','width':'40px','position':'fixed','bottom':'0','right':'0','padding':'2px 5px','overflow':'hidden','-webkit-border-top-left-radius':' 12px','-moz-border-radius-topleft':' 12px','border-top-left-radius':' 12px','-moz-box-shadow':' -3px -3px 3px rgba(0,0,0,0.5)','-webkit-box-shadow':' -3px -3px 3px rgba(0,0,0,0.5)','box-shadow':' -3px -3px 3px rgba(0,0,0,0.5)'});
-	jQuery('#socialdrawer a').css({'float':'left','width':'32px','margin':'3px 2px 2px 2px','padding':'0','cursor':'pointer'});
-	jQuery('#socialdrawer span').css({'float':'left','margin':'2px 3px','text-shadow':' 1px 1px 1px #FFF','color':'#444','font-size':'12px','line-height':'1em'});
-        jQuery('#socialdrawer img').hide();
-	// hover
-	st.click(function(){
-		jQuery(this).animate({height:'40px', width:'160px', opacity: 0.95}, 300);
-		jQuery('#socialdrawer img').show();
-	    });
-	//leave
-	st.mouseleave(function(){ 
-	    st.animate({height:'20px', width: '40px', opacity: .7}, 300); 
-	    jQuery('#socialdrawer img').hide();
-	    return false;
-	    }  );
-    });

applications/welcome/static/js/web2py.js

@@ -617,7 +617,7 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
             if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {

applications/welcome/views/appadmin.html

@@ -148,7 +148,7 @@
           {{=T.M("(**%.0d MB**)", total['bytes'] / 1048576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("**not available** (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("**not available** (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
     </p>
     <p>
@@ -176,7 +176,7 @@
           {{=T.M("(**%.0d MB**)", ram['bytes'] / 10485576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
     </p>
     <p>
@@ -205,7 +205,7 @@
           {{=T.M("(**%.0d MB**)", disk['bytes'] / 1048576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
       </p>
       <p>

applications/welcome/views/default/index.html

@@ -23,7 +23,7 @@
 </ol>
 <center style="padding:50px">
   <a class="btn btn-primary" href="{{=URL('admin','default','index')}}">
-    <i class="glyphicon glyphicon-cog"></i>
+    <i class="fa fa-cog"></i>
     {{=T("admin")}}
   </a>
   <a class="btn btn-secondary" href="{{=URL('examples','default','index')}}">{{=T("Online examples")}}</a>

applications/welcome/views/default/user.html

@@ -10,10 +10,10 @@
         {{
         if request.args(0)=='login':
             if not 'register' in auth.settings.actions_disabled:
-                form.add_button(T('Sign Up'),URL(args='register', vars={'_next': request.vars._next} if request.vars._next else None),_class='btn btn-default')
+                form.add_button(T('Sign Up'),URL(args='register', vars={'_next': request.vars._next} if request.vars._next else None),_class='btn btn-default btn-secondary')
             pass
             if not 'request_reset_password' in auth.settings.actions_disabled:
-                form.add_button(T('Lost Password'),URL(args='request_reset_password'),_class='btn btn-default')
+                form.add_button(T('Lost Password'),URL(args='request_reset_password'),_class='btn btn-default btn-secondary')
             pass
         pass
         =form

applications/welcome/views/layout.html

@@ -20,6 +20,7 @@
          http://google.com/webmasters -->
     <meta name="google-site-verification" content="">
     <!-- include stylesheets -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"/>
     <link rel="stylesheet" href="{{=URL('static','css/bootstrap.min.css')}}"/>
     <link rel="stylesheet" href="{{=URL('static','css/web2py-bootstrap4.css')}}"/>
     <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
@@ -39,7 +40,7 @@
         <span class="navbar-toggler-icon"></span>
       </button>
       <a class="navbar-brand" href="http://web2py.com">web2py</a>
-      <div class="collapse navbar-collapse">
+      <div id="navbarNavDropdown" class="collapse navbar-collapse">
         <ul class="navbar-nav">
           {{for _item in response.menu or []:}}
           {{if len(_item)<4 or not _item[3]:}}
@@ -75,7 +76,7 @@
               {{else:}}
               <a class="dropdown-item" href="{{=URL('default','user/login')}}">{{=T('Login')}}</a>
               <a class="dropdown-item" href="{{=URL('default','user/register')}}">{{=T('Sign up')}}</a>
-              <a class="dropdown-item" href="{{=URL('default','user/request_password')}}">{{=T('Lost Password')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/retrieve_password')}}">{{=T('Lost Password')}}</a>
               {{pass}}
             </div>
           </li>
@@ -122,7 +123,5 @@
       });
     </script>
     {{pass}}
-    <!-- Share ============================y============ -->
-    <script src="{{=URL('static','js/share.js',vars=dict(static=URL('static','images')))}}"></script>
   </body>
 </html>

appveyor.yml

@@ -10,6 +10,10 @@ environment:
       COVERAGE_PROCESS_START: gluon/tests/coverage.ini
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:/Python36"
+      COVERAGE_PROCESS_START: gluon/tests/coverage.ini
+      PYTHON_ARCH: "64"
+
 clone_depth: 50
 
 init:
@@ -20,7 +24,6 @@ install:
   - python -m ensurepip
   - pip install codecov
   - git submodule update --init --recursive
-  - pip install pycrypto
   # Check that we have the expected version and architecture for Python
   - "python --version"
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""

gluon/_compat.py

@@ -26,7 +26,7 @@ if PY2:
     from email.MIMEText import MIMEText
     from email.Charset import add_charset, QP as charset_QP
     from urllib import FancyURLopener, urlencode, urlopen
-    from urllib import quote as urllib_quote, unquote as urllib_unquote
+    from urllib import quote as urllib_quote, unquote as urllib_unquote, quote_plus as urllib_quote_plus
     from string import maketrans
     from types import ClassType
     import cgi
@@ -96,7 +96,7 @@ else:
     from email.header import Header
     from email.charset import Charset, add_charset, QP as charset_QP
     from urllib.request import FancyURLopener, urlopen
-    from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode
+    from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode, quote_plus as urllib_quote_plus
     from http import cookiejar as cookielib
     from xmlrpc.client import ProtocolError
     import html # warning, this is the python3 module and not the web2py html module

gluon/contrib/login_methods/janrain_account.py

@@ -78,10 +78,13 @@ class RPXAccount(object):
 
     def get_user(self):
         request = self.request
-        if request.vars.token:
+        # Janrain now sends the token via both a POST body and the query
+        # string, so we should keep only one of these.
+        token = request.post_vars.token or request.get_vars.token
+        if token:
             user = Storage()
             data = urllib.urlencode(
-                dict(apiKey=self.api_key, token=request.vars.token))
+                dict(apiKey=self.api_key, token=token))
             auth_info_json = fetch(self.auth_url + '?' + data)
             auth_info = json.loads(auth_info_json)
 

gluon/contrib/login_methods/rpx_account.py

@@ -13,12 +13,11 @@
 
 import os
 import re
-import urllib
 from gluon import *
 from gluon.tools import fetch
 from gluon.storage import Storage
 import json
-
+from gluon._compat import urlencode
 
 class RPXAccount(object):
 
@@ -78,10 +77,13 @@ class RPXAccount(object):
 
     def get_user(self):
         request = self.request
-        if request.vars.token:
+        # Janrain now sends the token via both a POST body and the query
+        # string, so we should keep only one of these.
+        token = request.post_vars.token or request.get_vars.token
+        if token:
             user = Storage()
-            data = urllib.urlencode(
-                dict(apiKey=self.api_key, token=request.vars.token))
+            data = urlencode(
+                dict(apiKey=self.api_key, token=token))
             auth_info_json = fetch(self.auth_url + '?' + data)
             auth_info = json.loads(auth_info_json)
 

gluon/contrib/login_methods/saml2_auth.py

@@ -104,11 +104,12 @@ def obj2dict(obj, processed=None):
                                     types.BuiltinFunctionType,
                                     types.BuiltinMethodType))
 
-def saml2_handler(session, request, config_filename = None):
+def saml2_handler(session, request, config_filename = None, entityid = None):
     config_filename = config_filename or os.path.join(request.folder,'private','sp_conf')
     client = Saml2Client(config_file = config_filename)
-    idps = client.metadata.with_descriptor("idpsso")
-    entityid = idps.keys()[0]
+    if not entityid:
+    	idps = client.metadata.with_descriptor("idpsso")
+    	entityid = idps.keys()[0]
     bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST]
     binding, destination = client.pick_binding(
         "single_sign_on_service", bindings, "idpsso", entity_id=entityid)
@@ -119,7 +120,7 @@ def saml2_handler(session, request, config_filename = None):
     if not request.vars.SAMLResponse:
         req_id, req = client.create_authn_request(destination, binding=binding)
         relay_state = web2py_uuid().replace('-','')
-        session.saml_outstanding_queries = {req_id: request.url}
+        session.saml_outstanding_queries = {req_id: request.url}	
         session.saml_req_id = req_id
         http_args = client.apply_binding(binding, str(req), destination,
                                          relay_state=relay_state)
@@ -145,7 +146,7 @@ class Saml2Auth(object):
             username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
-            ), logout_url=None, change_password_url=None):
+            ), logout_url=None, change_password_url=None, entityid=None):
         self.config_file = config_file
         self.maps = maps
 
@@ -154,9 +155,12 @@ class Saml2Auth(object):
 
         # URL to let users change their password in the IDP system
         self.saml_change_password_url = change_password_url
+	
+	# URL to specify an IDP if using federation metadata or an MDQ
+	self.entityid = entityid
 
     def login_url(self, next="/"):
-        d = saml2_handler(current.session, current.request)
+        d = saml2_handler(current.session, current.request, entityid=self.entityid)
         if 'url' in d:
             redirect(d['url'])
         elif 'error' in d:

gluon/globals.py

@@ -464,7 +464,7 @@ class Response(Storage):
         for meta in iteritems((self.meta or {})):
             k, v = meta
             if isinstance(v, dict):
-                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
+                s += '<meta' + ''.join(' %s="%s"' % (to_native(xmlescape(key)),
                                                      to_native(xmlescape(v[key]))) for key in v) + ' />\n'
             else:
                 s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))

gluon/html.py

@@ -1419,20 +1419,21 @@ class LINK(DIV):
 
 class SCRIPT(DIV):
 
-    tag = 'script'
+    tag = b'script'
+    tagname = to_bytes(tag)
 
     def xml(self):
         (fa, co) = self._xml()
-        fa = to_native(fa)
+        fa = to_bytes(fa)
         # no escaping of subcomponents
-        co = '\n'.join([str(component) for component in
+        co = b'\n'.join([to_bytes(component) for component in
                        self.components])
         if co:
             # <script [attributes]><!--//--><![CDATA[//><!--
             # script body
             # //--><!]]></script>
             # return '<%s%s><!--//--><![CDATA[//><!--\n%s\n//--><!]]></%s>' % (self.tag, fa, co, self.tag)
-            return '<%s%s><!--\n%s\n//--></%s>' % (self.tag, fa, co, self.tag)
+            return b'<%s%s><!--\n%s\n//--></%s>' % (self.tagname, fa, co, self.tagname)
         else:
             return DIV.xml(self)
 
@@ -1440,18 +1441,19 @@ class SCRIPT(DIV):
 class STYLE(DIV):
 
     tag = 'style'
+    tagname = to_bytes(tag)
 
     def xml(self):
         (fa, co) = self._xml()
-        fa = to_native(fa)
+        fa = to_bytes(fa)
         # no escaping of subcomponents
-        co = '\n'.join([str(component) for component in
+        co = b'\n'.join([to_bytes(component) for component in
                        self.components])
         if co:
             # <style [attributes]><!--/*--><![CDATA[/*><!--*/
             # style body
             # /*]]>*/--></style>
-            return '<%s%s><!--/*--><![CDATA[/*><!--*/\n%s\n/*]]>*/--></%s>' % (self.tag, fa, co, self.tag)
+            return b'<%s%s><!--/*--><![CDATA[/*><!--*/\n%s\n/*]]>*/--></%s>' % (self.tagname, fa, co, self.tagname)
         else:
             return DIV.xml(self)
 

gluon/languages.py

@@ -431,10 +431,16 @@ class lazyT(object):
         return str(self) if self.M else local_html_escape(str(self), quote=False)
 
     def encode(self, *a, **b):
-        return str(self).encode(*a, **b)
+        if PY2 and a[0] != 'utf8':
+            return to_unicode(str(self)).encode(*a, **b)
+        else:
+            return str(self)
 
     def decode(self, *a, **b):
-        return str(self).decode(*a, **b)
+        if PY2:
+            return str(self).decode(*a, **b)
+        else:
+            return str(self)
 
     def read(self):
         return str(self)

gluon/main.py

@@ -183,12 +183,13 @@ def serve_controller(request, response, session):
         response._view_environment.update(page)
         page = run_view_in(response._view_environment)
 
-    # logic to garbage collect after exec, not always, once every 100 requests
-    global requests
-    requests = ('requests' in globals()) and (requests + 1) % 100 or 0
-    if not requests:
-        gc.collect()
-    # end garbage collection logic
+    if not request.env.web2py_disable_garbage_collect:
+        # logic to garbage collect after exec, not always, once every 100 requests
+        global requests
+        requests = ('requests' in globals()) and (requests + 1) % 100 or 0
+        if not requests:
+            gc.collect()
+        # end garbage collection logic
 
     # ##################################################
     # set default headers it not set

gluon/rewrite.py

@@ -22,12 +22,11 @@ import re
 import logging
 import traceback
 import threading
-import urllib
 from gluon.storage import Storage, List
 from gluon.http import HTTP
 from gluon.fileutils import abspath, read_file
 from gluon.settings import global_settings
-from gluon._compat import urllib_unquote, urllib_quote, iteritems, xrange
+from gluon._compat import urllib_unquote, urllib_quote, iteritems, xrange, urllib_quote_plus
 
 isdir = os.path.isdir
 isfile = os.path.isfile
@@ -235,7 +234,7 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
                     path_info, query_string = uri, ''
                 query_string += \
                     'code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
-                    (status, ticket, urllib.quote_plus(
+                    (status, ticket, urllib_quote_plus(
                         request.env.request_uri), request.url)
                 if uri.startswith('http://') or uri.startswith('https://'):
                     # make up a response
@@ -270,12 +269,12 @@ def try_redirect_on_error(http_object, request, ticket=None):
                 elif '?' in redir:
                     url = '%s&code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
                         (redir, status, ticket,
-                         urllib.quote_plus(request.env.request_uri),
+                         urllib_quote_plus(request.env.request_uri),
                          request.url)
                 else:
                     url = '%s?code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
                         (redir, status, ticket,
-                         urllib.quote_plus(request.env.request_uri),
+                         urllib_quote_plus(request.env.request_uri),
                          request.url)
                 return HTTP(303, 'You are being redirected <a href="%s">here</a>' % url, Location=url)
     return http_object

gluon/rocket.py

@@ -1750,7 +1750,7 @@ class WSGIWorker(Worker):
         if self.request_method != 'HEAD':
             try:
                 if self.chunked:
-                    self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data)))
+                    self.conn.sendall(b'%x\r\n%s\r\n' % (len(data), to_bytes(data, 'ISO-8859-1')))
                 else:
                     self.conn.sendall(to_bytes(data))
             except socket.timeout:

gluon/scheduler.py

@@ -525,7 +525,7 @@ class MetaScheduler(threading.Thread):
         self.have_heartbeat = True   # set to False to kill
         self.empty_runs = 0
 
-    def async(self, task):
+    def local_async(self, task):
         """Start the background process.
 
         Args:
@@ -913,7 +913,7 @@ class Scheduler(MetaScheduler):
                     self.w_stats.empty_runs = 0
                     self.w_stats.status = RUNNING
                     self.w_stats.total += 1
-                    self.wrapped_report_task(task, self.async(task))
+                    self.wrapped_report_task(task, self.local_async(task))
                     if not self.w_stats.status == DISABLED:
                         self.w_stats.status = ACTIVE
                 else:

gluon/sqlhtml.py

@@ -15,12 +15,11 @@ Holds:
 """
 
 import datetime
-import urllib
 import re
 import copy
 
 import os
-from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode
+from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode, urlencode
 from gluon.http import HTTP, redirect
 from gluon.html import XmlComponent, truncate_string
 from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG
@@ -1184,7 +1183,7 @@ def formstyle_bootstrap4_inline_factory(col_label_size=3):
             if isinstance(label, LABEL):
                 label['_class'] = add_class(label.get('_class'), 'form-control-label %s' % label_col_class)
 
-            parent.append(DIV(label, _controls, _class='form-group', _id=id))
+            parent.append(DIV(label, _controls, _class='form-group row', _id=id))
         return parent
     return _inner
 
@@ -1345,9 +1344,16 @@ class SQLFORM(FORM):
         # if no fields are provided, build it from the provided table
         # will only use writable or readable fields, unless forced to ignore
         if fields is None:
-            fields = [f.name for f in table if
-                      (ignore_rw or f.writable or f.readable) and 
-                      not (f.compute and not record)]
+            if not readonly:
+                if not record:
+                    # create form should only show writable fields
+                    fields = [f.name for f in table if (ignore_rw or f.writable) and not f.compute]
+                else:
+                    # update form should also show readable fields and computed fields (but in reaodnly mode)
+                    fields = [f.name for f in table if (ignore_rw or f.writable or f.readable)]
+            else:
+                # read only form should show all readable fields
+                fields = [f.name for f in table if (ignore_rw or f.readable)]
         self.fields = fields
 
         # make sure we have an id
@@ -1676,6 +1682,7 @@ class SQLFORM(FORM):
             keepvalues = True if self.record else False
 
         if self.readonly:
+            self.deleted = False
             return False
 
         if request_vars.__class__.__name__ == 'Request':
@@ -1832,7 +1839,7 @@ class SQLFORM(FORM):
                 continue
 
             field = self.table[fieldname]
-            if field.type == 'id':
+            if field.type == 'id' or field.compute:
                 continue
             if field.type == 'boolean':
                 if self.vars.get(fieldname, False):
@@ -2300,7 +2307,7 @@ class SQLFORM(FORM):
                       cornerall='',
                       cornertop='',
                       cornerbottom='',
-                      button='button btn btn-default',
+                      button='button btn btn-default btn-secondary',
                       buttontext='buttontext button',
                       buttonadd='icon plus icon-plus glyphicon glyphicon-plus',
                       buttonback='icon leftarrow icon-arrow-left glyphicon glyphicon-arrow-left',
@@ -2741,8 +2748,8 @@ class SQLFORM(FORM):
                           _id=skeywords_id, _class='form-control',
                           _onfocus="jQuery('#%s').change();jQuery('#%s').slideDown();" % (spanel_id, sfields_id) if advanced_search else ''
                           ),
-                    INPUT(_type='submit', _value=T('Search'), _class="btn btn-default"),
-                    INPUT(_type='submit', _value=T('Clear'), _class="btn btn-default",
+                    INPUT(_type='submit', _value=T('Search'), _class="btn btn-default btn-secondary"),
+                    INPUT(_type='submit', _value=T('Clear'), _class="btn btn-default btn-secondary",
                           _onclick="jQuery('#%s').val('');" % skeywords_id),
                     *hidden_fields,
                     _method="GET", _action=url), search_menu)
@@ -3128,7 +3135,7 @@ class SQLFORM(FORM):
                     order=request.vars.order or '',
                     _export_type=k,
                     keywords=keywords or ''))
-                export_links.append(A(T(label), _href=link, _title=title, _class='btn btn-default'))
+                export_links.append(A(T(label), _href=link, _title=title, _class='btn btn-default btn-secondary'))
             export_menu = \
                 DIV(T('Export:'), _class="w2p_export_menu", *export_links)
         else:
@@ -3542,7 +3549,7 @@ class SQLTABLE(TABLE):
                             if ref.find('.') >= 0:
                                 tref, fref = ref.split('.')
                                 if hasattr(sqlrows.db[tref], '_primarykey'):
-                                    href = '%s/%s?%s' % (linkto, tref, urllib.urlencode({fref: r}))
+                                    href = '%s/%s?%s' % (linkto, tref, urlencode({fref: r}))
                         r = A(represent(field, r, record), _href=str(href))
                     elif field.represent:
                         if field not in repr_cache:
@@ -3553,7 +3560,7 @@ class SQLTABLE(TABLE):
                 elif linkto and hasattr(field._table, '_primarykey')\
                         and fieldname in field._table._primarykey:
                     # have to test this with multi-key tables
-                    key = urllib.urlencode(dict([
+                    key = urlencode(dict([
                                 ((tablename in record
                                       and isinstance(record, Row)
                                       and isinstance(record[tablename], Row)) and

gluon/tests/test_globals.py

@@ -230,3 +230,14 @@ class testResponse(unittest.TestCase):
         current.session._fixup_before_save()
         cookie = str(current.response.cookies)
         self.assertTrue('httponly' not in cookie.lower())
+
+    def test_include_meta(self):
+        response = Response()
+        response.meta[u'web2py'] = 'web2py'
+        response.include_meta()
+        self.assertEqual(response.body.getvalue(), '\n<meta name="web2py" content="web2py" />\n')
+        response = Response()
+        response.meta[u'meta_dict'] = {u'tag_name':'tag_value'}
+        response.include_meta()
+        self.assertEqual(response.body.getvalue(), '\n<meta tag_name="tag_value" />\n')
+

gluon/tests/test_html.py

@@ -340,18 +340,20 @@ class TestBareHelpers(unittest.TestCase):
 
     def test_SCRIPT(self):
         self.assertEqual(SCRIPT('<>', _a='1', _b='2').xml(),
-                         '''<script a="1" b="2"><!--
+                         b'''<script a="1" b="2"><!--
 <>
 //--></script>''')
         self.assertEqual(SCRIPT('<>').xml(),
-                         '''<script><!--
+                         b'''<script><!--
 <>
 //--></script>''')
         self.assertEqual(SCRIPT().xml(), b'<script></script>')
+        self.assertEqual(SCRIPT(';').xml() + DIV().xml(), 
+                         b'<script><!--\n;\n//--></script><div></div>')
 
     def test_STYLE(self):
         self.assertEqual(STYLE('<>', _a='1', _b='2').xml(),
-                         '<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
+                         b'<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
         # Try to hit : return DIV.xml(self)
         self.assertEqual(STYLE().xml(), b'<style></style>')
 

gluon/tests/test_languages.py

@@ -12,7 +12,9 @@ import tempfile
 import unittest
 
 from gluon import languages
-from gluon._compat import PY2
+from gluon._compat import PY2, to_unicode, to_bytes
+from gluon.storage import Messages
+from gluon.html import SPAN
 
 MP_WORKING = 0
 try:
@@ -108,6 +110,8 @@ class TestTranslations(unittest.TestCase):
         T.force('it')
         self.assertEqual(str(T('Hello World')),
                          'Salve Mondo')
+        self.assertEqual(to_unicode(T('Hello World')),
+                         'Salve Mondo')
 
 class TestDummyApp(unittest.TestCase):
 
@@ -179,3 +183,43 @@ def index():
         for key in ['hello', 'world', '%s %%{shop}', 'ahoy']:
             self.assertTrue(key in en_dict)
             self.assertTrue(key in pt_dict)
+
+class TestMessages(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        messages = Messages(T)
+        messages.update({'email_sent':'Email sent', 'test': "ä"})
+        self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
+
+class TestHTMLTag(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        elem = SPAN(T("Complete"))
+        self.assertEqual(elem.flatten(), "Complete")
+        elem = SPAN(T("Cannot be empty", language="ru"))
+        self.assertEqual(elem.xml(), to_bytes('<span>Пустое значение недопустимо</span>'))
+        self.assertEqual(elem.flatten(), 'Пустое значение недопустимо')

gluon/tools.py

@@ -12,7 +12,7 @@ Auth, Mail, PluginManager and various utilities
 
 import base64
 from functools import reduce
-from gluon._compat import pickle, thread, urllib2, Cookie, StringIO
+from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, urlencode
 from gluon._compat import configparser, MIMEBase, MIMEMultipart, MIMEText, Header
 from gluon._compat import Encoders, Charset, long, urllib_quote, iteritems
 from gluon._compat import to_bytes, to_native, add_charset
@@ -27,7 +27,6 @@ import time
 import fnmatch
 import traceback
 import smtplib
-import urllib
 import email.utils
 import random
 import hmac
@@ -873,7 +872,7 @@ class Recaptcha(DIV):
                 and len(recaptcha_challenge_field)):
             self.errors['captcha'] = self.error_message
             return False
-        params = urllib.urlencode({
+        params = urlencode({
             'privatekey': private_key,
             'remoteip': remoteip,
             'challenge': recaptcha_challenge_field,
@@ -1026,7 +1025,7 @@ class Recaptcha2(DIV):
         if not recaptcha_response_field:
             self.errors['captcha'] = self.error_message
             return False
-        params = urllib.urlencode({
+        params = urlencode({
             'secret': self.private_key,
             'remoteip': remoteip,
             'response': recaptcha_response_field,
@@ -2498,9 +2497,9 @@ class Auth(AuthAPI):
                 success = True
 
         def build_response(body):
-            return '<?xml version="1.0" encoding="UTF-8"?>\n' +\
-                TAG['cas:serviceResponse'](
-                    body, **{'_xmlns:cas': 'http://www.yale.edu/tp/cas'}).xml()
+            xml_body = to_native(TAG['cas:serviceResponse'](
+                    body, **{'_xmlns:cas': 'http://www.yale.edu/tp/cas'}).xml())
+            return '<?xml version="1.0" encoding="UTF-8"?>\n' + xml_body
         if success:
             if version == 1:
                 message = 'yes\n%s' % user[userfield]
@@ -3051,7 +3050,7 @@ class Auth(AuthAPI):
 
         if self.settings.register_verify_password:
             if self.settings.register_fields is None:
-                self.settings.register_fields = [f.name for f in table_user if f.writable]
+                self.settings.register_fields = [f.name for f in table_user if f.writable and not f.compute]
                 k = self.settings.register_fields.index(passfield)
                 self.settings.register_fields.insert(k + 1, "password_two")
             extra_fields = [
@@ -4738,7 +4737,7 @@ def fetch(url, data=None, headers=None,
           user_agent='Mozilla/5.0'):
     headers = headers or {}
     if data is not None:
-        data = urllib.urlencode(data)
+        data = urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
     headers['Cookie'] = ' '.join(

gluon/utf8.py

@@ -11,7 +11,7 @@ Utilities and class for UTF8 strings managing
 ----------------------------------------------
 """
 from __future__ import print_function
-from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native
+from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload
 
 __all__ = ['Utf8']
 

gluon/validators.py

@@ -1656,173 +1656,292 @@ class IS_GENERIC_URL(Validator):
         # else the URL is not valid
         return (value, translate(self.error_message))
 
-# Sources (obtained 2015-Feb-24):
+# Sources (obtained 2017-Nov-11):
 #    http://data.iana.org/TLD/tlds-alpha-by-domain.txt
 # see scripts/parse_top_level_domains.py for an easy update
 
 official_top_level_domains = [
     # a
-    'abogado', 'ac', 'academy', 'accountants', 'active', 'actor',
-    'ad', 'adult', 'ae', 'aero', 'af', 'ag', 'agency', 'ai',
-    'airforce', 'al', 'allfinanz', 'alsace', 'am', 'amsterdam', 'an',
-    'android', 'ao', 'apartments', 'aq', 'aquarelle', 'ar', 'archi',
-    'army', 'arpa', 'as', 'asia', 'associates', 'at', 'attorney',
-    'au', 'auction', 'audio', 'autos', 'aw', 'ax', 'axa', 'az',
+    'aaa', 'aarp', 'abarth', 'abb', 'abbott', 'abbvie', 'abc',
+    'able', 'abogado', 'abudhabi', 'ac', 'academy', 'accenture',
+    'accountant', 'accountants', 'aco', 'active', 'actor', 'ad',
+    'adac', 'ads', 'adult', 'ae', 'aeg', 'aero', 'aetna', 'af',
+    'afamilycompany', 'afl', 'africa', 'ag', 'agakhan', 'agency',
+    'ai', 'aig', 'aigo', 'airbus', 'airforce', 'airtel', 'akdn',
+    'al', 'alfaromeo', 'alibaba', 'alipay', 'allfinanz', 'allstate',
+    'ally', 'alsace', 'alstom', 'am', 'americanexpress',
+    'americanfamily', 'amex', 'amfam', 'amica', 'amsterdam',
+    'analytics', 'android', 'anquan', 'anz', 'ao', 'aol',
+    'apartments', 'app', 'apple', 'aq', 'aquarelle', 'ar', 'arab',
+    'aramco', 'archi', 'army', 'arpa', 'art', 'arte', 'as', 'asda',
+    'asia', 'associates', 'at', 'athleta', 'attorney', 'au',
+    'auction', 'audi', 'audible', 'audio', 'auspost', 'author',
+    'auto', 'autos', 'avianca', 'aw', 'aws', 'ax', 'axa', 'az',
+    'azure',
     # b
-    'ba', 'band', 'bank', 'bar', 'barclaycard', 'barclays',
-    'bargains', 'bayern', 'bb', 'bd', 'be', 'beer', 'berlin', 'best',
-    'bf', 'bg', 'bh', 'bi', 'bid', 'bike', 'bingo', 'bio', 'biz',
-    'bj', 'black', 'blackfriday', 'bloomberg', 'blue', 'bm', 'bmw',
-    'bn', 'bnpparibas', 'bo', 'boo', 'boutique', 'br', 'brussels',
-    'bs', 'bt', 'budapest', 'build', 'builders', 'business', 'buzz',
-    'bv', 'bw', 'by', 'bz', 'bzh',
+    'ba', 'baby', 'baidu', 'banamex', 'bananarepublic', 'band',
+    'bank', 'bar', 'barcelona', 'barclaycard', 'barclays',
+    'barefoot', 'bargains', 'baseball', 'basketball', 'bauhaus',
+    'bayern', 'bb', 'bbc', 'bbt', 'bbva', 'bcg', 'bcn', 'bd', 'be',
+    'beats', 'beauty', 'beer', 'bentley', 'berlin', 'best',
+    'bestbuy', 'bet', 'bf', 'bg', 'bh', 'bharti', 'bi', 'bible',
+    'bid', 'bike', 'bing', 'bingo', 'bio', 'biz', 'bj', 'black',
+    'blackfriday', 'blanco', 'blockbuster', 'blog', 'bloomberg',
+    'blue', 'bm', 'bms', 'bmw', 'bn', 'bnl', 'bnpparibas', 'bo',
+    'boats', 'boehringer', 'bofa', 'bom', 'bond', 'boo', 'book',
+    'booking', 'boots', 'bosch', 'bostik', 'boston', 'bot',
+    'boutique', 'box', 'br', 'bradesco', 'bridgestone', 'broadway',
+    'broker', 'brother', 'brussels', 'bs', 'bt', 'budapest',
+    'bugatti', 'build', 'builders', 'business', 'buy', 'buzz', 'bv',
+    'bw', 'by', 'bz', 'bzh',
     # c
-    'ca', 'cab', 'cal', 'camera', 'camp', 'cancerresearch', 'canon',
-    'capetown', 'capital', 'caravan', 'cards', 'care', 'career',
-    'careers', 'cartier', 'casa', 'cash', 'casino', 'cat',
-    'catering', 'cbn', 'cc', 'cd', 'center', 'ceo', 'cern', 'cf',
-    'cg', 'ch', 'channel', 'chat', 'cheap', 'christmas', 'chrome',
-    'church', 'ci', 'citic', 'city', 'ck', 'cl', 'claims',
-    'cleaning', 'click', 'clinic', 'clothing', 'club', 'cm', 'cn',
-    'co', 'coach', 'codes', 'coffee', 'college', 'cologne', 'com',
-    'community', 'company', 'computer', 'condos', 'construction',
-    'consulting', 'contractors', 'cooking', 'cool', 'coop',
-    'country', 'cr', 'credit', 'creditcard', 'cricket', 'crs',
-    'cruises', 'cu', 'cuisinella', 'cv', 'cw', 'cx', 'cy', 'cymru',
-    'cz',
+    'ca', 'cab', 'cafe', 'cal', 'call', 'calvinklein', 'cam',
+    'camera', 'camp', 'cancerresearch', 'canon', 'capetown',
+    'capital', 'capitalone', 'car', 'caravan', 'cards', 'care',
+    'career', 'careers', 'cars', 'cartier', 'casa', 'case', 'caseih',
+    'cash', 'casino', 'cat', 'catering', 'catholic', 'cba', 'cbn',
+    'cbre', 'cbs', 'cc', 'cd', 'ceb', 'center', 'ceo', 'cern', 'cf',
+    'cfa', 'cfd', 'cg', 'ch', 'chanel', 'channel', 'chase', 'chat',
+    'cheap', 'chintai', 'christmas', 'chrome', 'chrysler', 'church',
+    'ci', 'cipriani', 'circle', 'cisco', 'citadel', 'citi', 'citic',
+    'city', 'cityeats', 'ck', 'cl', 'claims', 'cleaning', 'click',
+    'clinic', 'clinique', 'clothing', 'cloud', 'club', 'clubmed',
+    'cm', 'cn', 'co', 'coach', 'codes', 'coffee', 'college',
+    'cologne', 'com', 'comcast', 'commbank', 'community', 'company',
+    'compare', 'computer', 'comsec', 'condos', 'construction',
+    'consulting', 'contact', 'contractors', 'cooking',
+    'cookingchannel', 'cool', 'coop', 'corsica', 'country', 'coupon',
+    'coupons', 'courses', 'cr', 'credit', 'creditcard',
+    'creditunion', 'cricket', 'crown', 'crs', 'cruise', 'cruises',
+    'csc', 'cu', 'cuisinella', 'cv', 'cw', 'cx', 'cy', 'cymru',
+    'cyou', 'cz',
     # d
-    'dabur', 'dad', 'dance', 'dating', 'day', 'dclk', 'de', 'deals',
-    'degree', 'delivery', 'democrat', 'dental', 'dentist', 'desi',
-    'design', 'dev', 'diamonds', 'diet', 'digital', 'direct',
-    'directory', 'discount', 'dj', 'dk', 'dm', 'dnp', 'do', 'docs',
-    'domains', 'doosan', 'durban', 'dvag', 'dz',
+    'dabur', 'dad', 'dance', 'data', 'date', 'dating', 'datsun',
+    'day', 'dclk', 'dds', 'de', 'deal', 'dealer', 'deals', 'degree',
+    'delivery', 'dell', 'deloitte', 'delta', 'democrat', 'dental',
+    'dentist', 'desi', 'design', 'dev', 'dhl', 'diamonds', 'diet',
+    'digital', 'direct', 'directory', 'discount', 'discover', 'dish',
+    'diy', 'dj', 'dk', 'dm', 'dnp', 'do', 'docs', 'doctor', 'dodge',
+    'dog', 'doha', 'domains', 'dot', 'download', 'drive', 'dtv',
+    'dubai', 'duck', 'dunlop', 'duns', 'dupont', 'durban', 'dvag',
+    'dvr', 'dz',
     # e
-    'eat', 'ec', 'edu', 'education', 'ee', 'eg', 'email', 'emerck',
-    'energy', 'engineer', 'engineering', 'enterprises', 'equipment',
-    'er', 'es', 'esq', 'estate', 'et', 'eu', 'eurovision', 'eus',
-    'events', 'everbank', 'exchange', 'expert', 'exposed',
+    'earth', 'eat', 'ec', 'eco', 'edeka', 'edu', 'education', 'ee',
+    'eg', 'email', 'emerck', 'energy', 'engineer', 'engineering',
+    'enterprises', 'epost', 'epson', 'equipment', 'er', 'ericsson',
+    'erni', 'es', 'esq', 'estate', 'esurance', 'et', 'etisalat',
+    'eu', 'eurovision', 'eus', 'events', 'everbank', 'exchange',
+    'expert', 'exposed', 'express', 'extraspace',
     # f
-    'fail', 'fans', 'farm', 'fashion', 'feedback', 'fi', 'finance',
-    'financial', 'firmdale', 'fish', 'fishing', 'fit', 'fitness',
-    'fj', 'fk', 'flights', 'florist', 'flowers', 'flsmidth', 'fly',
-    'fm', 'fo', 'foo', 'football', 'forsale', 'foundation', 'fr',
-    'frl', 'frogans', 'fund', 'furniture', 'futbol',
+    'fage', 'fail', 'fairwinds', 'faith', 'family', 'fan', 'fans',
+    'farm', 'farmers', 'fashion', 'fast', 'fedex', 'feedback',
+    'ferrari', 'ferrero', 'fi', 'fiat', 'fidelity', 'fido', 'film',
+    'final', 'finance', 'financial', 'fire', 'firestone', 'firmdale',
+    'fish', 'fishing', 'fit', 'fitness', 'fj', 'fk', 'flickr',
+    'flights', 'flir', 'florist', 'flowers', 'fly', 'fm', 'fo',
+    'foo', 'food', 'foodnetwork', 'football', 'ford', 'forex',
+    'forsale', 'forum', 'foundation', 'fox', 'fr', 'free',
+    'fresenius', 'frl', 'frogans', 'frontdoor', 'frontier', 'ftr',
+    'fujitsu', 'fujixerox', 'fun', 'fund', 'furniture', 'futbol',
+    'fyi',
     # g
-    'ga', 'gal', 'gallery', 'garden', 'gb', 'gbiz', 'gd', 'gdn',
-    'ge', 'gent', 'gf', 'gg', 'ggee', 'gh', 'gi', 'gift', 'gifts',
-    'gives', 'gl', 'glass', 'gle', 'global', 'globo', 'gm', 'gmail',
-    'gmo', 'gmx', 'gn', 'goldpoint', 'goog', 'google', 'gop', 'gov',
-    'gp', 'gq', 'gr', 'graphics', 'gratis', 'green', 'gripe', 'gs',
-    'gt', 'gu', 'guide', 'guitars', 'guru', 'gw', 'gy',
+    'ga', 'gal', 'gallery', 'gallo', 'gallup', 'game', 'games',
+    'gap', 'garden', 'gb', 'gbiz', 'gd', 'gdn', 'ge', 'gea', 'gent',
+    'genting', 'george', 'gf', 'gg', 'ggee', 'gh', 'gi', 'gift',
+    'gifts', 'gives', 'giving', 'gl', 'glade', 'glass', 'gle',
+    'global', 'globo', 'gm', 'gmail', 'gmbh', 'gmo', 'gmx', 'gn',
+    'godaddy', 'gold', 'goldpoint', 'golf', 'goo', 'goodhands',
+    'goodyear', 'goog', 'google', 'gop', 'got', 'gov', 'gp', 'gq',
+    'gr', 'grainger', 'graphics', 'gratis', 'green', 'gripe',
+    'grocery', 'group', 'gs', 'gt', 'gu', 'guardian', 'gucci',
+    'guge', 'guide', 'guitars', 'guru', 'gw', 'gy',
     # h
-    'hamburg', 'hangout', 'haus', 'healthcare', 'help', 'here',
-    'hermes', 'hiphop', 'hiv', 'hk', 'hm', 'hn', 'holdings',
-    'holiday', 'homes', 'horse', 'host', 'hosting', 'house', 'how',
-    'hr', 'ht', 'hu',
+    'hair', 'hamburg', 'hangout', 'haus', 'hbo', 'hdfc', 'hdfcbank',
+    'health', 'healthcare', 'help', 'helsinki', 'here', 'hermes',
+    'hgtv', 'hiphop', 'hisamitsu', 'hitachi', 'hiv', 'hk', 'hkt',
+    'hm', 'hn', 'hockey', 'holdings', 'holiday', 'homedepot',
+    'homegoods', 'homes', 'homesense', 'honda', 'honeywell', 'horse',
+    'hospital', 'host', 'hosting', 'hot', 'hoteles', 'hotels',
+    'hotmail', 'house', 'how', 'hr', 'hsbc', 'ht', 'hu', 'hughes',
+    'hyatt', 'hyundai',
     # i
-    'ibm', 'id', 'ie', 'ifm', 'il', 'im', 'immo', 'immobilien', 'in',
-    'industries', 'info', 'ing', 'ink', 'institute', 'insure', 'int',
-    'international', 'investments', 'io', 'iq', 'ir', 'irish', 'is',
-    'it', 'iwc',
+    'ibm', 'icbc', 'ice', 'icu', 'id', 'ie', 'ieee', 'ifm', 'ikano',
+    'il', 'im', 'imamat', 'imdb', 'immo', 'immobilien', 'in',
+    'industries', 'infiniti', 'info', 'ing', 'ink', 'institute',
+    'insurance', 'insure', 'int', 'intel', 'international', 'intuit',
+    'investments', 'io', 'ipiranga', 'iq', 'ir', 'irish', 'is',
+    'iselect', 'ismaili', 'ist', 'istanbul', 'it', 'itau', 'itv',
+    'iveco', 'iwc',
     # j
-    'jcb', 'je', 'jetzt', 'jm', 'jo', 'jobs', 'joburg', 'jp',
-    'juegos',
+    'jaguar', 'java', 'jcb', 'jcp', 'je', 'jeep', 'jetzt', 'jewelry',
+    'jio', 'jlc', 'jll', 'jm', 'jmp', 'jnj', 'jo', 'jobs', 'joburg',
+    'jot', 'joy', 'jp', 'jpmorgan', 'jprs', 'juegos', 'juniper',
     # k
-    'kaufen', 'kddi', 'ke', 'kg', 'kh', 'ki', 'kim', 'kitchen',
-    'kiwi', 'km', 'kn', 'koeln', 'kp', 'kr', 'krd', 'kred', 'kw',
-    'ky', 'kyoto', 'kz',
+    'kaufen', 'kddi', 'ke', 'kerryhotels', 'kerrylogistics',
+    'kerryproperties', 'kfh', 'kg', 'kh', 'ki', 'kia', 'kim',
+    'kinder', 'kindle', 'kitchen', 'kiwi', 'km', 'kn', 'koeln',
+    'komatsu', 'kosher', 'kp', 'kpmg', 'kpn', 'kr', 'krd', 'kred',
+    'kuokgroup', 'kw', 'ky', 'kyoto', 'kz',
     # l
-    'la', 'lacaixa', 'land', 'lat', 'latrobe', 'lawyer', 'lb', 'lc',
-    'lds', 'lease', 'legal', 'lgbt', 'li', 'lidl', 'life',
-    'lighting', 'limited', 'limo', 'link', 'lk', 'loans',
-    'localhost', 'london', 'lotte', 'lotto', 'lr', 'ls', 'lt',
-    'ltda', 'lu', 'luxe', 'luxury', 'lv', 'ly',
+    'la', 'lacaixa', 'ladbrokes', 'lamborghini', 'lamer',
+    'lancaster', 'lancia', 'lancome', 'land', 'landrover', 'lanxess',
+    'lasalle', 'lat', 'latino', 'latrobe', 'law', 'lawyer', 'lb',
+    'lc', 'lds', 'lease', 'leclerc', 'lefrak', 'legal', 'lego',
+    'lexus', 'lgbt', 'li', 'liaison', 'lidl', 'life',
+    'lifeinsurance', 'lifestyle', 'lighting', 'like', 'lilly',
+    'limited', 'limo', 'lincoln', 'linde', 'link', 'lipsy', 'live',
+    'living', 'lixil', 'lk', 'loan', 'loans', 'localhost', 'locker',
+    'locus', 'loft', 'lol', 'london', 'lotte', 'lotto', 'love',
+    'lpl', 'lplfinancial', 'lr', 'ls', 'lt', 'ltd', 'ltda', 'lu',
+    'lundbeck', 'lupin', 'luxe', 'luxury', 'lv', 'ly',
     # m
-    'ma', 'madrid', 'maison', 'management', 'mango', 'market',
-    'marketing', 'marriott', 'mc', 'md', 'me', 'media', 'meet',
-    'melbourne', 'meme', 'memorial', 'menu', 'mg', 'mh', 'miami',
-    'mil', 'mini', 'mk', 'ml', 'mm', 'mn', 'mo', 'mobi', 'moda',
-    'moe', 'monash', 'money', 'mormon', 'mortgage', 'moscow',
-    'motorcycles', 'mov', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu',
-    'museum', 'mv', 'mw', 'mx', 'my', 'mz',
+    'ma', 'macys', 'madrid', 'maif', 'maison', 'makeup', 'man',
+    'management', 'mango', 'map', 'market', 'marketing', 'markets',
+    'marriott', 'marshalls', 'maserati', 'mattel', 'mba', 'mc',
+    'mckinsey', 'md', 'me', 'med', 'media', 'meet', 'melbourne',
+    'meme', 'memorial', 'men', 'menu', 'meo', 'merckmsd', 'metlife',
+    'mg', 'mh', 'miami', 'microsoft', 'mil', 'mini', 'mint', 'mit',
+    'mitsubishi', 'mk', 'ml', 'mlb', 'mls', 'mm', 'mma', 'mn', 'mo',
+    'mobi', 'mobile', 'mobily', 'moda', 'moe', 'moi', 'mom',
+    'monash', 'money', 'monster', 'mopar', 'mormon', 'mortgage',
+    'moscow', 'moto', 'motorcycles', 'mov', 'movie', 'movistar',
+    'mp', 'mq', 'mr', 'ms', 'msd', 'mt', 'mtn', 'mtr', 'mu',
+    'museum', 'mutual', 'mv', 'mw', 'mx', 'my', 'mz',
     # n
-    'na', 'nagoya', 'name', 'navy', 'nc', 'ne', 'net', 'network',
-    'neustar', 'new', 'nexus', 'nf', 'ng', 'ngo', 'nhk', 'ni',
-    'nico', 'ninja', 'nl', 'no', 'np', 'nr', 'nra', 'nrw', 'ntt',
-    'nu', 'nyc', 'nz',
+    'na', 'nab', 'nadex', 'nagoya', 'name', 'nationwide', 'natura',
+    'navy', 'nba', 'nc', 'ne', 'nec', 'net', 'netbank', 'netflix',
+    'network', 'neustar', 'new', 'newholland', 'news', 'next',
+    'nextdirect', 'nexus', 'nf', 'nfl', 'ng', 'ngo', 'nhk', 'ni',
+    'nico', 'nike', 'nikon', 'ninja', 'nissan', 'nissay', 'nl', 'no',
+    'nokia', 'northwesternmutual', 'norton', 'now', 'nowruz',
+    'nowtv', 'np', 'nr', 'nra', 'nrw', 'ntt', 'nu', 'nyc', 'nz',
     # o
-    'okinawa', 'om', 'one', 'ong', 'onl', 'ooo', 'org', 'organic',
-    'osaka', 'otsuka', 'ovh',
+    'obi', 'observer', 'off', 'office', 'okinawa', 'olayan',
+    'olayangroup', 'oldnavy', 'ollo', 'om', 'omega', 'one', 'ong',
+    'onl', 'online', 'onyourside', 'ooo', 'open', 'oracle', 'orange',
+    'org', 'organic', 'origins', 'osaka', 'otsuka', 'ott', 'ovh',
     # p
-    'pa', 'paris', 'partners', 'parts', 'party', 'pe', 'pf', 'pg',
-    'ph', 'pharmacy', 'photo', 'photography', 'photos', 'physio',
-    'pics', 'pictures', 'pink', 'pizza', 'pk', 'pl', 'place',
-    'plumbing', 'pm', 'pn', 'pohl', 'poker', 'porn', 'post', 'pr',
-    'praxi', 'press', 'pro', 'prod', 'productions', 'prof',
-    'properties', 'property', 'ps', 'pt', 'pub', 'pw', 'py',
+    'pa', 'page', 'panasonic', 'panerai', 'paris', 'pars',
+    'partners', 'parts', 'party', 'passagens', 'pay', 'pccw', 'pe',
+    'pet', 'pf', 'pfizer', 'pg', 'ph', 'pharmacy', 'phd', 'philips',
+    'phone', 'photo', 'photography', 'photos', 'physio', 'piaget',
+    'pics', 'pictet', 'pictures', 'pid', 'pin', 'ping', 'pink',
+    'pioneer', 'pizza', 'pk', 'pl', 'place', 'play', 'playstation',
+    'plumbing', 'plus', 'pm', 'pn', 'pnc', 'pohl', 'poker',
+    'politie', 'porn', 'post', 'pr', 'pramerica', 'praxi', 'press',
+    'prime', 'pro', 'prod', 'productions', 'prof', 'progressive',
+    'promo', 'properties', 'property', 'protection', 'pru',
+    'prudential', 'ps', 'pt', 'pub', 'pw', 'pwc', 'py',
     # q
-    'qa', 'qpon', 'quebec',
+    'qa', 'qpon', 'quebec', 'quest', 'qvc',
     # r
-    're', 'realtor', 'recipes', 'red', 'rehab', 'reise', 'reisen',
-    'reit', 'ren', 'rentals', 'repair', 'report', 'republican',
-    'rest', 'restaurant', 'reviews', 'rich', 'rio', 'rip', 'ro',
-    'rocks', 'rodeo', 'rs', 'rsvp', 'ru', 'ruhr', 'rw', 'ryukyu',
+    'racing', 'radio', 'raid', 're', 'read', 'realestate', 'realtor',
+    'realty', 'recipes', 'red', 'redstone', 'redumbrella', 'rehab',
+    'reise', 'reisen', 'reit', 'reliance', 'ren', 'rent', 'rentals',
+    'repair', 'report', 'republican', 'rest', 'restaurant', 'review',
+    'reviews', 'rexroth', 'rich', 'richardli', 'ricoh',
+    'rightathome', 'ril', 'rio', 'rip', 'rmit', 'ro', 'rocher',
+    'rocks', 'rodeo', 'rogers', 'room', 'rs', 'rsvp', 'ru', 'rugby',
+    'ruhr', 'run', 'rw', 'rwe', 'ryukyu',
     # s
-    'sa', 'saarland', 'sale', 'samsung', 'sarl', 'saxo', 'sb', 'sc',
-    'sca', 'scb', 'schmidt', 'school', 'schule', 'schwarz',
-    'science', 'scot', 'sd', 'se', 'services', 'sew', 'sexy', 'sg',
-    'sh', 'shiksha', 'shoes', 'shriram', 'si', 'singles', 'sj', 'sk',
-    'sky', 'sl', 'sm', 'sn', 'so', 'social', 'software', 'sohu',
-    'solar', 'solutions', 'soy', 'space', 'spiegel', 'sr', 'st',
-    'style', 'su', 'supplies', 'supply', 'support', 'surf',
-    'surgery', 'suzuki', 'sv', 'sx', 'sy', 'sydney', 'systems', 'sz',
+    'sa', 'saarland', 'safe', 'safety', 'sakura', 'sale', 'salon',
+    'samsclub', 'samsung', 'sandvik', 'sandvikcoromant', 'sanofi',
+    'sap', 'sapo', 'sarl', 'sas', 'save', 'saxo', 'sb', 'sbi', 'sbs',
+    'sc', 'sca', 'scb', 'schaeffler', 'schmidt', 'scholarships',
+    'school', 'schule', 'schwarz', 'science', 'scjohnson', 'scor',
+    'scot', 'sd', 'se', 'search', 'seat', 'secure', 'security',
+    'seek', 'select', 'sener', 'services', 'ses', 'seven', 'sew',
+    'sex', 'sexy', 'sfr', 'sg', 'sh', 'shangrila', 'sharp', 'shaw',
+    'shell', 'shia', 'shiksha', 'shoes', 'shop', 'shopping',
+    'shouji', 'show', 'showtime', 'shriram', 'si', 'silk', 'sina',
+    'singles', 'site', 'sj', 'sk', 'ski', 'skin', 'sky', 'skype',
+    'sl', 'sling', 'sm', 'smart', 'smile', 'sn', 'sncf', 'so',
+    'soccer', 'social', 'softbank', 'software', 'sohu', 'solar',
+    'solutions', 'song', 'sony', 'soy', 'space', 'spiegel', 'spot',
+    'spreadbetting', 'sr', 'srl', 'srt', 'st', 'stada', 'staples',
+    'star', 'starhub', 'statebank', 'statefarm', 'statoil', 'stc',
+    'stcgroup', 'stockholm', 'storage', 'store', 'stream', 'studio',
+    'study', 'style', 'su', 'sucks', 'supplies', 'supply', 'support',
+    'surf', 'surgery', 'suzuki', 'sv', 'swatch', 'swiftcover',
+    'swiss', 'sx', 'sy', 'sydney', 'symantec', 'systems', 'sz',
     # t
-    'taipei', 'tatar', 'tattoo', 'tax', 'tc', 'td', 'technology',
-    'tel', 'temasek', 'tennis', 'tf', 'tg', 'th', 'tienda', 'tips',
-    'tires', 'tirol', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'today',
-    'tokyo', 'tools', 'top', 'toshiba', 'town', 'toys', 'tp', 'tr',
-    'trade', 'training', 'travel', 'trust', 'tt', 'tui', 'tv', 'tw',
-    'tz',
+    'tab', 'taipei', 'talk', 'taobao', 'target', 'tatamotors',
+    'tatar', 'tattoo', 'tax', 'taxi', 'tc', 'tci', 'td', 'tdk',
+    'team', 'tech', 'technology', 'tel', 'telecity', 'telefonica',
+    'temasek', 'tennis', 'teva', 'tf', 'tg', 'th', 'thd', 'theater',
+    'theatre', 'tiaa', 'tickets', 'tienda', 'tiffany', 'tips',
+    'tires', 'tirol', 'tj', 'tjmaxx', 'tjx', 'tk', 'tkmaxx', 'tl',
+    'tm', 'tmall', 'tn', 'to', 'today', 'tokyo', 'tools', 'top',
+    'toray', 'toshiba', 'total', 'tours', 'town', 'toyota', 'toys',
+    'tr', 'trade', 'trading', 'training', 'travel', 'travelchannel',
+    'travelers', 'travelersinsurance', 'trust', 'trv', 'tt', 'tube',
+    'tui', 'tunes', 'tushu', 'tv', 'tvs', 'tw', 'tz',
     # u
-    'ua', 'ug', 'uk', 'university', 'uno', 'uol', 'us', 'uy', 'uz',
+    'ua', 'ubank', 'ubs', 'uconnect', 'ug', 'uk', 'unicom',
+    'university', 'uno', 'uol', 'ups', 'us', 'uy', 'uz',
     # v
-    'va', 'vacations', 'vc', 've', 'vegas', 'ventures',
-    'versicherung', 'vet', 'vg', 'vi', 'viajes', 'video', 'villas',
-    'vision', 'vlaanderen', 'vn', 'vodka', 'vote', 'voting', 'voto',
-    'voyage', 'vu',
+    'va', 'vacations', 'vana', 'vanguard', 'vc', 've', 'vegas',
+    'ventures', 'verisign', 'versicherung', 'vet', 'vg', 'vi',
+    'viajes', 'video', 'vig', 'viking', 'villas', 'vin', 'vip',
+    'virgin', 'visa', 'vision', 'vista', 'vistaprint', 'viva',
+    'vivo', 'vlaanderen', 'vn', 'vodka', 'volkswagen', 'volvo',
+    'vote', 'voting', 'voto', 'voyage', 'vu', 'vuelos',
     # w
-    'wales', 'wang', 'watch', 'webcam', 'website', 'wed', 'wedding',
-    'wf', 'whoswho', 'wien', 'wiki', 'williamhill', 'wme', 'work',
-    'works', 'world', 'ws', 'wtc', 'wtf',
+    'wales', 'walmart', 'walter', 'wang', 'wanggou', 'warman',
+    'watch', 'watches', 'weather', 'weatherchannel', 'webcam',
+    'weber', 'website', 'wed', 'wedding', 'weibo', 'weir', 'wf',
+    'whoswho', 'wien', 'wiki', 'williamhill', 'win', 'windows',
+    'wine', 'winners', 'wme', 'wolterskluwer', 'woodside', 'work',
+    'works', 'world', 'wow', 'ws', 'wtc', 'wtf',
     # x
-    'xn--1qqw23a', 'xn--3bst00m', 'xn--3ds443g', 'xn--3e0b707e',
-    'xn--45brj9c', 'xn--45q11c', 'xn--4gbrim', 'xn--55qw42g',
-    'xn--55qx5d', 'xn--6frz82g', 'xn--6qq986b3xl', 'xn--80adxhks',
-    'xn--80ao21a', 'xn--80asehdb', 'xn--80aswg', 'xn--90a3ac',
-    'xn--90ais', 'xn--b4w605ferd', 'xn--c1avg', 'xn--cg4bki',
-    'xn--clchc0ea0b2g2a9gcd', 'xn--czr694b', 'xn--czrs0t',
-    'xn--czru2d', 'xn--d1acj3b', 'xn--d1alf', 'xn--fiq228c5hs',
-    'xn--fiq64b', 'xn--fiqs8s', 'xn--fiqz9s', 'xn--flw351e',
-    'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--gecrj9c', 'xn--h2brj9c',
-    'xn--hxt814e', 'xn--i1b6b1a6a2e', 'xn--io0a7i', 'xn--j1amh',
-    'xn--j6w193g', 'xn--kprw13d', 'xn--kpry57d', 'xn--kput3i',
-    'xn--l1acc', 'xn--lgbbat1ad8j', 'xn--mgb9awbf',
-    'xn--mgba3a4f16a', 'xn--mgbaam7a8h', 'xn--mgbab2bd',
-    'xn--mgbayh7gpa', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg',
-    'xn--mgberp4a5d4ar', 'xn--mgbx4cd0ab', 'xn--ngbc5azd',
-    'xn--node', 'xn--nqv7f', 'xn--nqv7fs00ema', 'xn--o3cw4h',
-    'xn--ogbpf8fl', 'xn--p1acf', 'xn--p1ai', 'xn--pgbs0dh',
-    'xn--q9jyb4c', 'xn--qcka1pmc', 'xn--rhqv96g', 'xn--s9brj9c',
-    'xn--ses554g', 'xn--unup4y', 'xn--vermgensberater-ctb',
-    'xn--vermgensberatung-pwb', 'xn--vhquv', 'xn--wgbh1c',
+    'xbox', 'xerox', 'xfinity', 'xihuan', 'xin', 'xn--11b4c3d',
+    'xn--1ck2e1b', 'xn--1qqw23a', 'xn--2scrj9c', 'xn--30rr7y',
+    'xn--3bst00m', 'xn--3ds443g', 'xn--3e0b707e', 'xn--3hcrj9c',
+    'xn--3oq18vl8pn36a', 'xn--3pxu8k', 'xn--42c2d9a', 'xn--45br5cyl',
+    'xn--45brj9c', 'xn--45q11c', 'xn--4gbrim', 'xn--54b7fta0cc',
+    'xn--55qw42g', 'xn--55qx5d', 'xn--5su34j936bgsg', 'xn--5tzm5g',
+    'xn--6frz82g', 'xn--6qq986b3xl', 'xn--80adxhks', 'xn--80ao21a',
+    'xn--80aqecdr1a', 'xn--80asehdb', 'xn--80aswg', 'xn--8y0a063a',
+    'xn--90a3ac', 'xn--90ae', 'xn--90ais', 'xn--9dbq2a',
+    'xn--9et52u', 'xn--9krt00a', 'xn--b4w605ferd',
+    'xn--bck1b9a5dre4c', 'xn--c1avg', 'xn--c2br7g', 'xn--cck2b3b',
+    'xn--cg4bki', 'xn--clchc0ea0b2g2a9gcd', 'xn--czr694b',
+    'xn--czrs0t', 'xn--czru2d', 'xn--d1acj3b', 'xn--d1alf',
+    'xn--e1a4c', 'xn--eckvdtc9d', 'xn--efvy88h', 'xn--estv75g',
+    'xn--fct429k', 'xn--fhbei', 'xn--fiq228c5hs', 'xn--fiq64b',
+    'xn--fiqs8s', 'xn--fiqz9s', 'xn--fjq720a', 'xn--flw351e',
+    'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--fzys8d69uvgm',
+    'xn--g2xx48c', 'xn--gckr3f0f', 'xn--gecrj9c', 'xn--gk3at1e',
+    'xn--h2breg3eve', 'xn--h2brj9c', 'xn--h2brj9c8c', 'xn--hxt814e',
+    'xn--i1b6b1a6a2e', 'xn--imr513n', 'xn--io0a7i', 'xn--j1aef',
+    'xn--j1amh', 'xn--j6w193g', 'xn--jlq61u9w7b', 'xn--jvr189m',
+    'xn--kcrx77d1x4a', 'xn--kprw13d', 'xn--kpry57d', 'xn--kpu716f',
+    'xn--kput3i', 'xn--l1acc', 'xn--lgbbat1ad8j', 'xn--mgb9awbf',
+    'xn--mgba3a3ejt', 'xn--mgba3a4f16a', 'xn--mgba7c0bbn0a',
+    'xn--mgbaakc7dvf', 'xn--mgbaam7a8h', 'xn--mgbab2bd',
+    'xn--mgbai9azgqp6j', 'xn--mgbayh7gpa', 'xn--mgbb9fbpob',
+    'xn--mgbbh1a', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg',
+    'xn--mgbca7dzdo', 'xn--mgberp4a5d4ar', 'xn--mgbgu82a',
+    'xn--mgbi4ecexp', 'xn--mgbpl2fh', 'xn--mgbt3dhd', 'xn--mgbtx2b',
+    'xn--mgbx4cd0ab', 'xn--mix891f', 'xn--mk1bu44c', 'xn--mxtq1m',
+    'xn--ngbc5azd', 'xn--ngbe9e0a', 'xn--ngbrx', 'xn--node',
+    'xn--nqv7f', 'xn--nqv7fs00ema', 'xn--nyqy26a', 'xn--o3cw4h',
+    'xn--ogbpf8fl', 'xn--p1acf', 'xn--p1ai', 'xn--pbt977c',
+    'xn--pgbs0dh', 'xn--pssy2u', 'xn--q9jyb4c', 'xn--qcka1pmc',
+    'xn--qxam', 'xn--rhqv96g', 'xn--rovu88b', 'xn--rvc1e0am3e',
+    'xn--s9brj9c', 'xn--ses554g', 'xn--t60b56a', 'xn--tckwe',
+    'xn--tiq49xqyj', 'xn--unup4y', 'xn--vermgensberater-ctb',
+    'xn--vermgensberatung-pwb', 'xn--vhquv', 'xn--vuq861b',
+    'xn--w4r85el8fhu5dnra', 'xn--w4rs40l', 'xn--wgbh1c',
     'xn--wgbl6a', 'xn--xhq521b', 'xn--xkc2al3hye2a',
-    'xn--xkc2dl3a5ee0h', 'xn--yfro4i67o', 'xn--ygbi2ammx',
-    'xn--zfr164b', 'xxx', 'xyz',
+    'xn--xkc2dl3a5ee0h', 'xn--y9a3aq', 'xn--yfro4i67o',
+    'xn--ygbi2ammx', 'xn--zfr164b', 'xperia', 'xxx', 'xyz',
     # y
-    'yachts', 'yandex', 'ye', 'yodobashi', 'yoga', 'yokohama',
-    'youtube', 'yt',
+    'yachts', 'yahoo', 'yamaxun', 'yandex', 'ye', 'yodobashi',
+    'yoga', 'yokohama', 'you', 'youtube', 'yt', 'yun',
     # z
-    'za', 'zip', 'zm', 'zone', 'zuerich', 'zw'
+    'za', 'zappos', 'zara', 'zero', 'zip', 'zippo', 'zm', 'zone',
+    'zuerich', 'zw'
 ]
 
 

scripts/access.wsgi

@@ -35,11 +35,17 @@
 #
 
 URL_CHECK_ACCESS = 'http://127.0.0.1:8002/%(app)s/default/check_access'
+PY2 = sys.version_info[0] == 2
 
 def allow_access(environ,host):
+    if PY2:
+        import urllib2
+        from urllib import urlencode
+    else:
+        from urllib import request as urllib2
+        from urllib.parse import urlencode
+
     import os
-    import urllib
-    import urllib2
     import datetime
     header = '%s @ %s ' % (datetime.datetime.now(),host) + '='*20
     pprint = '\n'.join('%s:%s' % item for item in environ.items())
@@ -56,7 +62,7 @@ def allow_access(environ,host):
         if key.startswith('HTTP_'):
             headers[key[5:]] = environ[key] # this passes the cookies through!
     try:
-        data = urllib.urlencode({'request_uri':environ['REQUEST_URI']})
+        data = urlencode({'request_uri':environ['REQUEST_URI']})
         request = urllib2.Request(URL_CHECK_ACCESS % dict(app=app),data,headers)
         response = urllib2.urlopen(request).read().strip().lower()
         if response.startswith('true'): return True

scripts/tickets2db.py

@@ -31,7 +31,7 @@ hashes = {}
 
 while 1:
     if request.tickets_db:
-        print "You're storing tickets yet in database"
+        print("You're storing tickets yet in database")
         sys.exit(1)
 
     for file in os.listdir(errors_path):

scripts/tickets2slack.py

@@ -22,7 +22,7 @@ import json
 try:
     import requests
 except ImportError as e:
-    print "missing module 'Requests', aborting."
+    print("missing module 'Requests', aborting.")
     sys.exit(1)
 
 from gluon import URL

scripts/update_languages.py

@@ -18,7 +18,7 @@ from gluon.utf8 import Utf8
 from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
 from gluon.languages import findT
 
-# This script can be run with no arguments (which sets the language folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (language folder path and default language).
+# This script can be run with no arguments (which sets the application folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (application folder path and default language).
 # When run, it will update the default language, as well as strip all of the strings found in the non-default languages but not in the default language, and add the strings found in the default language to the non-default languages it is not, making sure translators don't do additional work that will never be used.
 
 def read_dict_aux(filename):
@@ -53,21 +53,21 @@ def write_file(file, contents):
 	file.close()
 
 def update_languages(cwd, default_lang):
-	defaultfp = os.path.join(cwd, '%s.py' %default_lang)
+	defaultfp = os.path.join(cwd, "languages", '%s.py' %default_lang)
 	findT(cwd, default_lang)
 	default = read_dict(defaultfp)
 
-	for x in os.listdir(cwd):
-		if x == default_lang or x.startswith("plural-"): continue
+	for lang in os.listdir(os.path.join(cwd, "languages")):
+		if lang == default_lang+".py" or lang.startswith("plural-"): continue
 		
-		i18n = read_dict(os.path.join(cwd, x))
+		i18n = read_dict(os.path.join(cwd, "languages", lang))
 		if i18n:
-			nd = default
-			for k_d1 in i18n:
-				if k_d1 in default:
-					nd[k_d1] = i18n[k_d1]
-			write_file(open(x, 'w'), nd)
-			print x
+			new_dict = default
+			for phrase in i18n:
+				if phrase in default:
+					new_dict[phrase] = i18n[phrase]
+			write_file(open(os.path.join(cwd, "languages", lang), 'w'), new_dict)
+			print lang
 
 if __name__ == "__main__":
 	cwd = os.getcwd()

scripts/zip_static_files.py

@@ -14,7 +14,7 @@ def zip_static(filelist=[]):
         extension = os.path.splitext(fi)
         extension = len(extension) > 1 and extension[1] or None
         if not extension or extension not in ALLOWED_EXTS:
-            print 'skipping %s' % os.path.basename(fi)
+            print('skipping %s' % os.path.basename(fi))
             continue
         fstats = os.stat(fi)
         atime, mtime = fstats.st_atime, fstats.st_mtime
@@ -23,10 +23,10 @@ def zip_static(filelist=[]):
             zstats = os.stat(gfi)
             zatime, zmtime = zstats.st_atime, zstats.st_mtime
             if zatime == atime and zmtime == mtime:
-                print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
+                print('skipping %s, already gzipped to the latest version' % os.path.basename(fi))
                 continue
-        print 'gzipping %s to %s' % (
-            os.path.basename(fi), os.path.basename(gfi))
+        print('gzipping %s to %s' % (
+            os.path.basename(fi), os.path.basename(gfi)))
         f_in = open(fi, 'rb')
         f_out = gzip.open(gfi, 'wb')
         f_out.writelines(f_in)
@@ -36,7 +36,7 @@ def zip_static(filelist=[]):
         saved = fstats.st_size - os.stat(gfi).st_size
         tsave += saved
 
-    print 'saved %s KB' % (int(tsave) / 1000.0)
+    print('saved %s KB' % (int(tsave) / 1000.0))
 
 if __name__ == '__main__':
     ALLOWED_EXTS = ['.css', '.js']

setup.py

@@ -65,6 +65,9 @@ def start():
                     'gluon/contrib/pysimplesoap',
                     'gluon/contrib/pg8000',
                     'gluon/contrib/plural_rules',
+                    'gluon/contrib/minify',
+                    'gluon/contrib/pyaes',
+                    'gluon/contrib/pyuca',
                     'gluon/tests',
                     ],
           package_data={'gluon': ['env.tar']},