From aa0313c59b69f367a6ba4247a2ba51444040ad78 Mon Sep 17 00:00:00 2001 From: Jose de Soto Date: Fri, 21 Jul 2017 10:41:58 +0200 Subject: [PATCH 01/43] del_membership prevent update self user if user_id has a value --- gluon/authapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/authapi.py b/gluon/authapi.py index 3347cb68..f28f57a1 100644 --- a/gluon/authapi.py +++ b/gluon/authapi.py @@ -557,7 +557,7 @@ class AuthAPI(object): self.log_event(self.messages['del_membership_log'], dict(user_id=user_id, group_id=group_id)) ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete() - if group_id in self.user_groups: + if group_id in self.user_groups and user_id == self.user_id: del self.user_groups[group_id] return ret From 19efbfecfaa30c9b5bdc58d247aacb4638fdbfaf Mon Sep 17 00:00:00 2001 From: Jan Kotyz Date: Thu, 27 Jul 2017 11:27:41 +0200 Subject: [PATCH 02/43] Fixes 1700 --- gluon/tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gluon/tools.py b/gluon/tools.py index 1f54e746..9c535f4e 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1320,7 +1320,7 @@ class AuthJWT(object): # is the following safe or should we use # calendar.timegm(datetime.datetime.utcnow().timetuple()) # result seem to be the same (seconds since epoch, in UTC) - now = time.mktime(datetime.datetime.now().timetuple()) + now = time.mktime(datetime.datetime.utcnow().timetuple()) expires = now + self.expiration payload = dict( hmac_key=session_auth['hmac_key'], @@ -1332,7 +1332,7 @@ class AuthJWT(object): return payload def refresh_token(self, orig_payload): - now = time.mktime(datetime.datetime.now().timetuple()) + now = time.mktime(datetime.datetime.utcnow().timetuple()) if self.verify_expiration: orig_exp = orig_payload['exp'] if orig_exp + self.leeway < now: @@ -2863,7 +2863,7 @@ class Auth(AuthAPI): auth.settings.auth_two_factor_enabled = True auth.messages.two_factor_comment = "Verify your OTP Client for the code." - auth.settings.two_factor_methods = [lambda user, + auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)] auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)] From 561828fa604f4665f0c4e9618d69ba1381309a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Fri, 28 Jul 2017 02:20:54 +0100 Subject: [PATCH 03/43] Fix autocomplete called after user_signature chk Fixes #1699 which was caused by the autocomplete widget only being called after a user_signature check and the signature not being there. The default values for user_signature and hash_vars make this work with the grid when it has the user_signature=True option without any problem when that isn't the case. Users can disable it if they want or change it according to what they're verifying in their controllers. The autocomplete widget will still fail with the default kwargs in a situation where it is called in a controller function that first verifies the URL signature with hash_vars=True. For users having that problem, there's no way around this without inserting a security flaw, so they must sadly change their application code to give the autocomplete widget the needed user_signature and hash_vars argument. --- gluon/sqlhtml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 0468c0f7..b957ff0d 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -658,7 +658,8 @@ class AutocompleteWidget(object): orderby=None, limitby=(0, 10), distinct=False, keyword='_autocomplete_%(tablename)s_%(fieldname)s', min_length=2, help_fields=None, help_string=None, - at_beginning=True, default_var='ac'): + at_beginning=True, default_var='ac', user_signature=True, + hash_vars=False): self.help_fields = help_fields or [] self.help_string = help_string @@ -683,7 +684,8 @@ class AutocompleteWidget(object): if hasattr(request, 'application'): urlvars = request.vars urlvars[default_var] = 1 - self.url = URL(args=request.args, vars=urlvars) + self.url = URL(args=request.args, vars=urlvars, + user_signature=user_signature, hash_vars=hash_vars) self.run_callback = True else: self.url = request From 0e613f2d7fd8d0fea4e0d76d8dec11ca26b0e75d Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 28 Jul 2017 16:05:35 -0500 Subject: [PATCH 04/43] allow DAL(..., adapter_args=dict(migrator=InDBMigrator)) --- gluon/dal.py | 1 + gluon/packages/dal | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gluon/dal.py b/gluon/dal.py index ceecf2e0..215fd4e1 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -14,6 +14,7 @@ from pydal import DAL as DAL from pydal import Field from pydal.objects import Row, Rows, Table, Query, Set, Expression from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon +from pydal.migrator import Migrator, InDBMigrator from gluon.serializers import custom_json, xml from gluon.utils import web2py_uuid from gluon import sqlhtml diff --git a/gluon/packages/dal b/gluon/packages/dal index 3e0fd7c0..6eece919 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 3e0fd7c01c340dec0f18a228fb91b3334b89f7f6 +Subproject commit 6eece9197148daf9e470036cc5ab1ad87ac57183 From cd2ec98a2600ac51bac92ca73e8619135eff12ff Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 29 Jul 2017 01:31:25 -0500 Subject: [PATCH 05/43] grid.dbset --- gluon/sqlhtml.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 0468c0f7..ec236a58 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -2159,6 +2159,7 @@ class SQLFORM(FORM): represent_none=None, showblobs=False): + dbset = None formstyle = formstyle or current.response.formstyle if isinstance(query, Set): query = query.query @@ -3037,6 +3038,7 @@ class SQLFORM(FORM): res.view_form = view_form res.search_form = search_form res.rows = rows + res.dbset = dbset return res @staticmethod From f3bba29287bb36a8b437c0de3baa8607022de47d Mon Sep 17 00:00:00 2001 From: Jose de Soto Date: Mon, 31 Jul 2017 17:46:46 +0200 Subject: [PATCH 06/43] Allow response.json to be sortable or not --- gluon/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluon/serializers.py b/gluon/serializers.py index d53740d7..032cff81 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -119,8 +119,8 @@ def xml(value, encoding='UTF-8', key='document', quote=True): return ('' % encoding) + str(xml_rec(value, key, quote)) -def json(value, default=custom_json, indent=None): - value = json_parser.dumps(value, default=default, sort_keys=True, indent=indent) +def json(value, default=custom_json, indent=None, sort_keys=False): + value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent) # replace JavaScript incompatible spacing # http://timelessrepo.com/json-isnt-a-javascript-subset # PY3 FIXME From 1014d3e86e7a3b76ae7228a25f8ad2e921411ac3 Mon Sep 17 00:00:00 2001 From: Jose de Soto Date: Mon, 31 Jul 2017 18:33:15 +0200 Subject: [PATCH 07/43] new parameter to auto create or not users with alternate login methods --- gluon/tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gluon/tools.py b/gluon/tools.py index 1f54e746..4b34be77 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1792,6 +1792,7 @@ class Auth(AuthAPI): servicevalidate='serviceValidate', proxyvalidate='proxyValidate', logout='logout'), + cas_create_user=True, extra_fields={}, actions_disabled=[], controller=controller, @@ -2284,6 +2285,7 @@ class Auth(AuthAPI): If the user doesn't yet exist, then they are created. """ table_user = self.table_user() + create_user = self.settings.cas_create_user user = None checks = [] # make a guess about who this user is @@ -2316,6 +2318,11 @@ class Auth(AuthAPI): update_keys[key] = keys[key] user.update_record(**update_keys) elif checks: + if create_user is False: + # Remove current open session a send message + self.logout(next=None, onlogout=None, log=None) + raise HTTP(403, "Forbidden. User need to be created first.") + if 'first_name' not in keys and 'first_name' in table_user.fields: guess = keys.get('email', 'anonymous').split('@')[0] keys['first_name'] = keys.get('username', guess) From d5167f2ed6e889b0b3ca874542f807eadbeb3c5b Mon Sep 17 00:00:00 2001 From: Jose de Soto Date: Mon, 31 Jul 2017 19:00:24 +0200 Subject: [PATCH 08/43] change_password_url parameter for alternate login methods --- gluon/contrib/login_methods/cas_auth.py | 13 +++++++++++- gluon/contrib/login_methods/saml2_auth.py | 24 ++++++++++++++++++++++- gluon/tools.py | 10 ++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/gluon/contrib/login_methods/cas_auth.py b/gluon/contrib/login_methods/cas_auth.py index 1a07c5f1..73be8ae1 100644 --- a/gluon/contrib/login_methods/cas_auth.py +++ b/gluon/contrib/login_methods/cas_auth.py @@ -49,7 +49,8 @@ class CasAuth(object): email=lambda v: v.get('email', None), user_id=lambda v: v['user']), casversion=1, - casusername='cas:user' + casusername='cas:user', + change_password_url=None ): self.urlbase = urlbase self.cas_login_url = "%s/%s" % (self.urlbase, actions[0]) @@ -64,6 +65,9 @@ class CasAuth(object): #vars=current.request.vars, scheme=True) + # URL to let users change their password in the IDP system + self.cas_change_password_url = change_password_url + def login_url(self, next="/"): current.session.token = self._CAS_login() return next @@ -74,6 +78,10 @@ class CasAuth(object): self._CAS_logout() return next + def change_password_url(self, next="/"): + self._CAS_change_password() + return next + def get_user(self): user = current.session.token if user: @@ -135,3 +143,6 @@ class CasAuth(object): redirects to the CAS logout page """ redirect("%s?service=%s" % (self.cas_logout_url, self.cas_my_url)) + + def _CAS_change_password(self): + redirect(self.cas_change_password_url) diff --git a/gluon/contrib/login_methods/saml2_auth.py b/gluon/contrib/login_methods/saml2_auth.py index 42fe24db..462d9845 100644 --- a/gluon/contrib/login_methods/saml2_auth.py +++ b/gluon/contrib/login_methods/saml2_auth.py @@ -145,10 +145,16 @@ class Saml2Auth(object): username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], - )): + ), logout_url=None, change_password_url=None): self.config_file = config_file self.maps = maps + # URL for redirecting users to when they sign out + self.saml_logout_url = logout_url + + # URL to let users change their password in the IDP system + self.saml_change_password_url = change_password_url + def login_url(self, next="/"): d = saml2_handler(current.session, current.request) if 'url' in d: @@ -170,6 +176,12 @@ class Saml2Auth(object): def logout_url(self, next="/"): current.session.saml2_info = None + current.session.auth = None + self._SAML_logout() + return next + + def change_password_url(self, next="/"): + self._SAML_change_password() return next def get_user(self): @@ -180,3 +192,13 @@ class Saml2Auth(object): d[key] = self.maps[key](user) return d return None + + def _SAML_logout(self): + """ + exposed SAML.logout() + redirects to the SAML logout page + """ + redirect(self.saml_logout_url) + + def _SAML_change_password(self): + redirect(self.saml_change_password_url) diff --git a/gluon/tools.py b/gluon/tools.py index 4b34be77..a4b28aa0 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -3663,6 +3663,16 @@ class Auth(AuthAPI): if not self.is_logged_in(): redirect(self.settings.login_url, client_side=self.settings.client_side) + + # Go to external link to change the password + if self.settings.login_form != self: + cas = self.settings.login_form + # To prevent error if change_password_url function is not defined in alternate login + if hasattr(cas, 'change_password_url'): + next = cas.change_password_url(next) + if next is not None: + redirect(next) + db = self.db table_user = self.table_user() s = db(table_user.id == self.user.id) From 213c4ee7d116b18f13be4df641d3536b3fa333c2 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 1 Aug 2017 10:26:33 -0500 Subject: [PATCH 09/43] fixed use of whitespaces --- gluon/authapi.py | 10 +++++----- gluon/newcron.py | 2 +- gluon/packages/dal | 2 +- gluon/sqlhtml.py | 8 ++++---- gluon/tools.py | 6 +++--- gluon/widget.py | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/gluon/authapi.py b/gluon/authapi.py index f28f57a1..d0922384 100644 --- a/gluon/authapi.py +++ b/gluon/authapi.py @@ -20,10 +20,10 @@ DEFAULT = lambda: None class AuthAPI(object): """ AuthAPI is a barebones Auth implementation which does not have a concept of - HTML forms or redirects, emailing or even an URL, you are responsible for + HTML forms or redirects, emailing or even an URL, you are responsible for all that if you use it. The main Auth functions such as login, logout, register, profile are designed - in a Dict In -> Dict Out logic so, for instance, if you set + in a Dict In -> Dict Out logic so, for instance, if you set registration_requires_verification you are responsible for sending the key to the user and even rolling back the transaction if you can't do it. @@ -245,13 +245,13 @@ class AuthAPI(object): migrate = db._migrate if fake_migrate is None: fake_migrate = db._fake_migrate - + settings = self.settings if username is None: username = settings.use_username else: settings.use_username = username - + if not self.signature: self.define_signature() if signature is True: @@ -1012,7 +1012,7 @@ class AuthAPI(object): ): """ Verify a given registration_key actually exists in the user table. - Resets the key to empty string '' or 'pending' if + Resets the key to empty string '' or 'pending' if setttings.registration_requires_approval is true. Keyword Args: diff --git a/gluon/newcron.py b/gluon/newcron.py index d422f074..d53ecbe5 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -225,7 +225,7 @@ def parsecronline(line): params = line.strip().split(None, 6) if len(params) < 7: return None - daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, + daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 'fri': 5, 'sat': 6} for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): if not s in [None, '*']: diff --git a/gluon/packages/dal b/gluon/packages/dal index 6eece919..f66ad176 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 6eece9197148daf9e470036cc5ab1ad87ac57183 +Subproject commit f66ad176943f55d80a4e3f9f8cab659a3852c343 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 810d1306..cb317e82 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -1922,8 +1922,8 @@ class SQLFORM(FORM): del attributes['table_name'] # Clone fields, while passing tables straight through - fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields] - + fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields] + return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes) @staticmethod @@ -3161,8 +3161,8 @@ class SQLFORM(FORM): # if isinstance(linked_tables, dict): # linked_tables = linked_tables.get(table._tablename, []) if linked_tables is None or referee in linked_tables: - field.represent = (lambda id, r=None, referee=referee, rep=field.represent: - A(callable(rep) and rep(id) or id, + field.represent = (lambda id, r=None, referee=referee, rep=field.represent: + A(callable(rep) and rep(id) or id, cid=request.cid, _href=url(args=['view', referee, id]))) except (KeyError, ValueError, TypeError): redirect(URL(args=table._tablename)) diff --git a/gluon/tools.py b/gluon/tools.py index 9e541112..0cf9b205 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1792,7 +1792,7 @@ class Auth(AuthAPI): servicevalidate='serviceValidate', proxyvalidate='proxyValidate', logout='logout'), - cas_create_user=True, + cas_create_user=True, extra_fields={}, actions_disabled=[], controller=controller, @@ -2285,7 +2285,7 @@ class Auth(AuthAPI): If the user doesn't yet exist, then they are created. """ table_user = self.table_user() - create_user = self.settings.cas_create_user + create_user = self.settings.cas_create_user user = None checks = [] # make a guess about who this user is @@ -2318,7 +2318,7 @@ class Auth(AuthAPI): update_keys[key] = keys[key] user.update_record(**update_keys) elif checks: - if create_user is False: + if create_user is False: # Remove current open session a send message self.logout(next=None, onlogout=None, log=None) raise HTTP(403, "Forbidden. User need to be created first.") diff --git a/gluon/widget.py b/gluon/widget.py index 0804fa96..d56be20a 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -140,7 +140,7 @@ class web2pyDialog(object): else: import tkinter from tkinter import messagebox - + bg_color = 'white' root.withdraw() @@ -463,7 +463,7 @@ class web2pyDialog(object): import tkMessageBox as messagebox else: from tkinter import messagebox - + messagebox.showerror('web2py start server', message) def start(self): From 4a47bb8e83acaf451eac36f440e7a368eecc7369 Mon Sep 17 00:00:00 2001 From: Andrew Willimott Date: Wed, 2 Aug 2017 21:47:26 +1200 Subject: [PATCH 10/43] =?UTF-8?q?d3=5Fgraph=5Fmodel=20fixed,=20no=20longer?= =?UTF-8?q?=20hardcoded=20for=20a=20=E2=80=9Cdb=E2=80=9D=20database?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- applications/admin/controllers/appadmin.py | 49 +++++++++---------- applications/examples/controllers/appadmin.py | 49 +++++++++---------- applications/welcome/controllers/appadmin.py | 49 +++++++++---------- 3 files changed, 72 insertions(+), 75 deletions(-) diff --git a/applications/admin/controllers/appadmin.py b/applications/admin/controllers/appadmin.py index 139ad1aa..74240e59 100644 --- a/applications/admin/controllers/appadmin.py +++ b/applications/admin/controllers/appadmin.py @@ -657,37 +657,36 @@ def d3_graph_model(): Create a list of table dicts, called "nodes" """ - data = {} nodes = [] links = [] - subgraphs = dict() + for database in databases: + db = eval_in_global_env(database) + for tablename in db.tables: + fields = [] + for field in db[tablename]: + f_type = field.type + if not isinstance(f_type,str): + disp = ' ' + elif f_type == 'string': + disp = field.length + elif f_type == 'id': + disp = "PK" + elif f_type.startswith('reference') or \ + f_type.startswith('list:reference'): + disp = "FK" + else: + disp = ' ' + fields.append(dict(name= field.name, type=field.type, disp = disp)) - for tablename in db.tables: - fields = [] - for field in db[tablename]: - f_type = field.type - if not isinstance(f_type,str): - disp = ' ' - elif f_type == 'string': - disp = field.length - elif f_type == 'id': - disp = "PK" - elif f_type.startswith('reference') or \ - f_type.startswith('list:reference'): - disp = "FK" - else: - disp = ' ' - fields.append(dict(name= field.name, type=field.type, disp = disp)) + if isinstance(f_type,str) and ( + f_type.startswith('reference') or + f_type.startswith('list:reference')): + referenced_table = f_type.split()[1].split('.')[0] - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] + links.append(dict(source=tablename, target = referenced_table)) - links.append(dict(source=tablename, target = referenced_table)) - - nodes.append(dict(name=tablename, type="table", fields = fields)) + nodes.append(dict(name=tablename, type="table", fields = fields)) # d3 v4 allows individual modules to be specified. The complete d3 library is included below. response.files.append(URL('admin','static','js/d3.min.js')) diff --git a/applications/examples/controllers/appadmin.py b/applications/examples/controllers/appadmin.py index 139ad1aa..74240e59 100644 --- a/applications/examples/controllers/appadmin.py +++ b/applications/examples/controllers/appadmin.py @@ -657,37 +657,36 @@ def d3_graph_model(): Create a list of table dicts, called "nodes" """ - data = {} nodes = [] links = [] - subgraphs = dict() + for database in databases: + db = eval_in_global_env(database) + for tablename in db.tables: + fields = [] + for field in db[tablename]: + f_type = field.type + if not isinstance(f_type,str): + disp = ' ' + elif f_type == 'string': + disp = field.length + elif f_type == 'id': + disp = "PK" + elif f_type.startswith('reference') or \ + f_type.startswith('list:reference'): + disp = "FK" + else: + disp = ' ' + fields.append(dict(name= field.name, type=field.type, disp = disp)) - for tablename in db.tables: - fields = [] - for field in db[tablename]: - f_type = field.type - if not isinstance(f_type,str): - disp = ' ' - elif f_type == 'string': - disp = field.length - elif f_type == 'id': - disp = "PK" - elif f_type.startswith('reference') or \ - f_type.startswith('list:reference'): - disp = "FK" - else: - disp = ' ' - fields.append(dict(name= field.name, type=field.type, disp = disp)) + if isinstance(f_type,str) and ( + f_type.startswith('reference') or + f_type.startswith('list:reference')): + referenced_table = f_type.split()[1].split('.')[0] - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] + links.append(dict(source=tablename, target = referenced_table)) - links.append(dict(source=tablename, target = referenced_table)) - - nodes.append(dict(name=tablename, type="table", fields = fields)) + nodes.append(dict(name=tablename, type="table", fields = fields)) # d3 v4 allows individual modules to be specified. The complete d3 library is included below. response.files.append(URL('admin','static','js/d3.min.js')) diff --git a/applications/welcome/controllers/appadmin.py b/applications/welcome/controllers/appadmin.py index 139ad1aa..74240e59 100644 --- a/applications/welcome/controllers/appadmin.py +++ b/applications/welcome/controllers/appadmin.py @@ -657,37 +657,36 @@ def d3_graph_model(): Create a list of table dicts, called "nodes" """ - data = {} nodes = [] links = [] - subgraphs = dict() + for database in databases: + db = eval_in_global_env(database) + for tablename in db.tables: + fields = [] + for field in db[tablename]: + f_type = field.type + if not isinstance(f_type,str): + disp = ' ' + elif f_type == 'string': + disp = field.length + elif f_type == 'id': + disp = "PK" + elif f_type.startswith('reference') or \ + f_type.startswith('list:reference'): + disp = "FK" + else: + disp = ' ' + fields.append(dict(name= field.name, type=field.type, disp = disp)) - for tablename in db.tables: - fields = [] - for field in db[tablename]: - f_type = field.type - if not isinstance(f_type,str): - disp = ' ' - elif f_type == 'string': - disp = field.length - elif f_type == 'id': - disp = "PK" - elif f_type.startswith('reference') or \ - f_type.startswith('list:reference'): - disp = "FK" - else: - disp = ' ' - fields.append(dict(name= field.name, type=field.type, disp = disp)) + if isinstance(f_type,str) and ( + f_type.startswith('reference') or + f_type.startswith('list:reference')): + referenced_table = f_type.split()[1].split('.')[0] - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] + links.append(dict(source=tablename, target = referenced_table)) - links.append(dict(source=tablename, target = referenced_table)) - - nodes.append(dict(name=tablename, type="table", fields = fields)) + nodes.append(dict(name=tablename, type="table", fields = fields)) # d3 v4 allows individual modules to be specified. The complete d3 library is included below. response.files.append(URL('admin','static','js/d3.min.js')) From b7270df9c3f3b87895e9420afdc11b68465ebc12 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 4 Aug 2017 09:58:05 -0500 Subject: [PATCH 11/43] fixed issue topic/web2py/UnN6AyOh2Lg thanks Paul --- gluon/sqlhtml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index cb317e82..0653c4c7 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -1939,11 +1939,13 @@ class SQLFORM(FORM): if settings.global_settings.web2py_runtime_gae: return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields]) else: + if not (sfields and key and key.split()): + return fields[0].table return reduce(lambda a,b:a&b,[ reduce(lambda a,b: a|b, [ field.contains(k) for field in sfields] ) for k in key.split()]) - + # from https://groups.google.com/forum/#!topic/web2py/hKe6lI25Bv4 # needs testing... #words = key.split(' ') if key else [] From ebc614bf91877d82ce34bb57c8d5ce9261f3620c Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 4 Aug 2017 10:21:43 -0500 Subject: [PATCH 12/43] fixed has_ssl, thanks leonel --- gluon/main.py | 2 +- gluon/packages/dal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gluon/main.py b/gluon/main.py index 32103ea6..e5d73e21 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -745,7 +745,7 @@ class HttpServer(object): sock_list = [ip, port] if not ssl_certificate or not ssl_private_key: logger.info('SSL is off') - elif not rocket.ssl: + elif not rocket.has_ssl: logger.warning('Python "ssl" module unavailable. SSL is OFF') elif not exists(ssl_certificate): logger.warning('unable to open SSL certificate. SSL is OFF') diff --git a/gluon/packages/dal b/gluon/packages/dal index f66ad176..c707d558 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit f66ad176943f55d80a4e3f9f8cab659a3852c343 +Subproject commit c707d558995d0318022f3a26935f1100020975fc From a744835f2159dfe1de1f2d882ab81a076ad664d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Sat, 5 Aug 2017 13:27:44 +0100 Subject: [PATCH 13/43] Fix response.download with nonasccii filenames Fixes #1718 --- gluon/globals.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gluon/globals.py b/gluon/globals.py index 267ea661..7ec9a0a1 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -14,7 +14,7 @@ Contains the classes for the global used variables: """ from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \ - unicodeT, long, hashlib_md5 + unicodeT, long, hashlib_md5, urllib_quote from gluon.storage import Storage, List from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE from gluon.contenttype import contenttype @@ -641,6 +641,11 @@ class Response(Storage): if download_filename is None: download_filename = filename if attachment: + # Browsers still don't have a simple uniform way to have non ascii + # characters in the filename so for now we are percent encoding it + if isinstance(download_filename, unicodeT): + download_filename = download_filename.encode('utf-8') + download_filename = urllib_quote(download_filename) headers['Content-Disposition'] = \ 'attachment; filename="%s"' % download_filename.replace('"', '\"') return self.stream(stream, chunk_size=chunk_size, request=request) From 90e20a4f393fe5bdced41ad30c02f8dbfd8bc34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Sun, 6 Aug 2017 17:01:27 +0100 Subject: [PATCH 14/43] Fix BEAUTIFY trying to display uploaded file contents Fixes #1717 --- gluon/html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/html.py b/gluon/html.py index f5383109..73eae9b8 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -2431,7 +2431,7 @@ class BEAUTIFY(DIV): if level == 0: return for c in self.components: - if hasattr(c, 'value') and not callable(c.value): + if hasattr(c, 'value') and not callable(c.value) and not isinstance(c, cgi.FieldStorage): if c.value: components.append(c.value) if hasattr(c, 'xml') and callable(c.xml): From 0b41ed36f92a1973dda66e7479e062cd9658678a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Sun, 6 Aug 2017 19:20:01 +0100 Subject: [PATCH 15/43] mobilize is back Fixes #1721 --- gluon/contrib/user_agent_parser.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/gluon/contrib/user_agent_parser.py b/gluon/contrib/user_agent_parser.py index bba1e436..61fee028 100644 --- a/gluon/contrib/user_agent_parser.py +++ b/gluon/contrib/user_agent_parser.py @@ -673,3 +673,20 @@ def simple_detect(agent): if os_version: os = " ".join((os, os_version)) return os, browser + + +class mobilize(object): + """ + Decorator for controller functions so they use different views for mobile devices. + """ + def __init__(self, func): + self.func = func + + def __call__(self): + from gluon import current + user_agent = current.request.user_agent() + if user_agent.is_mobile: + items = current.response.view.split('.') + items.insert(-1, 'mobile') + current.response.view = '.'.join(items) + return self.func() From 10b6b93cb2ac6118bad2207f87903722110a7745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Mon, 7 Aug 2017 00:17:43 +0100 Subject: [PATCH 16/43] minor py3 compatibility change --- gluon/restricted.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gluon/restricted.py b/gluon/restricted.py index 3fe5bfb0..5f3ffc10 100644 --- a/gluon/restricted.py +++ b/gluon/restricted.py @@ -10,7 +10,7 @@ Restricted environment to execute application's code """ import sys -from gluon._compat import pickle, ClassType +from gluon._compat import pickle, ClassType, unicodeT, to_bytes import traceback import types import os @@ -192,10 +192,10 @@ class RestrictedError(Exception): # safely show an useful message to the user try: output = self.output - if isinstance(output, unicode): - output = output.encode("utf8") - elif not isinstance(output, str): + if not isinstance(output, str, bytes, bytearray): output = str(output) + if isinstance(output, unicodeT): + output = to_bytes(output) except: output = "" return output From 3eea1f68f477f1d391edda93e71eedae82ad07bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Mon, 7 Aug 2017 00:20:29 +0100 Subject: [PATCH 17/43] Make sure you return bytes when you str(body) --- gluon/http.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/gluon/http.py b/gluon/http.py index 539a00fd..179dc4fe 100644 --- a/gluon/http.py +++ b/gluon/http.py @@ -11,7 +11,7 @@ HTTP statuses helpers """ import re -from gluon._compat import iteritems, unicodeT +from gluon._compat import iteritems, unicodeT, to_bytes __all__ = ['HTTP', 'redirect'] @@ -111,6 +111,8 @@ class HTTP(Exception): if not body: body = status if isinstance(body, (str, bytes, bytearray)): + if isinstance(body, unicodeT): + body = to_bytes(body) # This must be done before len headers['Content-Length'] = len(body) rheaders = [] for k, v in iteritems(headers): @@ -123,12 +125,15 @@ class HTTP(Exception): return [''] elif isinstance(body, (str, bytes, bytearray)): if isinstance(body, unicodeT): - body = body.encode('utf-8') + body = to_bytes(body) return [body] elif hasattr(body, '__iter__'): return body else: - return [str(body)] + body = str(body) + if isinstance(body, unicodeT): + body = to_bytes(body) + return [body] @property def message(self): From dc29f333656c0a1b7dcb17e792dc33fc0ae28629 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 7 Aug 2017 07:18:42 -0500 Subject: [PATCH 18/43] addressed __ssl.pyd issue #1716, thanks Jhleite --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 1f6e9099..6d72e470 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,8 @@ win: cp -r applications/welcome ../web2py_win/web2py/applications cp -r applications/examples ../web2py_win/web2py/applications cp applications/__init__.py ../web2py_win/web2py/applications + # per https://github.com/web2py/web2py/issues/1716 + mv ../web2py_win/_ssl.pyd ../web2py_win/_ssl.pyd.legacy cd ../web2py_win; zip -r web2py_win.zip web2py mv ../web2py_win/web2py_win.zip . run: From a29947f298e508a23425c9c151281a81a67904f5 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 7 Aug 2017 07:27:28 -0500 Subject: [PATCH 19/43] websocket_messaging in Python 3 #1696, thanks hirolau --- gluon/contrib/websocket_messaging.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/gluon/contrib/websocket_messaging.py b/gluon/contrib/websocket_messaging.py index ae8fbc4f..962e42a2 100644 --- a/gluon/contrib/websocket_messaging.py +++ b/gluon/contrib/websocket_messaging.py @@ -94,32 +94,10 @@ import optparse import time import sys import gluon.utils - -if (sys.version_info[0] == 2): - from urllib import urlencode, urlopen - def to_bytes(obj, charset='utf-8', errors='strict'): - if obj is None: - return None - if isinstance(obj, (bytes, bytearray, buffer)): - return bytes(obj) - if isinstance(obj, unicode): - return obj.encode(charset, errors) - raise TypeError('Expected bytes') -else: - from urllib.request import urlopen - from urllib.parse import urlencode - def to_bytes(obj, charset='utf-8', errors='strict'): - if obj is None: - return None - if isinstance(obj, (bytes, bytearray, memoryview)): - return bytes(obj) - if isinstance(obj, str): - return obj.encode(charset, errors) - raise TypeError('Expected bytes') +from gluon._compat import to_native, to_bytes, urlencode, urlopen listeners, names, tokens = {}, {}, {} - def websocket_send(url, message, hmac_key=None, group='default'): sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or '' params = urlencode( @@ -138,8 +116,8 @@ class PostHandler(tornado.web.RequestHandler): if hmac_key and not 'signature' in self.request.arguments: self.send_error(401) if 'message' in self.request.arguments: - message = self.request.arguments['message'][0] - group = self.request.arguments.get('group', ['default'])[0] + message = self.request.arguments['message'][0].decode(encoding='UTF-8') + group = self.request.arguments.get('group', ['default'])[0].decode(encoding='UTF-8') print('%s:MESSAGE to %s:%s' % (time.time(), group, message)) if hmac_key: signature = self.request.arguments['signature'][0] From 86ea728f4f51c1312216f552899b7e6bf75ed31d Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 7 Aug 2017 07:32:58 -0500 Subject: [PATCH 20/43] R-2.15.3 --- CHANGELOG | 2 +- Makefile | 2 +- VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3aa5d351..1df92c35 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -## 2.15.0b1 +## 2.15.1-3 - dropped support for python 2.6 - dropped web shell - experimental python 3 support diff --git a/Makefile b/Makefile index 6d72e470..543882fb 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ update: echo "remember that pymysql was tweaked" src: ### Use semantic versioning - echo 'Version 2.15.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION + echo 'Version 2.15.3-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION ### rm -f all junk files make clean ### clean up baisc apps diff --git a/VERSION b/VERSION index 09af535e..6b247f67 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.15.2-stable+timestamp.2017.07.19.01.21.31 +Version 2.15.3-stable+timestamp.2017.08.07.07.32.04 From 7e96ecafd720a2179da8ec3dbb3d14fb2b3f8da0 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 7 Aug 2017 07:41:11 -0500 Subject: [PATCH 21/43] R-2.15.3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 543882fb..3016e604 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ win: cp -r applications/examples ../web2py_win/web2py/applications cp applications/__init__.py ../web2py_win/web2py/applications # per https://github.com/web2py/web2py/issues/1716 - mv ../web2py_win/_ssl.pyd ../web2py_win/_ssl.pyd.legacy + mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy cd ../web2py_win; zip -r web2py_win.zip web2py mv ../web2py_win/web2py_win.zip . run: From 8533fa0d00106f621c865f9db04a6044c84bf484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Tue, 8 Aug 2017 00:50:55 +0100 Subject: [PATCH 22/43] put is_mobile and is_tablet in the result of user_agent() --- gluon/contrib/user_agent_parser.py | 3 +++ gluon/globals.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/gluon/contrib/user_agent_parser.py b/gluon/contrib/user_agent_parser.py index 61fee028..f966b616 100644 --- a/gluon/contrib/user_agent_parser.py +++ b/gluon/contrib/user_agent_parser.py @@ -678,6 +678,9 @@ def simple_detect(agent): class mobilize(object): """ Decorator for controller functions so they use different views for mobile devices. + + WARNING: If you update httpagentparser make sure to leave mobilize for + backwards compatibility. """ def __init__(self, func): self.func = func diff --git a/gluon/globals.py b/gluon/globals.py index 267ea661..e2ec33f1 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -331,11 +331,16 @@ class Request(Storage): user_agent = session._user_agent if user_agent: return user_agent - user_agent = user_agent_parser.detect(self.env.http_user_agent) + http_user_agent = self.env.http_user_agent + user_agent = user_agent_parser.detect(http_user_agent) for key, value in user_agent.items(): if isinstance(value, dict): user_agent[key] = Storage(value) - user_agent = session._user_agent = Storage(user_agent) + user_agent = Storage(user_agent) + user_agent.is_mobile = 'Mobile' in http_user_agent + user_agent.is_tablet = 'Tablet' in http_user_agent + session._user_agent = user_agent + return user_agent def requires_https(self): From 485f868cd10bec18e067110ce947dc0ad623d0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Mon, 7 Aug 2017 15:07:02 -0400 Subject: [PATCH 23/43] auth.has_membership(cached=True) check membership in auth.user_groups --- gluon/authapi.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/gluon/authapi.py b/gluon/authapi.py index d0922384..faef8c1e 100644 --- a/gluon/authapi.py +++ b/gluon/authapi.py @@ -561,23 +561,39 @@ class AuthAPI(object): del self.user_groups[group_id] return ret - def has_membership(self, group_id=None, user_id=None, role=None): + def has_membership(self, group_id=None, user_id=None, role=None, cached=False): """ Checks if user is member of group_id or role + + NOTE: To avoid database query at each page load that use auth.has_membership, someone can use cached=True. + If cached is set to True has_membership() check group_id or role only against auth.user_groups variable + which is populated properly only at login time. This means that if an user membership change during a + given session the user has to log off and log in again in order to auth.user_groups to be properly + recreated and reflecting the user membership modification. There is one exception to this log off and + log in process which is in case that the user change his own membership, in this case auth.user_groups + can be properly update for the actual connected user because web2py has access to the proper session + user_groups variable. To make use of this exception someone has to place an "auth.update_groups()" + instruction in his app code to force auth.user_groups to be updated. As mention this will only work if it + the user itself that change it membership not if another user, let say an administrator, change someone + else's membership. """ - group_id = group_id or self.id_group(role) - try: - group_id = int(group_id) - except: - group_id = self.id_group(group_id) # interpret group_id as a role if not user_id and self.user: user_id = self.user.id - membership = self.table_membership() - if group_id and user_id and self.db((membership.user_id == user_id) & - (membership.group_id == group_id)).select(): - r = True + if cached: + id_role = group_id or role + r = (user_id and id_role in self.user_groups.values()) or (user_id and id_role in self.user_groups) else: - r = False + group_id = group_id or self.id_group(role) + try: + group_id = int(group_id) + except: + group_id = self.id_group(group_id) # interpret group_id as a role + membership = self.table_membership() + if group_id and user_id and self.db((membership.user_id == user_id) & + (membership.group_id == group_id)).select(): + r = True + else: + r = False self.log_event(self.messages['has_membership_log'], dict(user_id=user_id, group_id=group_id, check=r)) return r From 6e5c8b62cc2cf3dab774a721863db1006fce3238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 9 Aug 2017 01:05:37 +0100 Subject: [PATCH 24/43] Make test_include_files demand the same order --- gluon/tests/test_globals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluon/tests/test_globals.py b/gluon/tests/test_globals.py index 390079ce..da403dbe 100644 --- a/gluon/tests/test_globals.py +++ b/gluon/tests/test_globals.py @@ -158,10 +158,10 @@ class testResponse(unittest.TestCase): response.files.append(URL('a', 'static', 'css/file.ts')) content = return_includes(response) self.assertEqual(content, + '' + '' + '' + - '' + - '' + '' ) response = Response() From 892fba9e54209228e06e7b4570ece1345b741d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 9 Aug 2017 01:08:28 +0100 Subject: [PATCH 25/43] fix unordered include_files result Now only concats adjacent internal to the application stuff, this also Fixes #1673 --- gluon/globals.py | 91 +++++++++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/gluon/globals.py b/gluon/globals.py index c47c596c..f9f5013b 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -473,45 +473,67 @@ class Response(Storage): response.cache_includes = (cache_method, time_expire). Example: (cache.disk, 60) # caches to disk for 1 minute. """ + app = current.request.application + + # We start by building a files list in which adjacent files internal to + # the application are placed in a list inside the files list. + # + # We will only minify and concat adjacent internal files as there's + # no way to know if changing the order with which the files are apppended + # will break things since the order matters in both CSS and JS and + # internal files may be interleaved with external ones. files = [] - ext_files = [] - has_js = has_css = False + # For the adjacent list we're going to use storage List to both distinguish + # from the regular list and so we can add attributes + internal = List() + internal.has_js = False + internal.has_css = False + done = set() # to remove duplicates for item in self.files: - if isinstance(item, (list, tuple)): - ext_files.append(item) + if not isinstance(item, list): + if item in done: + continue + done.add(item) + if isinstance(item, (list, tuple)) or not item.startswith('/' + app): # also consider items in other web2py applications to be external + if internal: + files.append(internal) + internal = List() + internal.has_js = False + internal.has_css = False + files.append(item) continue if extensions and not item.rpartition('.')[2] in extensions: continue - if item in files: - continue + internal.append(item) if item.endswith('.js'): - has_js = True + internal.has_js = True if item.endswith('.css'): - has_css = True - files.append(item) + internal.has_css = True + if internal: + files.append(internal) - if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)): - # cache for 5 minutes by default - key = hashlib_md5(repr(files)).hexdigest() - cache = self.cache_includes or (current.cache.ram, 60 * 5) - - def call_minify(files=files): - return minify.minify(files, - URL('static', 'temp'), - current.request.folder, - self.optimize_css, - self.optimize_js) - if cache: - cache_model, time_expire = cache - files = cache_model('response.files.minified/' + key, - call_minify, - time_expire) - else: - files = call_minify() - - files.extend(ext_files) - s = [] - for item in files: + # We're done we can now minify + if have_minify: + for i, f in enumerate(files): + if isinstance(f, List) and ((self.optimize_css and f.has_css) or (self.optimize_js and f.has_js)): + # cache for 5 minutes by default + key = hashlib_md5(repr(f)).hexdigest() + cache = self.cache_includes or (current.cache.ram, 60 * 5) + def call_minify(files=f): + return List(minify.minify(files, + URL('static', 'temp'), + current.request.folder, + self.optimize_css, + self.optimize_js)) + if cache: + cache_model, time_expire = cache + files[i] = cache_model('response.files.minified/' + key, + call_minify, + time_expire) + else: + files[i] = call_minify() + + def static_map(s, item): if isinstance(item, str): f = item.lower().split('?')[0] ext = f.rpartition('.')[2] @@ -531,6 +553,13 @@ class Response(Storage): if tmpl: s.append(tmpl % item[1]) + s = [] + for item in files: + if isinstance(item, List): + for f in item: + static_map(s, f) + else: + static_map(s, item) self.write(''.join(s), escape=False) def stream(self, From 135f41041dfe69b35f3fbaadf152918397a5b6ed Mon Sep 17 00:00:00 2001 From: LAdm Date: Thu, 10 Aug 2017 06:49:36 +0200 Subject: [PATCH 26/43] fixes #1724 call func instead of module in url_is_acceptable --- gluon/sanitizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/sanitizer.py b/gluon/sanitizer.py index 5cd2ea4a..b98b7477 100644 --- a/gluon/sanitizer.py +++ b/gluon/sanitizer.py @@ -145,7 +145,7 @@ class XssCleaner(HTMLParser): if url.startswith('#'): return True else: - parsed = urlparse(url) + parsed = urlparse.urlparse(url) return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or (parsed[0] == '' and parsed[2].startswith('/'))) From bd7ee209ea38b5e5a2b3826c8810238a4d8d7d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Tue, 15 Aug 2017 15:34:35 +0100 Subject: [PATCH 27/43] Fixes #1737 getproxies is in urllib.request for python 3 --- gluon/widget.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gluon/widget.py b/gluon/widget.py index d56be20a..20b602bb 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -1076,7 +1076,10 @@ def start_schedulers(options): return # Work around OS X problem: http://bugs.python.org/issue9405 - import urllib + if PY2: + import urllib + else: + import urllib.request as urllib urllib.getproxies() for app in apps: From f22e3a762493e07dbd9ccb7557e1791cdef85731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 16 Aug 2017 16:35:46 +0100 Subject: [PATCH 28/43] execfile shim for python 3 compatibility Fixes #1739 --- gluon/shell.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gluon/shell.py b/gluon/shell.py index b6c8f27b..38d12ec4 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -31,10 +31,16 @@ from gluon.globals import Request, Response, Session from gluon.storage import Storage, List from gluon.admin import w2p_unpack from pydal.base import BaseAdapter -from gluon._compat import iteritems, ClassType +from gluon._compat import iteritems, ClassType, PY2 logger = logging.getLogger("web2py") +if not PY2: + def execfile(filename, global_vars=None, local_vars=None): + with open(filename) as f: + code = compile(f.read(), filename, 'exec') + exec(code, global_vars, local_vars) + def enable_autocomplete_and_history(adir, env): try: From 9694c66703f4c08eb16029dc93704c8a6d713f8b Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 19 Aug 2017 08:55:53 +0200 Subject: [PATCH 29/43] Fix view file name in ticket traceback, fix #1740, fix #1676 --- gluon/compileapp.py | 9 ++++++--- gluon/restricted.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 116a23f5..ac9cc491 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -676,6 +676,7 @@ def run_view_in(environment): badv = 'invalid view (%s)' % view patterns = response.get('generic_patterns') layer = None + scode = None if patterns: regex = re_compile('|'.join(map(fnmatch.translate, patterns))) short_action = '%(controller)s/%(function)s.%(extension)s' % request @@ -718,12 +719,14 @@ def run_view_in(environment): # if the view is not compiled if not layer: - # Compile the template - ccode = parse_template(view, + # Parse template + scode = parse_template(view, pjoin(folder, 'views'), context=environment) + # Compile template + ccode = compile2(scode, filename) layer = filename - restricted(ccode, environment, layer=layer) + restricted(ccode, environment, layer=layer, scode=scode) # parse_template saves everything in response body return environment['response'].body.getvalue() diff --git a/gluon/restricted.py b/gluon/restricted.py index 5f3ffc10..115470e3 100644 --- a/gluon/restricted.py +++ b/gluon/restricted.py @@ -205,7 +205,7 @@ def compile2(code, layer): return compile(code, layer, 'exec') -def restricted(ccode, environment=None, layer='Unknown'): +def restricted(ccode, environment=None, layer='Unknown', scode=None): """ Runs code in environment and returns the output. If an exception occurs in code it raises a RestrictedError containing the traceback. Layer is @@ -230,7 +230,9 @@ def restricted(ccode, environment=None, layer='Unknown'): sys.excepthook(etype, evalue, tb) del tb output = "%s %s" % (etype, evalue) - raise RestrictedError(layer, ccode, output, environment) + # Save source code in ticket when available + scode = scode if scode else ccode + raise RestrictedError(layer, scode, output, environment) def snapshot(info=None, context=5, code=None, environment=None): From e880da0d0e85783eb373f2a6d703311a681c5609 Mon Sep 17 00:00:00 2001 From: abastardi Date: Wed, 23 Aug 2017 11:43:40 -0400 Subject: [PATCH 30/43] Fix submit button disable bug When a form includes more than one submit button, after being disabled all buttons are re-enabled with the value of the first button rather than having their original values restored. This change iterates over each button separately to disable and enable. --- applications/welcome/static/js/web2py.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/applications/welcome/static/js/web2py.js b/applications/welcome/static/js/web2py.js index 1b52245e..f59263f1 100644 --- a/applications/welcome/static/js/web2py.js +++ b/applications/welcome/static/js/web2py.js @@ -265,13 +265,17 @@ } }); /* help preventing double form submission for normal form (not LOADed) */ - $(doc).on('submit', 'form', function () { - var submit_button = $(this).find(web2py.formInputClickSelector); - web2py.disableElement(submit_button); + $(doc).on('submit', 'form', function (e) { + var submit_buttons = $(this).find(web2py.formInputClickSelector); + submit_buttons.each(function() { + web2py.disableElement($(this)); + }) /* safeguard in case the form doesn't trigger a refresh, see https://github.com/web2py/web2py/issues/1100 */ setTimeout(function () { - web2py.enableElement(submit_button); + submit_buttons.each(function() { + web2py.enableElement($(this)); + }); }, 5000); }); doc.ajaxSuccess(function (e, xhr) { From c9a71a7055a123bec0591dcf8a8c4bf30f3dadaa Mon Sep 17 00:00:00 2001 From: Yi Liu Date: Tue, 29 Aug 2017 11:26:22 -0700 Subject: [PATCH 31/43] MInor error in EOF instructions start -> stop --- scripts/setup-web2py-nginx-uwsgi-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh b/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh index 9d7fcee8..e34959d3 100644 --- a/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh +++ b/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh @@ -222,7 +222,7 @@ echo < Date: Thu, 31 Aug 2017 17:31:43 -0500 Subject: [PATCH 32/43] following pydal 17.08 --- gluon/packages/dal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index c707d558..f9f0fdfc 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit c707d558995d0318022f3a26935f1100020975fc +Subproject commit f9f0fdfc9a9bedb40a191e5b85b44cb08c672f17 From d06a1f9dc6cff17f92ba4abd4a58f1f02e9a6faa Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 1 Sep 2017 22:40:20 -0500 Subject: [PATCH 33/43] R-2.15.4 --- CHANGELOG | 3 ++- Makefile | 2 +- VERSION | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 1df92c35..ada9a5c6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,5 @@ -## 2.15.1-3 +## 2.15.1-4 +- pydal 17.08 - dropped support for python 2.6 - dropped web shell - experimental python 3 support diff --git a/Makefile b/Makefile index 3016e604..852dd016 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ update: echo "remember that pymysql was tweaked" src: ### Use semantic versioning - echo 'Version 2.15.3-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION + echo 'Version 2.15.4-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION ### rm -f all junk files make clean ### clean up baisc apps diff --git a/VERSION b/VERSION index 6b247f67..fb2019a7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.15.3-stable+timestamp.2017.08.07.07.32.04 +Version 2.15.4-stable+timestamp.2017.09.01.22.38.25 From 087280ec1741cbdbb659458333554d94d54f8c58 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 1 Sep 2017 22:47:54 -0500 Subject: [PATCH 34/43] fixed Makefile _ssd --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 852dd016..78f911a8 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ win: cp -r applications/examples ../web2py_win/web2py/applications cp applications/__init__.py ../web2py_win/web2py/applications # per https://github.com/web2py/web2py/issues/1716 - mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy + mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy | echo 'done' cd ../web2py_win; zip -r web2py_win.zip web2py mv ../web2py_win/web2py_win.zip . run: From 2861dc4215ae97f65b6f7bc78177aba5f5c35cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Tue, 5 Sep 2017 15:46:31 +0100 Subject: [PATCH 35/43] Fixes #1753 --- gluon/compileapp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index ac9cc491..dc0dae79 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -205,7 +205,7 @@ def LOAD(c=None, f='index', args=None, vars=None, other_response = Response() other_request.env.path_info = '/' + \ '/'.join([request.application, c, f] + - map(str, other_request.args)) + [str(a) for a in other_request.args]) other_request.env.query_string = \ vars and URL(vars=vars).split('?')[1] or '' other_request.env.http_web2py_component_location = \ @@ -288,7 +288,7 @@ class LoadFactory(object): other_response = globals.Response() other_request.env.path_info = '/' + \ '/'.join([request.application, c, f] + - map(str, other_request.args)) + [str(a) for a in other_request.args]) other_request.env.query_string = \ vars and html.URL(vars=vars).split('?')[1] or '' other_request.env.http_web2py_component_location = \ @@ -678,7 +678,7 @@ def run_view_in(environment): layer = None scode = None if patterns: - regex = re_compile('|'.join(map(fnmatch.translate, patterns))) + regex = re_compile('|'.join(fnmatch.translate(p) for p in patterns)) short_action = '%(controller)s/%(function)s.%(extension)s' % request allow_generic = regex.search(short_action) else: From 4aefb93ab44e0555d134603da88137cd73e534b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 6 Sep 2017 15:28:39 +0100 Subject: [PATCH 36/43] Fixes #1752 --- gluon/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/_compat.py b/gluon/_compat.py index f7562b12..1a68163d 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -151,7 +151,7 @@ def with_metaclass(meta, *bases): def to_unicode(obj, charset='utf-8', errors='strict'): if obj is None: return None - if not isinstance(obj, bytes): + if not hasattr(obj, 'decode') and not isinstance(obj, bytes): return text_type(obj) return obj.decode(charset, errors) From a6044068cd043a89db6b643a221dca7b1344d840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 6 Sep 2017 15:31:30 +0100 Subject: [PATCH 37/43] Fixes #1751 --- gluon/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/_compat.py b/gluon/_compat.py index 1a68163d..5b7ec948 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -63,7 +63,7 @@ if PY2: return None if isinstance(obj, (bytes, bytearray, buffer)): return bytes(obj) - if isinstance(obj, unicode): + if hasattr(obj, 'encode'): return obj.encode(charset, errors) raise TypeError('Expected bytes') From 83f90165284a25a2852db0c50e7fc81380be0087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 6 Sep 2017 15:33:47 +0100 Subject: [PATCH 38/43] fix in py3 too --- gluon/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/_compat.py b/gluon/_compat.py index 5b7ec948..e5fcbdd8 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -122,7 +122,7 @@ else: return None if isinstance(obj, (bytes, bytearray, memoryview)): return bytes(obj) - if isinstance(obj, str): + if hasattr(obj, 'encode'): return obj.encode(charset, errors) raise TypeError('Expected bytes') From 4b38186b518244ebbd91bf53073d273f32d70ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 6 Sep 2017 15:37:51 +0100 Subject: [PATCH 39/43] simplify condition if you ain't got decode you ain't bytes --- gluon/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/_compat.py b/gluon/_compat.py index e5fcbdd8..a61ddc51 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -151,7 +151,7 @@ def with_metaclass(meta, *bases): def to_unicode(obj, charset='utf-8', errors='strict'): if obj is None: return None - if not hasattr(obj, 'decode') and not isinstance(obj, bytes): + if not hasattr(obj, 'decode'): return text_type(obj) return obj.decode(charset, errors) From 912c22d593375d491e83dcb65ee12a2bec03f02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Wed, 6 Sep 2017 16:22:50 +0100 Subject: [PATCH 40/43] Don't use gluon.utf8.Utf8 with py3 there's no need for it and it breaks stuff --- gluon/languages.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gluon/languages.py b/gluon/languages.py index b626457c..aff3db14 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -18,10 +18,11 @@ import pkgutil import logging from cgi import escape from threading import RLock -from gluon.utf8 import Utf8 + from gluon.utils import local_html_escape from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin + from pydal.contrib.portalocker import read_locked, LockedFile from gluon.fileutils import listdir @@ -49,8 +50,10 @@ DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word if PY2: NUMBERS = (int, long, float) + from gluon.utf8 import Utf8 else: NUMBERS = (int, float) + Utf8 = str # pattern to find T(blah blah blah) expressions PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P'\ From 3ecdd1c11b05b805bc1a5d82bb3e28344a9b8f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Thu, 7 Sep 2017 10:18:41 +0100 Subject: [PATCH 41/43] Fix #1757 --- gluon/globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/globals.py b/gluon/globals.py index f9f5013b..cb52d331 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -331,7 +331,7 @@ class Request(Storage): user_agent = session._user_agent if user_agent: return user_agent - http_user_agent = self.env.http_user_agent + http_user_agent = self.env.http_user_agent or '' user_agent = user_agent_parser.detect(http_user_agent) for key, value in user_agent.items(): if isinstance(value, dict): From 8eda21ca8685ab5dcde4efcec85009806f3be327 Mon Sep 17 00:00:00 2001 From: abastardi Date: Wed, 13 Sep 2017 11:48:25 -0400 Subject: [PATCH 42/43] Fix bug with compiled views in compiled-only apps Compiled views were not being executed in apps containing only compiled files (i.e., generated via the admin "Pack compiled" functionality), resulting in "Invalid view" errors for all pages. --- gluon/compileapp.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index ac9cc491..1ec547d5 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -709,23 +709,22 @@ def run_view_in(environment): ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled)) layer = compiled break - if not os.path.exists(filename) and allow_generic: - view = 'generic.' + request.extension - filename = pjoin(folder, 'views', view) - if not os.path.exists(filename): - raise HTTP(404, - rewrite.THREAD_LOCAL.routes.error_message % badv, - web2py_error=badv) - # if the view is not compiled if not layer: + if not os.path.exists(filename) and allow_generic: + view = 'generic.' + request.extension + filename = pjoin(folder, 'views', view) + if not os.path.exists(filename): + raise HTTP(404, + rewrite.THREAD_LOCAL.routes.error_message % badv, + web2py_error=badv) # Parse template scode = parse_template(view, pjoin(folder, 'views'), context=environment) # Compile template ccode = compile2(scode, filename) - layer = filename + layer = filename restricted(ccode, environment, layer=layer, scode=scode) # parse_template saves everything in response body return environment['response'].body.getvalue() From a605e43fa6d2e0d15626429411cf074c0609c982 Mon Sep 17 00:00:00 2001 From: abastardi Date: Thu, 14 Sep 2017 18:49:31 -0400 Subject: [PATCH 43/43] Fix grid TSV export bug --- gluon/sqlhtml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 0653c4c7..555123e9 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -3570,7 +3570,9 @@ class ExportClass(object): if not self.rows.db._adapter.REGEX_TABLE_DOT_FIELD.match(col): row.append(record._extra[col]) else: - (t, f) = col.split('.') + # The grid code modifies rows.colnames, adding double quotes + # around the table and field names -- so they must be removed here. + (t, f) = [name.strip('"') for name in col.split('.')] field = self.rows.db[t][f] if isinstance(record.get(t, None), (Row, dict)): value = record[t][f]