diff --git a/CHANGELOG b/CHANGELOG index 3aa5d351..ada9a5c6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,5 @@ -## 2.15.0b1 +## 2.15.1-4 +- pydal 17.08 - dropped support for python 2.6 - dropped web shell - experimental python 3 support diff --git a/Makefile b/Makefile index 1f6e9099..78f911a8 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ update: echo "remember that pymysql was tweaked" src: ### Use semantic versioning - echo 'Version 2.15.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION + echo 'Version 2.15.4-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION ### rm -f all junk files make clean ### clean up baisc apps @@ -97,6 +97,8 @@ win: cp -r applications/welcome ../web2py_win/web2py/applications cp -r applications/examples ../web2py_win/web2py/applications cp applications/__init__.py ../web2py_win/web2py/applications + # per https://github.com/web2py/web2py/issues/1716 + mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy | echo 'done' cd ../web2py_win; zip -r web2py_win.zip web2py mv ../web2py_win/web2py_win.zip . run: diff --git a/VERSION b/VERSION index 09af535e..fb2019a7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.15.2-stable+timestamp.2017.07.19.01.21.31 +Version 2.15.4-stable+timestamp.2017.09.01.22.38.25 diff --git a/applications/admin/controllers/appadmin.py b/applications/admin/controllers/appadmin.py index 139ad1aa..74240e59 100644 --- a/applications/admin/controllers/appadmin.py +++ b/applications/admin/controllers/appadmin.py @@ -657,37 +657,36 @@ def d3_graph_model(): Create a list of table dicts, called "nodes" """ - data = {} nodes = [] links = [] - subgraphs = dict() + for database in databases: + db = eval_in_global_env(database) + for tablename in db.tables: + fields = [] + for field in db[tablename]: + f_type = field.type + if not isinstance(f_type,str): + disp = ' ' + elif f_type == 'string': + disp = field.length + elif f_type == 'id': + disp = "PK" + elif f_type.startswith('reference') or \ + f_type.startswith('list:reference'): + disp = "FK" + else: + disp = ' ' + fields.append(dict(name= field.name, type=field.type, disp = disp)) - for tablename in db.tables: - fields = [] - for field in db[tablename]: - f_type = field.type - if not isinstance(f_type,str): - disp = ' ' - elif f_type == 'string': - disp = field.length - elif f_type == 'id': - disp = "PK" - elif f_type.startswith('reference') or \ - f_type.startswith('list:reference'): - disp = "FK" - else: - disp = ' ' - fields.append(dict(name= field.name, type=field.type, disp = disp)) + if isinstance(f_type,str) and ( + f_type.startswith('reference') or + f_type.startswith('list:reference')): + referenced_table = f_type.split()[1].split('.')[0] - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] + links.append(dict(source=tablename, target = referenced_table)) - links.append(dict(source=tablename, target = referenced_table)) - - nodes.append(dict(name=tablename, type="table", fields = fields)) + nodes.append(dict(name=tablename, type="table", fields = fields)) # d3 v4 allows individual modules to be specified. The complete d3 library is included below. response.files.append(URL('admin','static','js/d3.min.js')) diff --git a/applications/examples/controllers/appadmin.py b/applications/examples/controllers/appadmin.py index 139ad1aa..74240e59 100644 --- a/applications/examples/controllers/appadmin.py +++ b/applications/examples/controllers/appadmin.py @@ -657,37 +657,36 @@ def d3_graph_model(): Create a list of table dicts, called "nodes" """ - data = {} nodes = [] links = [] - subgraphs = dict() + for database in databases: + db = eval_in_global_env(database) + for tablename in db.tables: + fields = [] + for field in db[tablename]: + f_type = field.type + if not isinstance(f_type,str): + disp = ' ' + elif f_type == 'string': + disp = field.length + elif f_type == 'id': + disp = "PK" + elif f_type.startswith('reference') or \ + f_type.startswith('list:reference'): + disp = "FK" + else: + disp = ' ' + fields.append(dict(name= field.name, type=field.type, disp = disp)) - for tablename in db.tables: - fields = [] - for field in db[tablename]: - f_type = field.type - if not isinstance(f_type,str): - disp = ' ' - elif f_type == 'string': - disp = field.length - elif f_type == 'id': - disp = "PK" - elif f_type.startswith('reference') or \ - f_type.startswith('list:reference'): - disp = "FK" - else: - disp = ' ' - fields.append(dict(name= field.name, type=field.type, disp = disp)) + if isinstance(f_type,str) and ( + f_type.startswith('reference') or + f_type.startswith('list:reference')): + referenced_table = f_type.split()[1].split('.')[0] - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] + links.append(dict(source=tablename, target = referenced_table)) - links.append(dict(source=tablename, target = referenced_table)) - - nodes.append(dict(name=tablename, type="table", fields = fields)) + nodes.append(dict(name=tablename, type="table", fields = fields)) # d3 v4 allows individual modules to be specified. The complete d3 library is included below. response.files.append(URL('admin','static','js/d3.min.js')) diff --git a/applications/welcome/controllers/appadmin.py b/applications/welcome/controllers/appadmin.py index 139ad1aa..74240e59 100644 --- a/applications/welcome/controllers/appadmin.py +++ b/applications/welcome/controllers/appadmin.py @@ -657,37 +657,36 @@ def d3_graph_model(): Create a list of table dicts, called "nodes" """ - data = {} nodes = [] links = [] - subgraphs = dict() + for database in databases: + db = eval_in_global_env(database) + for tablename in db.tables: + fields = [] + for field in db[tablename]: + f_type = field.type + if not isinstance(f_type,str): + disp = ' ' + elif f_type == 'string': + disp = field.length + elif f_type == 'id': + disp = "PK" + elif f_type.startswith('reference') or \ + f_type.startswith('list:reference'): + disp = "FK" + else: + disp = ' ' + fields.append(dict(name= field.name, type=field.type, disp = disp)) - for tablename in db.tables: - fields = [] - for field in db[tablename]: - f_type = field.type - if not isinstance(f_type,str): - disp = ' ' - elif f_type == 'string': - disp = field.length - elif f_type == 'id': - disp = "PK" - elif f_type.startswith('reference') or \ - f_type.startswith('list:reference'): - disp = "FK" - else: - disp = ' ' - fields.append(dict(name= field.name, type=field.type, disp = disp)) + if isinstance(f_type,str) and ( + f_type.startswith('reference') or + f_type.startswith('list:reference')): + referenced_table = f_type.split()[1].split('.')[0] - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] + links.append(dict(source=tablename, target = referenced_table)) - links.append(dict(source=tablename, target = referenced_table)) - - nodes.append(dict(name=tablename, type="table", fields = fields)) + nodes.append(dict(name=tablename, type="table", fields = fields)) # d3 v4 allows individual modules to be specified. The complete d3 library is included below. response.files.append(URL('admin','static','js/d3.min.js')) diff --git a/applications/welcome/static/js/web2py.js b/applications/welcome/static/js/web2py.js index 1b52245e..f59263f1 100644 --- a/applications/welcome/static/js/web2py.js +++ b/applications/welcome/static/js/web2py.js @@ -265,13 +265,17 @@ } }); /* help preventing double form submission for normal form (not LOADed) */ - $(doc).on('submit', 'form', function () { - var submit_button = $(this).find(web2py.formInputClickSelector); - web2py.disableElement(submit_button); + $(doc).on('submit', 'form', function (e) { + var submit_buttons = $(this).find(web2py.formInputClickSelector); + submit_buttons.each(function() { + web2py.disableElement($(this)); + }) /* safeguard in case the form doesn't trigger a refresh, see https://github.com/web2py/web2py/issues/1100 */ setTimeout(function () { - web2py.enableElement(submit_button); + submit_buttons.each(function() { + web2py.enableElement($(this)); + }); }, 5000); }); doc.ajaxSuccess(function (e, xhr) { diff --git a/gluon/_compat.py b/gluon/_compat.py index f7562b12..a61ddc51 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -63,7 +63,7 @@ if PY2: return None if isinstance(obj, (bytes, bytearray, buffer)): return bytes(obj) - if isinstance(obj, unicode): + if hasattr(obj, 'encode'): return obj.encode(charset, errors) raise TypeError('Expected bytes') @@ -122,7 +122,7 @@ else: return None if isinstance(obj, (bytes, bytearray, memoryview)): return bytes(obj) - if isinstance(obj, str): + if hasattr(obj, 'encode'): return obj.encode(charset, errors) raise TypeError('Expected bytes') @@ -151,7 +151,7 @@ def with_metaclass(meta, *bases): def to_unicode(obj, charset='utf-8', errors='strict'): if obj is None: return None - if not isinstance(obj, bytes): + if not hasattr(obj, 'decode'): return text_type(obj) return obj.decode(charset, errors) diff --git a/gluon/authapi.py b/gluon/authapi.py index 3347cb68..faef8c1e 100644 --- a/gluon/authapi.py +++ b/gluon/authapi.py @@ -20,10 +20,10 @@ DEFAULT = lambda: None class AuthAPI(object): """ AuthAPI is a barebones Auth implementation which does not have a concept of - HTML forms or redirects, emailing or even an URL, you are responsible for + HTML forms or redirects, emailing or even an URL, you are responsible for all that if you use it. The main Auth functions such as login, logout, register, profile are designed - in a Dict In -> Dict Out logic so, for instance, if you set + in a Dict In -> Dict Out logic so, for instance, if you set registration_requires_verification you are responsible for sending the key to the user and even rolling back the transaction if you can't do it. @@ -245,13 +245,13 @@ class AuthAPI(object): migrate = db._migrate if fake_migrate is None: fake_migrate = db._fake_migrate - + settings = self.settings if username is None: username = settings.use_username else: settings.use_username = username - + if not self.signature: self.define_signature() if signature is True: @@ -557,27 +557,43 @@ class AuthAPI(object): self.log_event(self.messages['del_membership_log'], dict(user_id=user_id, group_id=group_id)) ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete() - if group_id in self.user_groups: + if group_id in self.user_groups and user_id == self.user_id: del self.user_groups[group_id] return ret - def has_membership(self, group_id=None, user_id=None, role=None): + def has_membership(self, group_id=None, user_id=None, role=None, cached=False): """ Checks if user is member of group_id or role + + NOTE: To avoid database query at each page load that use auth.has_membership, someone can use cached=True. + If cached is set to True has_membership() check group_id or role only against auth.user_groups variable + which is populated properly only at login time. This means that if an user membership change during a + given session the user has to log off and log in again in order to auth.user_groups to be properly + recreated and reflecting the user membership modification. There is one exception to this log off and + log in process which is in case that the user change his own membership, in this case auth.user_groups + can be properly update for the actual connected user because web2py has access to the proper session + user_groups variable. To make use of this exception someone has to place an "auth.update_groups()" + instruction in his app code to force auth.user_groups to be updated. As mention this will only work if it + the user itself that change it membership not if another user, let say an administrator, change someone + else's membership. """ - group_id = group_id or self.id_group(role) - try: - group_id = int(group_id) - except: - group_id = self.id_group(group_id) # interpret group_id as a role if not user_id and self.user: user_id = self.user.id - membership = self.table_membership() - if group_id and user_id and self.db((membership.user_id == user_id) & - (membership.group_id == group_id)).select(): - r = True + if cached: + id_role = group_id or role + r = (user_id and id_role in self.user_groups.values()) or (user_id and id_role in self.user_groups) else: - r = False + group_id = group_id or self.id_group(role) + try: + group_id = int(group_id) + except: + group_id = self.id_group(group_id) # interpret group_id as a role + membership = self.table_membership() + if group_id and user_id and self.db((membership.user_id == user_id) & + (membership.group_id == group_id)).select(): + r = True + else: + r = False self.log_event(self.messages['has_membership_log'], dict(user_id=user_id, group_id=group_id, check=r)) return r @@ -1012,7 +1028,7 @@ class AuthAPI(object): ): """ Verify a given registration_key actually exists in the user table. - Resets the key to empty string '' or 'pending' if + Resets the key to empty string '' or 'pending' if setttings.registration_requires_approval is true. Keyword Args: diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 116a23f5..a59e96d1 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -205,7 +205,7 @@ def LOAD(c=None, f='index', args=None, vars=None, other_response = Response() other_request.env.path_info = '/' + \ '/'.join([request.application, c, f] + - map(str, other_request.args)) + [str(a) for a in other_request.args]) other_request.env.query_string = \ vars and URL(vars=vars).split('?')[1] or '' other_request.env.http_web2py_component_location = \ @@ -288,7 +288,7 @@ class LoadFactory(object): other_response = globals.Response() other_request.env.path_info = '/' + \ '/'.join([request.application, c, f] + - map(str, other_request.args)) + [str(a) for a in other_request.args]) other_request.env.query_string = \ vars and html.URL(vars=vars).split('?')[1] or '' other_request.env.http_web2py_component_location = \ @@ -676,8 +676,9 @@ def run_view_in(environment): badv = 'invalid view (%s)' % view patterns = response.get('generic_patterns') layer = None + scode = None if patterns: - regex = re_compile('|'.join(map(fnmatch.translate, patterns))) + regex = re_compile('|'.join(fnmatch.translate(p) for p in patterns)) short_action = '%(controller)s/%(function)s.%(extension)s' % request allow_generic = regex.search(short_action) else: @@ -708,22 +709,23 @@ def run_view_in(environment): ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled)) layer = compiled break - if not os.path.exists(filename) and allow_generic: - view = 'generic.' + request.extension - filename = pjoin(folder, 'views', view) - if not os.path.exists(filename): - raise HTTP(404, - rewrite.THREAD_LOCAL.routes.error_message % badv, - web2py_error=badv) - # if the view is not compiled if not layer: - # Compile the template - ccode = parse_template(view, + if not os.path.exists(filename) and allow_generic: + view = 'generic.' + request.extension + filename = pjoin(folder, 'views', view) + if not os.path.exists(filename): + raise HTTP(404, + rewrite.THREAD_LOCAL.routes.error_message % badv, + web2py_error=badv) + # Parse template + scode = parse_template(view, pjoin(folder, 'views'), context=environment) - layer = filename - restricted(ccode, environment, layer=layer) + # Compile template + ccode = compile2(scode, filename) + layer = filename + restricted(ccode, environment, layer=layer, scode=scode) # parse_template saves everything in response body return environment['response'].body.getvalue() diff --git a/gluon/contrib/login_methods/cas_auth.py b/gluon/contrib/login_methods/cas_auth.py index 1a07c5f1..73be8ae1 100644 --- a/gluon/contrib/login_methods/cas_auth.py +++ b/gluon/contrib/login_methods/cas_auth.py @@ -49,7 +49,8 @@ class CasAuth(object): email=lambda v: v.get('email', None), user_id=lambda v: v['user']), casversion=1, - casusername='cas:user' + casusername='cas:user', + change_password_url=None ): self.urlbase = urlbase self.cas_login_url = "%s/%s" % (self.urlbase, actions[0]) @@ -64,6 +65,9 @@ class CasAuth(object): #vars=current.request.vars, scheme=True) + # URL to let users change their password in the IDP system + self.cas_change_password_url = change_password_url + def login_url(self, next="/"): current.session.token = self._CAS_login() return next @@ -74,6 +78,10 @@ class CasAuth(object): self._CAS_logout() return next + def change_password_url(self, next="/"): + self._CAS_change_password() + return next + def get_user(self): user = current.session.token if user: @@ -135,3 +143,6 @@ class CasAuth(object): redirects to the CAS logout page """ redirect("%s?service=%s" % (self.cas_logout_url, self.cas_my_url)) + + def _CAS_change_password(self): + redirect(self.cas_change_password_url) diff --git a/gluon/contrib/login_methods/saml2_auth.py b/gluon/contrib/login_methods/saml2_auth.py index 42fe24db..462d9845 100644 --- a/gluon/contrib/login_methods/saml2_auth.py +++ b/gluon/contrib/login_methods/saml2_auth.py @@ -145,10 +145,16 @@ class Saml2Auth(object): username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0], - )): + ), logout_url=None, change_password_url=None): self.config_file = config_file self.maps = maps + # URL for redirecting users to when they sign out + self.saml_logout_url = logout_url + + # URL to let users change their password in the IDP system + self.saml_change_password_url = change_password_url + def login_url(self, next="/"): d = saml2_handler(current.session, current.request) if 'url' in d: @@ -170,6 +176,12 @@ class Saml2Auth(object): def logout_url(self, next="/"): current.session.saml2_info = None + current.session.auth = None + self._SAML_logout() + return next + + def change_password_url(self, next="/"): + self._SAML_change_password() return next def get_user(self): @@ -180,3 +192,13 @@ class Saml2Auth(object): d[key] = self.maps[key](user) return d return None + + def _SAML_logout(self): + """ + exposed SAML.logout() + redirects to the SAML logout page + """ + redirect(self.saml_logout_url) + + def _SAML_change_password(self): + redirect(self.saml_change_password_url) diff --git a/gluon/contrib/user_agent_parser.py b/gluon/contrib/user_agent_parser.py index bba1e436..f966b616 100644 --- a/gluon/contrib/user_agent_parser.py +++ b/gluon/contrib/user_agent_parser.py @@ -673,3 +673,23 @@ def simple_detect(agent): if os_version: os = " ".join((os, os_version)) return os, browser + + +class mobilize(object): + """ + Decorator for controller functions so they use different views for mobile devices. + + WARNING: If you update httpagentparser make sure to leave mobilize for + backwards compatibility. + """ + def __init__(self, func): + self.func = func + + def __call__(self): + from gluon import current + user_agent = current.request.user_agent() + if user_agent.is_mobile: + items = current.response.view.split('.') + items.insert(-1, 'mobile') + current.response.view = '.'.join(items) + return self.func() diff --git a/gluon/contrib/websocket_messaging.py b/gluon/contrib/websocket_messaging.py index ae8fbc4f..962e42a2 100644 --- a/gluon/contrib/websocket_messaging.py +++ b/gluon/contrib/websocket_messaging.py @@ -94,32 +94,10 @@ import optparse import time import sys import gluon.utils - -if (sys.version_info[0] == 2): - from urllib import urlencode, urlopen - def to_bytes(obj, charset='utf-8', errors='strict'): - if obj is None: - return None - if isinstance(obj, (bytes, bytearray, buffer)): - return bytes(obj) - if isinstance(obj, unicode): - return obj.encode(charset, errors) - raise TypeError('Expected bytes') -else: - from urllib.request import urlopen - from urllib.parse import urlencode - def to_bytes(obj, charset='utf-8', errors='strict'): - if obj is None: - return None - if isinstance(obj, (bytes, bytearray, memoryview)): - return bytes(obj) - if isinstance(obj, str): - return obj.encode(charset, errors) - raise TypeError('Expected bytes') +from gluon._compat import to_native, to_bytes, urlencode, urlopen listeners, names, tokens = {}, {}, {} - def websocket_send(url, message, hmac_key=None, group='default'): sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or '' params = urlencode( @@ -138,8 +116,8 @@ class PostHandler(tornado.web.RequestHandler): if hmac_key and not 'signature' in self.request.arguments: self.send_error(401) if 'message' in self.request.arguments: - message = self.request.arguments['message'][0] - group = self.request.arguments.get('group', ['default'])[0] + message = self.request.arguments['message'][0].decode(encoding='UTF-8') + group = self.request.arguments.get('group', ['default'])[0].decode(encoding='UTF-8') print('%s:MESSAGE to %s:%s' % (time.time(), group, message)) if hmac_key: signature = self.request.arguments['signature'][0] diff --git a/gluon/dal.py b/gluon/dal.py index ceecf2e0..215fd4e1 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -14,6 +14,7 @@ from pydal import DAL as DAL from pydal import Field from pydal.objects import Row, Rows, Table, Query, Set, Expression from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon +from pydal.migrator import Migrator, InDBMigrator from gluon.serializers import custom_json, xml from gluon.utils import web2py_uuid from gluon import sqlhtml diff --git a/gluon/globals.py b/gluon/globals.py index 267ea661..cb52d331 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -14,7 +14,7 @@ Contains the classes for the global used variables: """ from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \ - unicodeT, long, hashlib_md5 + unicodeT, long, hashlib_md5, urllib_quote from gluon.storage import Storage, List from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE from gluon.contenttype import contenttype @@ -331,11 +331,16 @@ class Request(Storage): user_agent = session._user_agent if user_agent: return user_agent - user_agent = user_agent_parser.detect(self.env.http_user_agent) + http_user_agent = self.env.http_user_agent or '' + user_agent = user_agent_parser.detect(http_user_agent) for key, value in user_agent.items(): if isinstance(value, dict): user_agent[key] = Storage(value) - user_agent = session._user_agent = Storage(user_agent) + user_agent = Storage(user_agent) + user_agent.is_mobile = 'Mobile' in http_user_agent + user_agent.is_tablet = 'Tablet' in http_user_agent + session._user_agent = user_agent + return user_agent def requires_https(self): @@ -468,45 +473,67 @@ class Response(Storage): response.cache_includes = (cache_method, time_expire). Example: (cache.disk, 60) # caches to disk for 1 minute. """ + app = current.request.application + + # We start by building a files list in which adjacent files internal to + # the application are placed in a list inside the files list. + # + # We will only minify and concat adjacent internal files as there's + # no way to know if changing the order with which the files are apppended + # will break things since the order matters in both CSS and JS and + # internal files may be interleaved with external ones. files = [] - ext_files = [] - has_js = has_css = False + # For the adjacent list we're going to use storage List to both distinguish + # from the regular list and so we can add attributes + internal = List() + internal.has_js = False + internal.has_css = False + done = set() # to remove duplicates for item in self.files: - if isinstance(item, (list, tuple)): - ext_files.append(item) + if not isinstance(item, list): + if item in done: + continue + done.add(item) + if isinstance(item, (list, tuple)) or not item.startswith('/' + app): # also consider items in other web2py applications to be external + if internal: + files.append(internal) + internal = List() + internal.has_js = False + internal.has_css = False + files.append(item) continue if extensions and not item.rpartition('.')[2] in extensions: continue - if item in files: - continue + internal.append(item) if item.endswith('.js'): - has_js = True + internal.has_js = True if item.endswith('.css'): - has_css = True - files.append(item) + internal.has_css = True + if internal: + files.append(internal) - if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)): - # cache for 5 minutes by default - key = hashlib_md5(repr(files)).hexdigest() - cache = self.cache_includes or (current.cache.ram, 60 * 5) - - def call_minify(files=files): - return minify.minify(files, - URL('static', 'temp'), - current.request.folder, - self.optimize_css, - self.optimize_js) - if cache: - cache_model, time_expire = cache - files = cache_model('response.files.minified/' + key, - call_minify, - time_expire) - else: - files = call_minify() - - files.extend(ext_files) - s = [] - for item in files: + # We're done we can now minify + if have_minify: + for i, f in enumerate(files): + if isinstance(f, List) and ((self.optimize_css and f.has_css) or (self.optimize_js and f.has_js)): + # cache for 5 minutes by default + key = hashlib_md5(repr(f)).hexdigest() + cache = self.cache_includes or (current.cache.ram, 60 * 5) + def call_minify(files=f): + return List(minify.minify(files, + URL('static', 'temp'), + current.request.folder, + self.optimize_css, + self.optimize_js)) + if cache: + cache_model, time_expire = cache + files[i] = cache_model('response.files.minified/' + key, + call_minify, + time_expire) + else: + files[i] = call_minify() + + def static_map(s, item): if isinstance(item, str): f = item.lower().split('?')[0] ext = f.rpartition('.')[2] @@ -526,6 +553,13 @@ class Response(Storage): if tmpl: s.append(tmpl % item[1]) + s = [] + for item in files: + if isinstance(item, List): + for f in item: + static_map(s, f) + else: + static_map(s, item) self.write(''.join(s), escape=False) def stream(self, @@ -641,6 +675,11 @@ class Response(Storage): if download_filename is None: download_filename = filename if attachment: + # Browsers still don't have a simple uniform way to have non ascii + # characters in the filename so for now we are percent encoding it + if isinstance(download_filename, unicodeT): + download_filename = download_filename.encode('utf-8') + download_filename = urllib_quote(download_filename) headers['Content-Disposition'] = \ 'attachment; filename="%s"' % download_filename.replace('"', '\"') return self.stream(stream, chunk_size=chunk_size, request=request) diff --git a/gluon/html.py b/gluon/html.py index f5383109..73eae9b8 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -2431,7 +2431,7 @@ class BEAUTIFY(DIV): if level == 0: return for c in self.components: - if hasattr(c, 'value') and not callable(c.value): + if hasattr(c, 'value') and not callable(c.value) and not isinstance(c, cgi.FieldStorage): if c.value: components.append(c.value) if hasattr(c, 'xml') and callable(c.xml): diff --git a/gluon/http.py b/gluon/http.py index 539a00fd..179dc4fe 100644 --- a/gluon/http.py +++ b/gluon/http.py @@ -11,7 +11,7 @@ HTTP statuses helpers """ import re -from gluon._compat import iteritems, unicodeT +from gluon._compat import iteritems, unicodeT, to_bytes __all__ = ['HTTP', 'redirect'] @@ -111,6 +111,8 @@ class HTTP(Exception): if not body: body = status if isinstance(body, (str, bytes, bytearray)): + if isinstance(body, unicodeT): + body = to_bytes(body) # This must be done before len headers['Content-Length'] = len(body) rheaders = [] for k, v in iteritems(headers): @@ -123,12 +125,15 @@ class HTTP(Exception): return [''] elif isinstance(body, (str, bytes, bytearray)): if isinstance(body, unicodeT): - body = body.encode('utf-8') + body = to_bytes(body) return [body] elif hasattr(body, '__iter__'): return body else: - return [str(body)] + body = str(body) + if isinstance(body, unicodeT): + body = to_bytes(body) + return [body] @property def message(self): diff --git a/gluon/languages.py b/gluon/languages.py index b626457c..aff3db14 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -18,10 +18,11 @@ import pkgutil import logging from cgi import escape from threading import RLock -from gluon.utf8 import Utf8 + from gluon.utils import local_html_escape from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin + from pydal.contrib.portalocker import read_locked, LockedFile from gluon.fileutils import listdir @@ -49,8 +50,10 @@ DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word if PY2: NUMBERS = (int, long, float) + from gluon.utf8 import Utf8 else: NUMBERS = (int, float) + Utf8 = str # pattern to find T(blah blah blah) expressions PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P'\ diff --git a/gluon/main.py b/gluon/main.py index 32103ea6..e5d73e21 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -745,7 +745,7 @@ class HttpServer(object): sock_list = [ip, port] if not ssl_certificate or not ssl_private_key: logger.info('SSL is off') - elif not rocket.ssl: + elif not rocket.has_ssl: logger.warning('Python "ssl" module unavailable. SSL is OFF') elif not exists(ssl_certificate): logger.warning('unable to open SSL certificate. SSL is OFF') diff --git a/gluon/newcron.py b/gluon/newcron.py index d422f074..d53ecbe5 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -225,7 +225,7 @@ def parsecronline(line): params = line.strip().split(None, 6) if len(params) < 7: return None - daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, + daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 'fri': 5, 'sat': 6} for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): if not s in [None, '*']: diff --git a/gluon/packages/dal b/gluon/packages/dal index 3e0fd7c0..f9f0fdfc 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 3e0fd7c01c340dec0f18a228fb91b3334b89f7f6 +Subproject commit f9f0fdfc9a9bedb40a191e5b85b44cb08c672f17 diff --git a/gluon/restricted.py b/gluon/restricted.py index 3fe5bfb0..115470e3 100644 --- a/gluon/restricted.py +++ b/gluon/restricted.py @@ -10,7 +10,7 @@ Restricted environment to execute application's code """ import sys -from gluon._compat import pickle, ClassType +from gluon._compat import pickle, ClassType, unicodeT, to_bytes import traceback import types import os @@ -192,10 +192,10 @@ class RestrictedError(Exception): # safely show an useful message to the user try: output = self.output - if isinstance(output, unicode): - output = output.encode("utf8") - elif not isinstance(output, str): + if not isinstance(output, str, bytes, bytearray): output = str(output) + if isinstance(output, unicodeT): + output = to_bytes(output) except: output = "" return output @@ -205,7 +205,7 @@ def compile2(code, layer): return compile(code, layer, 'exec') -def restricted(ccode, environment=None, layer='Unknown'): +def restricted(ccode, environment=None, layer='Unknown', scode=None): """ Runs code in environment and returns the output. If an exception occurs in code it raises a RestrictedError containing the traceback. Layer is @@ -230,7 +230,9 @@ def restricted(ccode, environment=None, layer='Unknown'): sys.excepthook(etype, evalue, tb) del tb output = "%s %s" % (etype, evalue) - raise RestrictedError(layer, ccode, output, environment) + # Save source code in ticket when available + scode = scode if scode else ccode + raise RestrictedError(layer, scode, output, environment) def snapshot(info=None, context=5, code=None, environment=None): diff --git a/gluon/sanitizer.py b/gluon/sanitizer.py index 5cd2ea4a..b98b7477 100644 --- a/gluon/sanitizer.py +++ b/gluon/sanitizer.py @@ -145,7 +145,7 @@ class XssCleaner(HTMLParser): if url.startswith('#'): return True else: - parsed = urlparse(url) + parsed = urlparse.urlparse(url) return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or (parsed[0] == '' and parsed[2].startswith('/'))) diff --git a/gluon/serializers.py b/gluon/serializers.py index d53740d7..032cff81 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -119,8 +119,8 @@ def xml(value, encoding='UTF-8', key='document', quote=True): return ('' % encoding) + str(xml_rec(value, key, quote)) -def json(value, default=custom_json, indent=None): - value = json_parser.dumps(value, default=default, sort_keys=True, indent=indent) +def json(value, default=custom_json, indent=None, sort_keys=False): + value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent) # replace JavaScript incompatible spacing # http://timelessrepo.com/json-isnt-a-javascript-subset # PY3 FIXME diff --git a/gluon/shell.py b/gluon/shell.py index b6c8f27b..38d12ec4 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -31,10 +31,16 @@ from gluon.globals import Request, Response, Session from gluon.storage import Storage, List from gluon.admin import w2p_unpack from pydal.base import BaseAdapter -from gluon._compat import iteritems, ClassType +from gluon._compat import iteritems, ClassType, PY2 logger = logging.getLogger("web2py") +if not PY2: + def execfile(filename, global_vars=None, local_vars=None): + with open(filename) as f: + code = compile(f.read(), filename, 'exec') + exec(code, global_vars, local_vars) + def enable_autocomplete_and_history(adir, env): try: diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index f12ba004..555123e9 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -658,7 +658,8 @@ class AutocompleteWidget(object): orderby=None, limitby=(0, 10), distinct=False, keyword='_autocomplete_%(tablename)s_%(fieldname)s', min_length=2, help_fields=None, help_string=None, - at_beginning=True, default_var='ac'): + at_beginning=True, default_var='ac', user_signature=True, + hash_vars=False): self.help_fields = help_fields or [] self.help_string = help_string @@ -683,7 +684,8 @@ class AutocompleteWidget(object): if hasattr(request, 'application'): urlvars = request.vars urlvars[default_var] = 1 - self.url = URL(args=request.args, vars=urlvars) + self.url = URL(args=request.args, vars=urlvars, + user_signature=user_signature, hash_vars=hash_vars) self.run_callback = True else: self.url = request @@ -1920,8 +1922,8 @@ class SQLFORM(FORM): del attributes['table_name'] # Clone fields, while passing tables straight through - fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields] - + fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields] + return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes) @staticmethod @@ -1937,11 +1939,13 @@ class SQLFORM(FORM): if settings.global_settings.web2py_runtime_gae: return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields]) else: + if not (sfields and key and key.split()): + return fields[0].table return reduce(lambda a,b:a&b,[ reduce(lambda a,b: a|b, [ field.contains(k) for field in sfields] ) for k in key.split()]) - + # from https://groups.google.com/forum/#!topic/web2py/hKe6lI25Bv4 # needs testing... #words = key.split(' ') if key else [] @@ -2161,6 +2165,7 @@ class SQLFORM(FORM): represent_none=None, showblobs=False): + dbset = None formstyle = formstyle or current.response.formstyle if isinstance(query, Set): query = query.query @@ -3039,6 +3044,7 @@ class SQLFORM(FORM): res.view_form = view_form res.search_form = search_form res.rows = rows + res.dbset = dbset return res @staticmethod @@ -3157,8 +3163,8 @@ class SQLFORM(FORM): # if isinstance(linked_tables, dict): # linked_tables = linked_tables.get(table._tablename, []) if linked_tables is None or referee in linked_tables: - field.represent = (lambda id, r=None, referee=referee, rep=field.represent: - A(callable(rep) and rep(id) or id, + field.represent = (lambda id, r=None, referee=referee, rep=field.represent: + A(callable(rep) and rep(id) or id, cid=request.cid, _href=url(args=['view', referee, id]))) except (KeyError, ValueError, TypeError): redirect(URL(args=table._tablename)) @@ -3564,7 +3570,9 @@ class ExportClass(object): if not self.rows.db._adapter.REGEX_TABLE_DOT_FIELD.match(col): row.append(record._extra[col]) else: - (t, f) = col.split('.') + # The grid code modifies rows.colnames, adding double quotes + # around the table and field names -- so they must be removed here. + (t, f) = [name.strip('"') for name in col.split('.')] field = self.rows.db[t][f] if isinstance(record.get(t, None), (Row, dict)): value = record[t][f] diff --git a/gluon/tests/test_globals.py b/gluon/tests/test_globals.py index 390079ce..da403dbe 100644 --- a/gluon/tests/test_globals.py +++ b/gluon/tests/test_globals.py @@ -158,10 +158,10 @@ class testResponse(unittest.TestCase): response.files.append(URL('a', 'static', 'css/file.ts')) content = return_includes(response) self.assertEqual(content, + '' + '' + '' + - '' + - '' + '' ) response = Response() diff --git a/gluon/tools.py b/gluon/tools.py index 1f54e746..0cf9b205 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1320,7 +1320,7 @@ class AuthJWT(object): # is the following safe or should we use # calendar.timegm(datetime.datetime.utcnow().timetuple()) # result seem to be the same (seconds since epoch, in UTC) - now = time.mktime(datetime.datetime.now().timetuple()) + now = time.mktime(datetime.datetime.utcnow().timetuple()) expires = now + self.expiration payload = dict( hmac_key=session_auth['hmac_key'], @@ -1332,7 +1332,7 @@ class AuthJWT(object): return payload def refresh_token(self, orig_payload): - now = time.mktime(datetime.datetime.now().timetuple()) + now = time.mktime(datetime.datetime.utcnow().timetuple()) if self.verify_expiration: orig_exp = orig_payload['exp'] if orig_exp + self.leeway < now: @@ -1792,6 +1792,7 @@ class Auth(AuthAPI): servicevalidate='serviceValidate', proxyvalidate='proxyValidate', logout='logout'), + cas_create_user=True, extra_fields={}, actions_disabled=[], controller=controller, @@ -2284,6 +2285,7 @@ class Auth(AuthAPI): If the user doesn't yet exist, then they are created. """ table_user = self.table_user() + create_user = self.settings.cas_create_user user = None checks = [] # make a guess about who this user is @@ -2316,6 +2318,11 @@ class Auth(AuthAPI): update_keys[key] = keys[key] user.update_record(**update_keys) elif checks: + if create_user is False: + # Remove current open session a send message + self.logout(next=None, onlogout=None, log=None) + raise HTTP(403, "Forbidden. User need to be created first.") + if 'first_name' not in keys and 'first_name' in table_user.fields: guess = keys.get('email', 'anonymous').split('@')[0] keys['first_name'] = keys.get('username', guess) @@ -2863,7 +2870,7 @@ class Auth(AuthAPI): auth.settings.auth_two_factor_enabled = True auth.messages.two_factor_comment = "Verify your OTP Client for the code." - auth.settings.two_factor_methods = [lambda user, + auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)] auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)] @@ -3656,6 +3663,16 @@ class Auth(AuthAPI): if not self.is_logged_in(): redirect(self.settings.login_url, client_side=self.settings.client_side) + + # Go to external link to change the password + if self.settings.login_form != self: + cas = self.settings.login_form + # To prevent error if change_password_url function is not defined in alternate login + if hasattr(cas, 'change_password_url'): + next = cas.change_password_url(next) + if next is not None: + redirect(next) + db = self.db table_user = self.table_user() s = db(table_user.id == self.user.id) diff --git a/gluon/widget.py b/gluon/widget.py index 0804fa96..20b602bb 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -140,7 +140,7 @@ class web2pyDialog(object): else: import tkinter from tkinter import messagebox - + bg_color = 'white' root.withdraw() @@ -463,7 +463,7 @@ class web2pyDialog(object): import tkMessageBox as messagebox else: from tkinter import messagebox - + messagebox.showerror('web2py start server', message) def start(self): @@ -1076,7 +1076,10 @@ def start_schedulers(options): return # Work around OS X problem: http://bugs.python.org/issue9405 - import urllib + if PY2: + import urllib + else: + import urllib.request as urllib urllib.getproxies() for app in apps: diff --git a/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh b/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh index 9d7fcee8..e34959d3 100644 --- a/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh +++ b/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh @@ -222,7 +222,7 @@ echo <