From 9f79dccb05dcaae0ab9af3e7a5736422aad632e0 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Tue, 20 Jun 2017 22:12:43 +0200 Subject: [PATCH 01/20] fix minify, added tests --- gluon/contrib/minify/cssmin.py | 5 ++++- gluon/contrib/minify/minify.py | 26 ++++++++++++++++++-------- gluon/globals.py | 7 +++---- gluon/tests/test_appadmin.py | 16 ++++++++++++++++ 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/gluon/contrib/minify/cssmin.py b/gluon/contrib/minify/cssmin.py index f9ed89f9..15d97a0b 100644 --- a/gluon/contrib/minify/cssmin.py +++ b/gluon/contrib/minify/cssmin.py @@ -10,8 +10,11 @@ Original author: Zachary Voase Modified for inclusion into web2py by: Ross Peoples """ +try: + from StringIO import StringIO +except ImportError: + from io import StringIO -from StringIO import StringIO # The pure-Python StringIO supports unicode. import re diff --git a/gluon/contrib/minify/minify.py b/gluon/contrib/minify/minify.py index 8749acd7..134cc664 100644 --- a/gluon/contrib/minify/minify.py +++ b/gluon/contrib/minify/minify.py @@ -8,30 +8,40 @@ Created by: Ross Peoples Modified by: Massimo Di Pierro """ -import cssmin -import jsmin +from . import cssmin +from . import jsmin import os import hashlib import re +import sys +PY2 = sys.version_info[0] == 2 +if PY2: + hashlib_md5 = hashlib.md5 +else: + hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8')) + +def open_py23(filename, mode): + if PY2: + f = open(filename, mode + 'b') + else: + f = open(filename, mode, encoding="utf8") + return f def read_binary_file(filename): - f = open(filename, 'rb') + f = open_py23(filename, 'r') data = f.read() f.close() return data - def write_binary_file(filename, data): - f = open(filename, 'wb') + f = open_py23(filename, 'w') f.write(data) f.close() - def fix_links(css, static_path): return re.sub(r'url\((["\'])\.\./', 'url(\\1' + static_path, css) - def minify(files, path_info, folder, optimize_css, optimize_js, ignore_concat=[], ignore_minify=['/jquery.js', '/anytime.js']): @@ -109,7 +119,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js, js.append(contents) else: js.append(filename) - dest_key = hashlib.md5(repr(processed)).hexdigest() + dest_key = hashlib_md5(repr(processed)).hexdigest() if css and concat_css: css = '\n\n'.join(contents for contents in css) if not inline_css: diff --git a/gluon/globals.py b/gluon/globals.py index 9d17fae6..5b963f0a 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -13,7 +13,7 @@ Contains the classes for the global used variables: - Session """ -from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT, long +from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT, long, hashlib_md5 from gluon.storage import Storage, List from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE from gluon.contenttype import contenttype @@ -457,7 +457,6 @@ class Response(Storage): self.write(s, escape=False) def include_files(self, extensions=None): - """ Includes files (usually in the head). Can minify and cache local files @@ -484,8 +483,7 @@ class Response(Storage): if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)): # cache for 5 minutes by default - key = hashlib.md5(repr(files)).hexdigest() - + key = hashlib_md5(repr(files)).hexdigest() cache = self.cache_includes or (current.cache.ram, 60 * 5) def call_minify(files=files): @@ -523,6 +521,7 @@ class Response(Storage): tmpl = template_mapping.get(f) if tmpl: s.append(tmpl % item[1]) + self.write(''.join(s), escape=False) def stream(self, diff --git a/gluon/tests/test_appadmin.py b/gluon/tests/test_appadmin.py index 41613152..22b1d05a 100644 --- a/gluon/tests/test_appadmin.py +++ b/gluon/tests/test_appadmin.py @@ -17,6 +17,7 @@ from gluon import fileutils from gluon.dal import DAL, Field, Table from gluon.http import HTTP from gluon.fileutils import open_file +from gluon.cache import CacheInRam DEFAULT_URI = os.getenv('DB', 'sqlite:memory') @@ -104,6 +105,21 @@ class TestAppAdmin(unittest.TestCase): self._test_index() remove_compiled_application(appname_path) + def test_index_minify(self): + # test for gluon/contrib/minify + self.env['response'].optimize_css = 'concat|minify' + self.env['response'].optimize_js = 'concat|minify' + self.env['current'].cache = Storage({'ram':CacheInRam()}) + appname_path = os.path.join(os.getcwd(), 'applications', 'welcome') + self._test_index() + file_l = os.listdir(os.path.join(appname_path, 'static', 'temp')) + file_l.sort() + self.assertTrue(len(file_l) == 2) + self.assertEqual(file_l[0][0:10], 'compressed') + self.assertEqual(file_l[1][0:10], 'compressed') + self.assertEqual(file_l[0][-3:], 'css') + self.assertEqual(file_l[1][-2:], 'js') + def test_select(self): request = self.env['request'] request.args = List(['db']) From 7ada2cf89aae44fb24fdb144963494741d1f832a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 11:27:54 -0400 Subject: [PATCH 02/20] Enhance tools.py PEP8 compliancy --- gluon/tools.py | 88 ++++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/gluon/tools.py b/gluon/tools.py index ce6db5dc..7c9c260f 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1128,7 +1128,6 @@ def addrow(form, a, b, c, style, _id, position=-1): class AuthJWT(object): - """ Experimental! @@ -1538,7 +1537,8 @@ class Auth(AuthAPI): # ## these are messages that can be customized default_messages = dict(AuthAPI.default_messages, access_denied='Insufficient privileges', - bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete the process', + bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete ' + 'the process', bulk_invite_subject='Invitation to join %(site)s', delete_label='Check to delete', email_sent='Email sent', @@ -1841,14 +1841,15 @@ class Auth(AuthAPI): # ## these are messages that can be customized messages = self.messages = Messages(current.T) messages.update(Auth.default_messages) - messages.update(ajax_failed_authentication=DIV(H4('NOT AUTHORIZED'), - 'Please ', - A('login', - _href=self.settings.login_url + - ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location)) - if current.request.env.http_web2py_component_location else ''), - ' to view this content.', - _class='not-authorized alert alert-block')) + messages.update(ajax_failed_authentication= + DIV(H4('NOT AUTHORIZED'), + 'Please ', + A('login', + _href=self.settings.login_url + + ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location)) + if current.request.env.http_web2py_component_location else ''), + ' to view this content.', + _class='not-authorized alert alert-block')) messages.lock_keys = True # for "remember me" option @@ -1877,7 +1878,7 @@ class Auth(AuthAPI): # _next variable in the request. if next: parts = next.split('/') - if not ':' in parts[0]: + if ':' not in parts[0]: return next elif len(parts) > 2 and parts[0].endswith(':') and parts[1:3] == ['', host]: return next @@ -2009,8 +2010,7 @@ class Auth(AuthAPI): items.append({'name': T('Lost password?'), 'href': href('request_reset_password'), 'icon': 'icon-lock'}) - if (self.settings.use_username and not - 'retrieve_username' in self.settings.actions_disabled): + if self.settings.use_username and 'retrieve_username' not in self.settings.actions_disabled: items.append({'name': T('Forgot username?'), 'href': href('retrieve_username'), 'icon': 'icon-edit'}) @@ -2182,9 +2182,7 @@ class Auth(AuthAPI): current_record.replace('_', ' ').title()) for table in tables: fieldnames = table.fields() - if ('id' in fieldnames and - 'modified_on' in fieldnames and - not current_record in fieldnames): + if 'id' in fieldnames and 'modified_on' in fieldnames and current_record not in fieldnames: table._enable_record_versioning(archive_db=archive_db, archive_name=archive_names, current_record=current_record, @@ -2214,7 +2212,8 @@ class Auth(AuthAPI): fake_migrate = db._fake_migrate settings = self.settings settings.enable_tokens = enable_tokens - signature_list = super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list + signature_list = \ + super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list now = current.request.now reference_table_user = 'reference %s' % settings.table_user_name @@ -2361,7 +2360,7 @@ class Auth(AuthAPI): if callable(basic_auth_realm): basic_auth_realm = basic_auth_realm() elif isinstance(basic_auth_realm, (unicode, str)): - basic_realm = unicode(basic_auth_realm) + basic_realm = unicode(basic_auth_realm) # Warning python 3.5 does not have method unicod elif basic_auth_realm is True: basic_realm = u'' + current.request.application http_401 = HTTP(401, u'Not Authorized', **{'WWW-Authenticate': u'Basic realm="' + basic_realm + '"'}) @@ -2463,7 +2462,7 @@ class Auth(AuthAPI): _href=service + query_sep + "ticket=" + ticket) else: redirect(service + query_sep + "ticket=" + ticket) - if self.is_logged_in() and not 'renew' in request.vars: + if self.is_logged_in() and 'renew' not in request.vars: return allow_access() elif not self.is_logged_in() and 'gateway' in request.vars: redirect(session._cas_service) @@ -2780,7 +2779,7 @@ class Auth(AuthAPI): # If auth.settings.auth_two_factor_enabled it will enable two factor # for all the app. Another way to anble two factor is that the user # must be part of a group that is called auth.settings.two_factor_authentication_group - if user and self.settings.auth_two_factor_enabled == True: + if user and self.settings.auth_two_factor_enabled is True: session.auth_two_factor_enabled = True elif user and self.settings.two_factor_authentication_group: role = self.settings.two_factor_authentication_group @@ -2810,7 +2809,7 @@ class Auth(AuthAPI): # Set the way we generate the code or we send the code. For example using SMS... two_factor_methods = self.settings.two_factor_methods - if two_factor_methods == []: + if not two_factor_methods: # TODO: Add some error checking to handle cases where email cannot be sent self.settings.mailer.send( to=user.email, @@ -2833,47 +2832,49 @@ class Auth(AuthAPI): hideerror=settings.hideerror): accepted_form = True - ''' + """ The lists is executed after form validation for each of the corresponding action. For example, in your model: In your models copy and paste: - #Before define tables, we add some extra field to auth_user + # Before define tables, we add some extra field to auth_user auth.settings.extra_fields['auth_user'] = [ Field('motp_secret', 'password', length=512, default='', label='MOTP Secret'), Field('motp_pin', 'string', length=128, default='', label='MOTP PIN')] - OFFSET = 60 #Be sure is the same in your OTP Client + OFFSET = 60 # Be sure is the same in your OTP Client - #Set session.auth_two_factor to None. Because the code is generated by external app. + # Set session.auth_two_factor to None. Because the code is generated by external app. # This will avoid to use the default setting and send a code by email. def _set_two_factor(user, auth_two_factor): return None def verify_otp(user, otp): - import time - from hashlib import md5 - epoch_time = int(time.time()) - time_start = int(str(epoch_time - OFFSET)[:-1]) - time_end = int(str(epoch_time + OFFSET)[:-1]) - for t in range(time_start - 1, time_end + 1): - to_hash = str(t) + user.motp_secret + user.motp_pin - hash = md5(to_hash).hexdigest()[:6] - if otp == hash: - return hash + import time + from hashlib import md5 + epoch_time = int(time.time()) + time_start = int(str(epoch_time - OFFSET)[:-1]) + time_end = int(str(epoch_time + OFFSET)[:-1]) + for t in range(time_start - 1, time_end + 1): + to_hash = str(t) + user.motp_secret + user.motp_pin + hash = md5(to_hash).hexdigest()[:6] + if otp == hash: + return hash auth.settings.auth_two_factor_enabled = True auth.messages.two_factor_comment = "Verify your OTP Client for the code." - auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)] + auth.settings.two_factor_methods = [lambda user, + auth_two_factor: _set_two_factor(user, auth_two_factor)] auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)] - ''' - if self.settings.two_factor_onvalidation != []: + """ + if self.settings.two_factor_onvalidation: for two_factor_onvalidation in self.settings.two_factor_onvalidation: try: - session.auth_two_factor = two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code']) + session.auth_two_factor = \ + two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code']) except: pass else: @@ -4147,7 +4148,7 @@ class Auth(AuthAPI): archive_table = table._db[archive_table_name] new_record = {current_record: form.vars.id} for fieldname in archive_table.fields: - if not fieldname in ['id', current_record]: + if fieldname not in ['id', current_record]: if archive_current and fieldname in form.vars: new_record[fieldname] = form.vars[fieldname] elif form.record and fieldname in form.record: @@ -4957,7 +4958,10 @@ class Service(object): Then call it with: - wget --post-data '{"jsonrpc": "2.0", "id": 1, "method": "myfunction", "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2 + wget --post-data '{"jsonrpc": "2.0", + "id": 1, + "method": "myfunction", + "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2 """ self.jsonrpc2_procedures[f.__name__] = f @@ -5781,7 +5785,7 @@ class Expose(object): return os.path.realpath(f) def issymlink_out(self, f): - "True if f is a symlink and is pointing outside of self.base" + """True if f is a symlink and is pointing outside of self.base""" return os.path.islink(f) and not self.in_base(f) @staticmethod From 583d106104df6321fd11614c712968ac786a4926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 11:33:00 -0400 Subject: [PATCH 03/20] Fix docstring py3 compatibility issues print -> print() --- gluon/tools.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gluon/tools.py b/gluon/tools.py index 7c9c260f..039d972b 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -5605,12 +5605,12 @@ class PluginManager(object): where the plugin is used:: - >>> print plugins.me.param1 + >>> print(plugins.me.param1) 3 - >>> print plugins.me.param2 + >>> print(plugins.me.param2) 6 >>> plugins.me.param3 = 8 - >>> print plugins.me.param3 + >>> print(plugins.me.param3) 8 Here are some tests:: @@ -5618,25 +5618,25 @@ class PluginManager(object): >>> a=PluginManager() >>> a.x=6 >>> b=PluginManager('check') - >>> print b.x + >>> print(b.x) 6 >>> b=PluginManager() # reset settings - >>> print b.x + >>> print(b.x) >>> b.x=7 - >>> print a.x + >>> print(a.x) 7 >>> a.y.z=8 - >>> print b.y.z + >>> print(b.y.z) 8 >>> test_thread_separation() 5 >>> plugins=PluginManager('me',db='mydb') - >>> print plugins.me.db + >>> print(plugins.me.db) mydb - >>> print 'me' in plugins + >>> print('me' in plugins) True - >>> print plugins.me.installed + >>> print(plugins.me.installed) True """ From 590de9c8900ac1d5da58758730ad9c805c7e8429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 11:57:31 -0400 Subject: [PATCH 04/20] Enhance validators.py PEP8 and fix docstring py3 compatibility --- gluon/validators.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/gluon/validators.py b/gluon/validators.py index d7fa6260..cff35918 100644 --- a/gluon/validators.py +++ b/gluon/validators.py @@ -21,7 +21,8 @@ import struct import decimal import unicodedata -from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native, string_types, urlparse +from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, \ + to_unicode, to_native, string_types, urlparse from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE from pydal.objects import Field, FieldVirtual, FieldMethod from functools import reduce @@ -455,7 +456,7 @@ class IS_IN_SET(Validator): items = [(k, self.labels[i]) for (i, k) in enumerate(self.theset)] if self.sort: items.sort(key=lambda o: str(o[1]).upper()) - if zero and not self.zero is None and not self.multiple: + if zero and self.zero is not None and not self.multiple: items.insert(0, ('', self.zero)) return items @@ -823,7 +824,7 @@ class IS_INT_IN_RANGE(Validator): def str2dec(number): s = str(number) - if not '.' in s: + if '.' not in s: s += '.00' else: s += '0' * (2 - len(s.split('.')[1])) @@ -1213,7 +1214,7 @@ class IS_EMAIL(Validator): domain_encoded = to_unicode(domain).encode('idna').decode('ascii') match_domain = self.domain_regex.match(domain_encoded) - match = (match_body != None) and (match_domain != None) + match = (match_body is not None) and (match_domain is not None) except (TypeError, UnicodeError): # Value may not be a string where we can look for matches. # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date. @@ -1247,7 +1248,7 @@ class IS_LIST_OF_EMAILS(object): f = IS_EMAIL() for email in self.split_emails.findall(value): error = f(email)[1] - if error and not email in bad_emails: + if error and email not in bad_emails: bad_emails.append(email) if not bad_emails: return (value, None) @@ -1461,9 +1462,9 @@ def unicode_to_ascii_authority(authority): if label: asciiLabels.append(to_native(encodings.idna.ToASCII(label))) else: - # encodings.idna.ToASCII does not accept an empty string, but - # it is necessary for us to allow for empty labels so that we - # don't modify the URL + # encodings.idna.ToASCII does not accept an empty string, but + # it is necessary for us to allow for empty labels so that we + # don't modify the URL asciiLabels.append('') # RFC 3490, Section 4, Step 5 return str(reduce(lambda x, y: x + unichr(0x002E) + y, asciiLabels)) @@ -1527,13 +1528,17 @@ def unicode_to_ascii_url(url, prepend_scheme): if prepended: scheme = '' - unparsed = urlparse.urlunparse((scheme, unicode_to_ascii_authority(authority), escape_unicode(path), '', escape_unicode(query), str(fragment))) + unparsed = urlparse.urlunparse((scheme, + unicode_to_ascii_authority(authority), + escape_unicode(path), + '', + escape_unicode(query), + str(fragment))) if unparsed.startswith('//'): - unparsed = unparsed[2:] # Remove the // urlunparse puts in the beginning + unparsed = unparsed[2:] # Remove the // urlunparse puts in the beginning return unparsed - class IS_GENERIC_URL(Validator): """ Rejects a URL string if any of the following is true: @@ -2622,7 +2627,7 @@ class ANY_OF(Validator): def __call__(self, value): for validator in self.subs: value, error = validator(value) - if error == None: + if error is None: break return value, error @@ -2762,7 +2767,7 @@ class LazyCrypt(object): else: digest_alg, key = self.crypt.digest_alg, '' if self.crypt.salt: - if self.crypt.salt == True: + if self.crypt.salt: salt = str(web2py_uuid()).replace('-', '')[-16:] else: salt = self.crypt.salt @@ -2847,7 +2852,7 @@ class CRYPT(object): Supports standard algorithms >>> for alg in ('md5','sha1','sha256','sha384','sha512'): - ... print str(CRYPT(digest_alg=alg,salt=True)('test')[0]) + ... print(str(CRYPT(digest_alg=alg,salt=True)('test')[0])) md5$...$... sha1$...$... sha256$...$... @@ -2859,13 +2864,13 @@ class CRYPT(object): Supports for pbkdf2 >>> alg = 'pbkdf2(1000,20,sha512)' - >>> print str(CRYPT(digest_alg=alg,salt=True)('test')[0]) + >>> print(str(CRYPT(digest_alg=alg,salt=True)('test')[0])) pbkdf2(1000,20,sha512)$...$... An optional hmac_key can be specified and it is used as salt prefix >>> a = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0]) - >>> print a + >>> print(a) md5$...$... Even if the algorithm changes the hash can still be validated From 833cb03ee1bee582f27caeaf5ff9aa6bd0f7de47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 13:22:30 -0400 Subject: [PATCH 05/20] Enhance admin.py PEP8 --- gluon/admin.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/gluon/admin.py b/gluon/admin.py index 3d8d13c8..ad5e18d7 100644 --- a/gluon/admin.py +++ b/gluon/admin.py @@ -54,7 +54,8 @@ def app_pack(app, request, raise_ex=False, filenames=None): """ try: - if filenames is None: app_cleanup(app, request) + if filenames is None: + app_cleanup(app, request) filename = apath('../deposit/web2py.app.%s.w2p' % app, request) w2p_pack(filename, apath(app, request), filenames=filenames) return filename @@ -104,7 +105,8 @@ def app_cleanup(app, request): if os.path.exists(path): for f in os.listdir(path): try: - if f[:1] != '.': os.unlink(os.path.join(path, f)) + if f[:1] != '.': + os.unlink(os.path.join(path, f)) except IOError: r = False @@ -113,7 +115,8 @@ def app_cleanup(app, request): if os.path.exists(path): for f in os.listdir(path): try: - if f[:1] != '.': recursive_unlink(os.path.join(path, f)) + if f[:1] != '.': + recursive_unlink(os.path.join(path, f)) except (OSError, IOError): r = False @@ -123,7 +126,8 @@ def app_cleanup(app, request): CacheOnDisk(folder=path).clear() for f in os.listdir(path): try: - if f[:1] != '.': recursive_unlink(os.path.join(path, f)) + if f[:1] != '.': + recursive_unlink(os.path.join(path, f)) except (OSError, IOError): r = False return r @@ -175,10 +179,9 @@ def app_create(app, request, force=False, key=None, info=False): return False try: w2p_unpack('welcome.w2p', path) - for subfolder in [ - 'models', 'views', 'controllers', 'databases', - 'modules', 'cron', 'errors', 'sessions', 'cache', - 'languages', 'static', 'private', 'uploads']: + for subfolder in ['models', 'views', 'controllers', 'databases', + 'modules', 'cron', 'errors', 'sessions', 'cache', + 'languages', 'static', 'private', 'uploads']: subpath = os.path.join(path, subfolder) if not os.path.exists(subpath): os.mkdir(subpath) @@ -368,7 +371,7 @@ def unzip(filename, dir, subfolder=''): for name in sorted(zf.namelist()): if not name.startswith(subfolder): continue - #print name[n:] + # print name[n:] if name.endswith('/'): folder = os.path.join(dir, name[n:]) if not os.path.exists(folder): @@ -435,6 +438,7 @@ def add_path_first(path): if not global_settings.web2py_runtime_gae: site.addsitedir(path) + def try_mkdir(path): if not os.path.exists(path): try: @@ -444,11 +448,12 @@ def try_mkdir(path): else: os.mkdir(path) except OSError as e: - if e.strerror == 'File exists': # In case of race condition. + if e.strerror == 'File exists': # In case of race condition. pass else: raise e + def create_missing_folders(): if not global_settings.web2py_runtime_gae: for path in ('applications', 'deposit', 'site-packages', 'logs'): From 7f5fc798c5248e2211f91986588f8cec74771142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 13:30:02 -0400 Subject: [PATCH 06/20] Enhance compileapp.py PEP8 --- gluon/compileapp.py | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 3dc08bcb..a40328ca 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -41,11 +41,12 @@ import imp import logging import types from functools import reduce -logger = logging.getLogger("web2py") from gluon import rewrite from gluon.custom_import import custom_import_install import py_compile +logger = logging.getLogger("web2py") + is_pypy = settings.global_settings.is_pypy is_gae = settings.global_settings.web2py_runtime_gae is_jython = settings.global_settings.is_jython @@ -111,7 +112,7 @@ class mybuiltin(object): NOTE could simple use a dict and populate it, NOTE not sure if this changes things though if monkey patching import..... """ - #__builtins__ + # __builtins__ def __getitem__(self, key): try: return getattr(builtin, key) @@ -185,7 +186,7 @@ def LOAD(c=None, f='index', args=None, vars=None, else: statement = "$.web2py.component('%s','%s');" % (url, target) attr['_data-w2p_remote'] = url - if not target is None: + if target is not None: return DIV(content, **attr) else: @@ -211,7 +212,8 @@ def LOAD(c=None, f='index', args=None, vars=None, request.env.path_info other_request.cid = target other_request.env.http_web2py_component_element = target - other_request.restful = types.MethodType(request.restful.__func__, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful() + other_request.restful = types.MethodType(request.restful.__func__, other_request) + # A bit nasty but needed to use LOAD on action decorates with @request.restful() other_response.view = '%s/%s.%s' % (c, f, other_request.extension) other_environment = copy.copy(current.globalenv) # NASTY @@ -405,7 +407,7 @@ def build_environment(request, response, session, store_current=True): """ Build the environment dictionary into which web2py files are executed. """ - #h,v = html,validators + # h,v = html,validators environment = dict(_base_environment_) if not request.env: @@ -418,7 +420,7 @@ def build_environment(request, response, session, store_current=True): r'^%s/%s/\w+\.py$' % (request.controller, request.function) ] - t = environment['T'] = translator(os.path.join(request.folder,'languages'), + t = environment['T'] = translator(os.path.join(request.folder, 'languages'), request.env.http_accept_language) c = environment['cache'] = Cache(request) @@ -506,10 +508,12 @@ def compile_models(folder): save_pyc(filename) os.unlink(filename) + def find_exposed_functions(data): - data = regex_longcomments.sub('',data) + data = regex_longcomments.sub('', data) return regex_expose.findall(data) + def compile_controllers(folder): """ Compiles all the controllers in the application specified by `folder` @@ -524,16 +528,19 @@ def compile_controllers(folder): command = data + "\nresponse._vars=response._caller(%s)\n" % \ function filename = pjoin(folder, 'compiled', - 'controllers.%s.%s.py' % (fname[:-3],function)) + 'controllers.%s.%s.py' % (fname[:-3], function)) write_file(filename, command) save_pyc(filename) os.unlink(filename) + def model_cmp(a, b, sep='.'): return cmp(a.count(sep), b.count(sep)) or cmp(a, b) + def model_cmp_sep(a, b, sep=os.path.sep): - return model_cmp(a,b,sep) + return model_cmp(a, b, sep) + def run_models_in(environment): """ @@ -544,7 +551,7 @@ def run_models_in(environment): request = current.request folder = request.folder c = request.controller - #f = environment['request'].function + # f = environment['request'].function response = current.response path = pjoin(folder, 'models') @@ -557,9 +564,11 @@ def run_models_in(environment): models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep) else: if compiled: - models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), key=lambda f: '{0:03d}'.format(f.count('.')) + f) + models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), + key=lambda f: '{0:03d}'.format(f.count('.')) + f) else: - models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f) + models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), + key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f) models_to_run = None for model in models: @@ -570,10 +579,10 @@ def run_models_in(environment): if models_to_run: if compiled: n = len(cpath)+8 - fname = model[n:-4].replace('.','/')+'.py' + fname = model[n:-4].replace('.', '/')+'.py' else: n = len(path)+1 - fname = model[n:].replace(os.path.sep,'/') + fname = model[n:].replace(os.path.sep, '/') if not regex.search(fname) and c != 'appadmin': continue elif compiled: @@ -583,6 +592,7 @@ def run_models_in(environment): ccode = getcfs(model, model, f) restricted(ccode, environment, layer=model) + def run_controller_in(controller, function, environment): """ Runs the controller.function() (for the app specified by @@ -631,7 +641,7 @@ def run_controller_in(controller, function, environment): web2py_error=badc) code = getcfs(filename, filename, lambda: read_file(filename)) exposed = find_exposed_functions(code) - if not function in exposed: + if function not in exposed: raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) @@ -678,7 +688,7 @@ def run_view_in(environment): layer = 'file stream' else: filename = pjoin(folder, 'views', view) - if os.path.exists(cpath): # compiled views + if os.path.exists(cpath): # compiled views x = view.replace('/', '.') files = ['views.%s.pyc' % x] is_compiled = os.path.exists(pjoin(cpath, files[0])) @@ -715,6 +725,7 @@ def run_view_in(environment): # parse_template saves everything in response body return environment['response'].body.getvalue() + def remove_compiled_application(folder): """ Deletes the folder `compiled` containing the compiled application. From 2080e0460f2ee15a403049e85b723b44ca727a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 13:32:42 -0400 Subject: [PATCH 07/20] Enhance contenttype.py PEP8 --- gluon/contenttype.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gluon/contenttype.py b/gluon/contenttype.py index 7dd28f35..e5f6dd3c 100644 --- a/gluon/contenttype.py +++ b/gluon/contenttype.py @@ -330,7 +330,7 @@ CONTENT_TYPE = { '.lha': 'application/x-lha', '.lhs': 'text/x-literate-haskell', '.lhz': 'application/x-lhz', - '.load' : 'text/html', + '.load': 'text/html', '.log': 'text/x-log', '.lrz': 'application/x-lrzip', '.ltx': 'text/x-tex', @@ -823,7 +823,7 @@ CONTENT_TYPE = { '.xsd': 'application/xml', '.xsl': 'application/xslt+xml', '.xslfo': 'text/x-xslfo', - '.xslm' : 'application/vnd.ms-excel.sheet.macroEnabled.12', + '.xslm': 'application/vnd.ms-excel.sheet.macroEnabled.12', '.xslt': 'application/xslt+xml', '.xspf': 'application/xspf+xml', '.xul': 'application/vnd.mozilla.xul+xml', @@ -843,7 +843,7 @@ def contenttype(filename, default='text/plain'): """ Returns the Content-Type string matching extension of the given filename. """ - filename=to_native(filename) + filename = to_native(filename) i = filename.rfind('.') if i >= 0: default = CONTENT_TYPE.get(filename[i:].lower(), default) From 8d5464692f475f4871a7022e90bd899b3984946f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 13:41:10 -0400 Subject: [PATCH 08/20] Enhance dal.py PEP8 --- gluon/dal.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gluon/dal.py b/gluon/dal.py index 95476a9f..ceecf2e0 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -14,6 +14,11 @@ from pydal import DAL as DAL from pydal import Field from pydal.objects import Row, Rows, Table, Query, Set, Expression from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon +from gluon.serializers import custom_json, xml +from gluon.utils import web2py_uuid +from gluon import sqlhtml +from pydal.drivers import DRIVERS + def _default_validators(db, field): """ @@ -78,14 +83,10 @@ def _default_validators(db, field): if (field.notnull or field.unique) and field_type not in excluded_fields: requires.insert(0, validators.IS_NOT_EMPTY()) elif not field.notnull and not field.unique and requires: - requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None) + requires[0] = \ + validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None) return requires -from gluon.serializers import custom_json, xml -from gluon.utils import web2py_uuid -from gluon import sqlhtml - - DAL.serializers = {'json': custom_json, 'xml': xml} DAL.validators_method = _default_validators DAL.uuid = lambda x: web2py_uuid() @@ -96,8 +97,7 @@ DAL.representers = { DAL.Field = Field DAL.Table = Table -#: add web2py contrib drivers to pyDAL -from pydal.drivers import DRIVERS +# add web2py contrib drivers to pyDAL if not DRIVERS.get('pymysql'): try: from .contrib import pymysql From a8fb41333bbf496d2258f12725703208a96f195a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 14:38:52 -0400 Subject: [PATCH 09/20] Enhance decoder.py PEP8 --- gluon/decoder.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/gluon/decoder.py b/gluon/decoder.py index 4fc068dd..57044bc6 100644 --- a/gluon/decoder.py +++ b/gluon/decoder.py @@ -15,13 +15,13 @@ import codecs # None represents a potentially variable byte. "##" in the XML spec... autodetect_dict = { # bytepattern : ("name", (0x00, 0x00, 0xFE, 0xFF): ("ucs4_be"), - (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"), - (0xFE, 0xFF, None, None): ("utf_16_be"), - (0xFF, 0xFE, None, None): ("utf_16_le"), - (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"), - (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"), - (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"), - (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC") + (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"), + (0xFE, 0xFF, None, None): ("utf_16_be"), + (0xFF, 0xFE, None, None): ("utf_16_le"), + (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"), + (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"), + (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"), + (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC") } @@ -36,10 +36,10 @@ def autoDetectXMLEncoding(buffer): # buffer at once but otherwise we'd have to decode a character at # a time looking for the quote character...that's a pain - encoding = "utf_8" # according to the XML spec, this is the default - # this code successively tries to refine the default - # whenever it fails to refine, it falls back to - # the last place encoding was set. + encoding = "utf_8" + # according to the XML spec, this is the default this code successively tries to refine the default + # whenever it fails to refine, it falls back to the last place encoding was set. + if len(buffer) >= 4: bytes = (byte1, byte2, byte3, byte4) = tuple(map(ord, buffer[0:4])) enc_info = autodetect_dict.get(bytes, None) @@ -51,8 +51,7 @@ def autoDetectXMLEncoding(buffer): enc_info = None if enc_info: - encoding = enc_info # we've got a guess... these are - #the new defaults + encoding = enc_info # we've got a guess... these are the new defaults # try to find a more precise encoding using xml declaration secret_decoder_ring = codecs.lookup(encoding)[1] From 58bedd4c1a324ded681b076ead664a39737fffcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 14:54:59 -0400 Subject: [PATCH 10/20] Enhance highlight.py PEP8 --- gluon/highlight.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/gluon/highlight.py b/gluon/highlight.py index d6faff87..951dab81 100644 --- a/gluon/highlight.py +++ b/gluon/highlight.py @@ -182,8 +182,8 @@ class Highlighter(object): )), 'PYTHONMultilineString': (python_tokenizer, (('ENDMULTILINESTRING', - re.compile(r'.*?("""|\'\'\')', - re.DOTALL), 'color: darkred'), )), + re.compile(r'.*?("""|\'\'\')', + re.DOTALL), 'color: darkred'), )), 'HTML': (html_tokenizer, ( ('GOTOPYTHON', re.compile(r'\{\{'), 'color: red'), ('COMMENT', re.compile(r'|'), @@ -209,7 +209,7 @@ class Highlighter(object): mode = self.mode while i < len(data): for (token, o_re, style) in Highlighter.all_styles[mode][1]: - if not token in self.suppress_tokens: + if token not in self.suppress_tokens: match = o_re.match(data, i) if match: if style: @@ -221,7 +221,7 @@ class Highlighter(object): new_mode = \ Highlighter.all_styles[mode][0](self, token, match, style) - if not new_mode is None: + if new_mode is not None: mode = new_mode i += max(1, len(match.group())) break @@ -241,9 +241,9 @@ class Highlighter(object): style = self.styles[token] if self.span_style != style: if style != 'Keep': - if not self.span_style is None: + if self.span_style is not None: self.output.append('') - if not style is None: + if style is not None: self.output.append('' % style) self.span_style = style @@ -260,7 +260,7 @@ def highlight( ): styles = styles or {} attributes = attributes or {} - if not 'CODE' in styles: + if 'CODE' not in styles: code_style = """ font-size: 11px; font-family: Bitstream Vera Sans Mono,monospace; @@ -272,7 +272,7 @@ def highlight( white-space: pre !important;\n""" else: code_style = styles['CODE'] - if not 'LINENUMBERS' in styles: + if 'LINENUMBERS' not in styles: linenumbers_style = """ font-size: 11px; font-family: Bitstream Vera Sans Mono,monospace; @@ -283,7 +283,7 @@ def highlight( color: #A0A0A0;\n""" else: linenumbers_style = styles['LINENUMBERS'] - if not 'LINEHIGHLIGHT' in styles: + if 'LINEHIGHLIGHT' not in styles: linehighlight_style = "background-color: #EBDDE2;" else: linehighlight_style = styles['LINEHIGHLIGHT'] @@ -333,8 +333,9 @@ def highlight( == '_' and value]) if fa: fa = ' ' + fa - return '
%s
%s
'\ - % (fa, linenumbers_style, numbers, code_style, code) + return '' \ + '
%s
' \ + '
%s
' % (fa, linenumbers_style, numbers, code_style, code) if __name__ == '__main__': @@ -342,5 +343,4 @@ if __name__ == '__main__': argfp = open(sys.argv[1]) data = argfp.read() argfp.close() - print('' + highlight(data, sys.argv[2])\ - + '') + print('' + highlight(data, sys.argv[2]) + '') From 472c0ff2fb87469d5b634df125c3646d9dee0844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 15:08:52 -0400 Subject: [PATCH 11/20] Enhance html.py PEP8 --- gluon/html.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/gluon/html.py b/gluon/html.py index 6446bd8c..f5383109 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -20,7 +20,8 @@ import urllib import base64 from gluon import sanitizer, decoder import itertools -from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long +from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, \ + urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long from gluon.utils import local_html_escape import marshal @@ -109,6 +110,7 @@ __all__ = [ DEFAULT_PASSWORD_DISPLAY = '*' * 8 + def xmlescape(data, quote=True): """ Returns an escaped string of the provided data @@ -124,10 +126,9 @@ def xmlescape(data, quote=True): if not(isinstance(data, (text_type, bytes))): # i.e., integers - data=str(data) + data = str(data) data = to_bytes(data, 'utf8', 'xmlcharrefreplace') - # ... and do the escaping data = local_html_escape(data, quote) return data @@ -671,6 +672,7 @@ def XML_pickle(data): return XML_unpickle, (marshal.dumps(str(data)),) copyreg.pickle(XML, XML_pickle, XML_unpickle) + @implements_bool class DIV(XmlComponent): """ @@ -1309,8 +1311,10 @@ class HTML(DIV): tag = b'html' strict = b'\n' - transitional = b'\n' - frameset = b'\n' + transitional = \ + b'\n' + frameset = \ + b'\n' html5 = b'\n' def xml(self): @@ -1861,7 +1865,7 @@ class INPUT(DIV): except: import traceback print(traceback.format_exc()) - msg = "Validation error, field:%s %s" % (name,validator) + msg = "Validation error, field:%s %s" % (name, validator) raise Exception(msg) if errors is not None: self.vars[name] = value @@ -1911,7 +1915,7 @@ class INPUT(DIV): name = self.attributes.get('_name', None) if name and hasattr(self, 'errors') \ and self.errors.get(name, None) \ - and self['hideerror'] != True: + and self['hideerror'] is not True: self['_class'] = (self['_class'] and self['_class'] + ' ' or '') + 'invalidinput' return DIV.xml(self) + DIV( DIV( @@ -1979,7 +1983,6 @@ class OPTGROUP(DIV): class SELECT(INPUT): - """ Examples: @@ -2014,7 +2017,7 @@ class SELECT(INPUT): if value is not None: if not self['_multiple']: for c in options: # my patch - if ((value is not None) and (str(c['_value']) == str(value))): + if (value is not None) and (str(c['_value']) == str(value)): c['_selected'] = 'selected' else: c['_selected'] = None @@ -2024,7 +2027,7 @@ class SELECT(INPUT): else: values = [str(value)] for c in options: # my patch - if ((value is not None) and (str(c['_value']) in values)): + if (value is not None) and (str(c['_value']) in values): c['_selected'] = 'selected' else: c['_selected'] = None @@ -2390,7 +2393,6 @@ class FORM(DIV): class BEAUTIFY(DIV): - """ Turns any list, dictionary, etc into decent looking html. @@ -2547,7 +2549,7 @@ class MENU(DIV): li['_class'] = li['_class'] + ' ' + self['li_active'] else: li['_class'] = self['li_active'] - if len(item) <= 4 or item[4] == True: + if len(item) <= 4 or item[4] is True: ul.append(li) return ul @@ -2561,7 +2563,7 @@ class MENU(DIV): # ex: ('', False, A('title', _href=URL(...), _title="title")) # ex: (A('title', _href=URL(...), _title="title"), False, None) custom_items.append(item) - elif len(item) <= 4 or item[4] == True: + elif len(item) <= 4 or item[4] is True: select.append(OPTION(CAT(prefix, item[0]), _value=item[2], _selected=item[1])) if len(item) > 3 and len(item[3]): @@ -2703,7 +2705,8 @@ class web2pyHTMLParser(HTMLParser): self.parent = self.parent.parent except: raise RuntimeError("unable to balance tag %s" % tagname) - if parent_tagname[:len(tagname)] == tagname: break + if parent_tagname[:len(tagname)] == tagname: + break def markdown_serializer(text, tag=None, attr=None): From aa1b71e431b18118e998eaa1ac096c2dcb58b195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 15:14:41 -0400 Subject: [PATCH 12/20] Enhance http.py PEP8 --- gluon/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluon/http.py b/gluon/http.py index e34fb8f0..ac630cc6 100644 --- a/gluon/http.py +++ b/gluon/http.py @@ -116,7 +116,7 @@ class HTTP(Exception): for k, v in iteritems(headers): if isinstance(v, list): rheaders += [(k, str(item)) for item in v] - elif not v is None: + elif v is not None: rheaders.append((k, str(v))) responder(status, rheaders) if env.get('request_method', '') == 'HEAD': @@ -148,7 +148,7 @@ class HTTP(Exception): web2py_error=self.headers.get('web2py_error')) def __str__(self): - "stringify me" + """stringify me""" return self.message From b5b98d6e19ced074b768034c726758732a614b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Wed, 21 Jun 2017 15:32:28 -0400 Subject: [PATCH 13/20] Close #1664 remaining py2.6 specific code --- gluon/import_all.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gluon/import_all.py b/gluon/import_all.py index a8cbffc6..85f14864 100644 --- a/gluon/import_all.py +++ b/gluon/import_all.py @@ -78,13 +78,9 @@ alert_dependency = ['hashlib', 'uuid'] # Now we remove the blacklisted modules if we are using the stated # python version. # -# List of modules deprecated in Python 2.6 or 2.7 that are in the above set +# List of modules deprecated in Python 2.7 that are in the above list py27_deprecated = ['mhlib', 'multifile', 'mimify', 'sets', 'MimeWriter'] # And ['optparse'] but we need it for now -if python_version >= '2.6': - base_modules += ['json', 'multiprocessing'] - base_modules = list(set(base_modules).difference(set(py26_deprecated))) - if python_version >= '2.7': base_modules += ['argparse', 'json', 'multiprocessing'] base_modules = list(set(base_modules).difference(set(py27_deprecated))) From 78f3af6fc1be7d8b8c78aeb3b5daf9935c427b20 Mon Sep 17 00:00:00 2001 From: abastardi Date: Wed, 21 Jun 2017 16:29:19 -0400 Subject: [PATCH 14/20] Use FormData when supported to allow Ajax file uploads --- applications/welcome/static/js/web2py.js | 36 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/applications/welcome/static/js/web2py.js b/applications/welcome/static/js/web2py.js index 313d00bc..1b52245e 100644 --- a/applications/welcome/static/js/web2py.js +++ b/applications/welcome/static/js/web2py.js @@ -12,6 +12,8 @@ $.error('web2py.js has already been loaded!'); } + var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined'; + String.prototype.reverse = function () { return this.split('').reverse().join(''); }; @@ -320,7 +322,15 @@ form.submit(function (e) { web2py.disableElement(form.find(web2py.formInputClickSelector)); web2py.hide_flash(); - web2py.ajax_page('post', url, form.serialize(), target, form); + + var formData; + if (FORMDATA_IS_SUPPORTED) { + formData = new FormData(form[0]); // Allows file uploads. + } else { + formData = form.serialize(); // Fallback for older browsers. + } + web2py.ajax_page('post', url, formData, target, form); + e.preventDefault(); }); form.on('click', web2py.formInputClickSelector, function (e) { @@ -339,11 +349,18 @@ if (web2py.isUndefined(element)) element = $(document); /* if target is not there, fill it with something that there isn't in the page*/ if (web2py.isUndefined(target) || target === '') target = 'w2p_none'; + + /* processData and contentType must be set to false when passing a FormData + object to jQuery.ajax. */ + var isFormData = Object.prototype.toString.call(data) === '[object FormData]'; + var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8'; if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */ $.ajax({ 'type': method, 'url': action, 'data': data, + 'processData': !isFormData, + 'contentType': contentType, 'beforeSend': function (xhr, settings) { xhr.setRequestHeader('web2py-component-location', document.location); xhr.setRequestHeader('web2py-component-element', target); @@ -699,8 +716,9 @@ }); }, /* Disables form elements: + - Does not disable elements with 'data-w2p_disable' attribute - Caches element value in 'w2p_enable_with' data store - - Replaces element text with value of 'data-disable-with' attribute + - Replaces element text with value of 'data-w2p_disable_with' attribute - Sets disabled property to true */ disableFormElements: function (form) { @@ -712,13 +730,15 @@ if (!web2py.isUndefined(disable)) { return false; } - if (web2py.isUndefined(disable_with)) { - element.data('w2p_disable_with', element[method]()); + if (!element.is(':file')) { // Altering file input values is not allowed. + if (web2py.isUndefined(disable_with)) { + element.data('w2p_disable_with', element[method]()); + } + if (web2py.isUndefined(element.data('w2p_enable_with'))) { + element.data('w2p_enable_with', element[method]()); + } + element[method](element.data('w2p_disable_with')); } - if (web2py.isUndefined(element.data('w2p_enable_with'))) { - element.data('w2p_enable_with', element[method]()); - } - element[method](element.data('w2p_disable_with')); element.prop('disabled', true); }); }, From 1a52b0ee3bd57879b5d017e830ba20fa42bb543e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20V=C3=A9zina?= Date: Thu, 22 Jun 2017 09:46:55 -0400 Subject: [PATCH 15/20] Enhance main.py PEP8 --- gluon/main.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/gluon/main.py b/gluon/main.py index 24327a89..32103ea6 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -11,7 +11,8 @@ The gluon wsgi application """ from __future__ import print_function -if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS +if False: + import import_all # DO NOT REMOVE PART OF FREEZE PROCESS import gc import os @@ -26,7 +27,7 @@ import random import string from gluon._compat import Cookie, urllib2 -#from thread import allocate_lock +# from thread import allocate_lock from gluon.fileutils import abspath, write_file from gluon.settings import global_settings @@ -67,14 +68,14 @@ import gluon.messageboxhandler logging.gluon = gluon # so we must restore it! Thanks ozancag import locale -locale.setlocale(locale.LC_CTYPE, "C") # IMPORTANT, web2py requires locale "C" +locale.setlocale(locale.LC_CTYPE, "C") # IMPORTANT, web2py requires locale "C" exists = os.path.exists pjoin = os.path.join try: logging.config.fileConfig(abspath("logging.conf")) -except: # fails on GAE or when logfile is missing +except: # fails on GAE or when logfile is missing logging.basicConfig() logger = logging.getLogger("web2py") @@ -254,6 +255,7 @@ class LazyWSGI(object): return [data] for item in middleware_apps: app = item(app) + def caller(app): return app(self.environ, self.start_response) return lambda caller=caller, app=app: caller(app) @@ -294,9 +296,9 @@ def wsgibase(environ, responder): response = Response() session = Session() env = request.env - #env.web2py_path = global_settings.applications_parent + # env.web2py_path = global_settings.applications_parent env.web2py_version = web2py_version - #env.update(global_settings) + # env.update(global_settings) static_file = False http_response = None try: @@ -325,7 +327,6 @@ def wsgibase(environ, responder): 'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT' response.stream(static_file, request=request) - # ################################################## # fill in request items # ################################################## @@ -356,17 +357,15 @@ def wsgibase(environ, responder): cmd_opts = global_settings.cmd_options request.update( - client = client, - folder = abspath('applications', app) + os.sep, - ajax = x_req_with == 'xmlhttprequest', - cid = env.http_web2py_component_element, - is_local = (env.remote_addr in local_hosts and - client == env.remote_addr), - is_shell = False, - is_scheduler = False, - is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \ - request.env.http_x_forwarded_proto in HTTPS_SCHEMES \ - or env.https == 'on' + client=client, + folder=abspath('applications', app) + os.sep, + ajax=x_req_with == 'xmlhttprequest', + cid=env.http_web2py_component_element, + is_local=(env.remote_addr in local_hosts and client == env.remote_addr), + is_shell=False, + is_scheduler=False, + is_https=env.wsgi_url_scheme in HTTPS_SCHEMES or + request.env.http_x_forwarded_proto in HTTPS_SCHEMES or env.https == 'on' ) request.url = environ['PATH_INFO'] @@ -390,7 +389,7 @@ def wsgibase(environ, responder): % 'invalid request', web2py_error='invalid application') elif not request.is_local and exists(disabled): - five0three = os.path.join(request.folder,'static','503.html') + five0three = os.path.join(request.folder, 'static', '503.html') if os.path.exists(five0three): raise HTTP(503, file(five0three, 'r').read()) else: @@ -406,7 +405,7 @@ def wsgibase(environ, responder): # get the GET and POST data # ################################################## - #parse_get_post_vars(request, environ) + # parse_get_post_vars(request, environ) # ################################################## # expose wsgi hooks for convenience @@ -625,7 +624,7 @@ def appfactory(wsgiapp=wsgibase, raise BaseException("Can't create dir %s" % profiler_dir) filepath = pjoin(profiler_dir, 'wtest') try: - filehandle = open( filepath, 'w' ) + filehandle = open(filepath, 'w') filehandle.close() os.unlink(filepath) except IOError: From 054320820cf9e8b96987c578eb9297b21b7b65ff Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 24 Jun 2017 07:03:10 +0200 Subject: [PATCH 16/20] fix performance regression with compiled views --- gluon/compileapp.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 3dc08bcb..1d01d302 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -596,13 +596,13 @@ def run_controller_in(controller, function, environment): badc = 'invalid controller (%s/%s)' % (controller, function) badf = 'invalid function (%s/%s)' % (controller, function) if os.path.exists(cpath): - filename = pjoin(cpath, 'controllers.%s.%s.pyc' - % (controller, function)) - if not os.path.exists(filename): + filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function)) + try: + ccode = getcfs(filename, filename, lambda: read_pyc(filename)) + except IOError: raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) - ccode = getcfs(filename, filename, lambda: read_pyc(filename)) elif function == '_TEST': # TESTING: adjust the path to include site packages from gluon.settings import global_settings @@ -623,13 +623,13 @@ def run_controller_in(controller, function, environment): code += TEST_CODE ccode = compile2(code, filename) else: - filename = pjoin(folder, 'controllers/%s.py' - % controller) - if not os.path.exists(filename): + filename = pjoin(folder, 'controllers/%s.py' % controller) + try: + code = getcfs(filename, filename, lambda: read_file(filename)) + except IOError: raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) - code = getcfs(filename, filename, lambda: read_file(filename)) exposed = find_exposed_functions(code) if not function in exposed: raise HTTP(404, @@ -705,12 +705,14 @@ def run_view_in(environment): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badv, web2py_error=badv) - layer = filename - # Compile the template - ccode = parse_template(view, - pjoin(folder, 'views'), - context=environment) + # if the view is not compiled + if not layer: + # Compile the template + ccode = parse_template(view, + pjoin(folder, 'views'), + context=environment) + layer = filename restricted(ccode, environment, layer=layer) # parse_template saves everything in response body return environment['response'].body.getvalue() From af7bfac1e2009226d2eeaace4ed83888b4b3b6a7 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 24 Jun 2017 08:07:15 +0200 Subject: [PATCH 17/20] Testing pypy3.5 in travis, switching to ubuntu trusty --- .travis.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 74198cea..e7c88e7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,18 @@ sudo: required cache: pip +dist: "trusty" + python: - '2.7' - - 'pypy' - '3.5' - '3.6' + - 'pypy-5.3.1' + - 'pypy3.5-5.7.1-beta' + +matrix: + allow_failures: + - python: 'pypy3.5-5.7.1-beta' install: - pip install -e . @@ -33,3 +40,6 @@ notifications: addons: postgresql: "9.4" + apt: + packages: + - postgresql-9.4-postgis-2.3 From 51ce3ffd36b8d2ad53d6d521c89c518e6c503885 Mon Sep 17 00:00:00 2001 From: Andrew Willimott Date: Fri, 7 Jul 2017 20:39:42 +1200 Subject: [PATCH 18/20] Path to css file changed to explicitly point to admin app --- applications/welcome/views/appadmin.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/welcome/views/appadmin.html b/applications/welcome/views/appadmin.html index 5330b163..59b85563 100644 --- a/applications/welcome/views/appadmin.html +++ b/applications/welcome/views/appadmin.html @@ -239,7 +239,7 @@ {{=T("No databases in this application")}} {{else:}}
- +