From 986406ed80d4f16b46f0c5b8947b2eca55329687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Mon, 18 Mar 2019 15:06:38 +0000 Subject: [PATCH 001/111] Teach admin how to deal with syntax errors --- applications/admin/controllers/default.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 4a142f22..eb2d32f2 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -814,8 +814,11 @@ def edit(): if len(request.args) > 2 and request.args[1] == 'controllers': controller = (request.args[2])[:-3] - functions = find_exposed_functions(data) - functions = functions and sorted(functions) or [] + try: + functions = find_exposed_functions(data) + functions = functions and sorted(functions) or [] + except SyntaxError as err: + functions = ['SyntaxError:Line:%d' % err.lineno] else: (controller, functions) = (None, None) @@ -1127,8 +1130,11 @@ def design(): functions = {} for c in controllers: data = safe_read(apath('%s/controllers/%s' % (app, c), r=request)) - items = find_exposed_functions(data) - functions[c] = items and sorted(items) or [] + try: + items = find_exposed_functions(data) + functions[c] = items and sorted(items) or [] + except SyntaxError as err: + functions[c] = ['SyntaxError:Line:%d' % err.lineno] # Get all views views = sorted( @@ -1265,8 +1271,11 @@ def plugin(): functions = {} for c in controllers: data = safe_read(apath('%s/controllers/%s' % (app, c), r=request)) - items = find_exposed_functions(data) - functions[c] = items and sorted(items) or [] + try: + items = find_exposed_functions(data) + functions[c] = items and sorted(items) or [] + except SyntaxError as err: + functions[c] = ['SyntaxError:Line:%d' % err.lineno] # Get all views views = sorted( From 50878f33bd45455ea9f83d9ca2f6dbfbd4a2e77a Mon Sep 17 00:00:00 2001 From: Nico Zanferrari Date: Fri, 22 Mar 2019 15:33:14 +0100 Subject: [PATCH 002/111] cleanup --- extras/build_web2py/setup_app.py | 160 ------------------------------- 1 file changed, 160 deletions(-) delete mode 100755 extras/build_web2py/setup_app.py diff --git a/extras/build_web2py/setup_app.py b/extras/build_web2py/setup_app.py deleted file mode 100755 index 92985418..00000000 --- a/extras/build_web2py/setup_app.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -This is a setup.py script generated by py2applet - -Usage: - python setup.py py2app -""" - -copy_apps = False -copy_scripts = True -copy_site_packages = True -remove_build_files = True -make_zip = True -zip_filename = "web2py_osx" - -from setuptools import setup -from gluon.import_all import base_modules, contributed_modules -from gluon.fileutils import readlines_file -import os -import fnmatch -import shutil -import sys -import re -import zipfile - -#read web2py version from VERSION file -web2py_version_line = readlines_file('VERSION')[0] -#use regular expression to get just the version number -v_re = re.compile('[0-9]+\.[0-9]+\.[0-9]+') -web2py_version = v_re.search(web2py_version_line).group(0) - -class reglob: - def __init__(self, directory, pattern="*"): - self.stack = [directory] - self.pattern = pattern - self.files = [] - self.index = 0 - - def __getitem__(self, index): - while 1: - try: - file = self.files[self.index] - self.index = self.index + 1 - except IndexError: - self.index = 0 - self.directory = self.stack.pop() - self.files = os.listdir(self.directory) - else: - fullname = os.path.join(self.directory, file) - if os.path.isdir(fullname) and not os.path.islink(fullname): - self.stack.append(fullname) - if not (file.startswith('.') or file.startswith('#') or file.endswith('~')) \ - and fnmatch.fnmatch(file, self.pattern): - return fullname - -setup(app=['web2py.py'], - version=web2py_version, - description="web2py web framework", - author="Massimo DiPierro", - license="LGPL v3", - data_files=[ - 'NEWINSTALL', - 'ABOUT', - 'LICENSE', - 'VERSION', - 'splashlogo.gif', - 'logging.example.conf', - 'options_std.py', - ], - options={'py2app': { - 'argv_emulation': True, - 'includes': base_modules, - }}, - setup_requires=['py2app']) - - -def copy_folders(source, destination): - """Copy files & folders from source to destination (within dist/)""" - print 'copying %s -> %s' % (source, destination) - base = 'dist/web2py.app/Contents/Resources/' - if os.path.exists(os.path.join(base, destination)): - shutil.rmtree(os.path.join(base, destination)) - shutil.copytree(os.path.join(source), os.path.join(base, destination)) - -#Should we include applications? -copy_folders('gluon','gluon') - -if copy_apps: - copy_folders('applications', 'applications') - print "Your application(s) have been added" -else: - #only copy web2py's default applications - copy_folders('applications/admin', 'applications/admin') - copy_folders('applications/welcome', 'applications/welcome') - copy_folders('applications/examples', 'applications/examples') - print "Only web2py's admin, examples & welcome applications have been added" - - -#should we copy project's site-packages into dist/site-packages -if copy_site_packages: - #copy site-packages - copy_folders('site-packages', 'site-packages') -else: - #no worries, web2py will create the (empty) folder first run - print "Skipping site-packages" - pass - -#should we copy project's scripts into dist/scripts -if copy_scripts: - #copy scripts - copy_folders('scripts', 'scripts') -else: - #no worries, web2py will create the (empty) folder first run - print "Skipping scripts" - pass - - -#borrowed from http://bytes.com/topic/python/answers/851018-how-zip-directory-python-using-zipfile -def recursive_zip(zipf, directory, folder=""): - for item in os.listdir(directory): - if os.path.isfile(os.path.join(directory, item)): - zipf.write(os.path.join(directory, item), folder + os.sep + item) - elif os.path.isdir(os.path.join(directory, item)): - recursive_zip( - zipf, os.path.join(directory, item), folder + os.sep + item) - -#should we create a zip file of the build? - -if make_zip: - #to keep consistent with how official web2py windows zip file is setup, - #create a web2py folder & copy dist's files into it - shutil.copytree('dist', 'zip_temp/web2py') - #create zip file - #use filename specified via command line - zipf = zipfile.ZipFile( - zip_filename + ".zip", "w", compression=zipfile.ZIP_DEFLATED) - path = 'zip_temp' # just temp so the web2py directory is included in our zip file - recursive_zip( - zipf, path) # leave the first folder as None, as path is root. - zipf.close() - shutil.rmtree('zip_temp') - print "Your Windows binary version of web2py can be found in " + \ - zip_filename + ".zip" - print "You may extract the archive anywhere and then run web2py/web2py.exe" - -#should py2exe build files be removed? -if remove_build_files: - shutil.rmtree('build') - shutil.rmtree('deposit') - shutil.rmtree('dist') - print "py2exe build files removed" - -#final info -if not make_zip and not remove_build_files: - print "Your Windows binary & associated files can also be found in /dist" - -print "Finished!" -print "Enjoy web2py " + web2py_version_line From f9db6a8306c92dca452f8be502dc5f5e5b64dd27 Mon Sep 17 00:00:00 2001 From: Nico Zanferrari Date: Fri, 22 Mar 2019 15:33:29 +0100 Subject: [PATCH 003/111] cleanup --- extras/build_web2py/setup_exe.conf | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 extras/build_web2py/setup_exe.conf diff --git a/extras/build_web2py/setup_exe.conf b/extras/build_web2py/setup_exe.conf deleted file mode 100644 index 9996db06..00000000 --- a/extras/build_web2py/setup_exe.conf +++ /dev/null @@ -1,27 +0,0 @@ -[Setup] -#py2exe often includes DLLS from windows which aren't licensed for -#open source distribution. Should they be removed? -remove_microsoft_dlls: Yes - -#copy all web2py apps currently installed? -#If no, only the default admin, welcome & example apps will be included -copy_apps: No - -#include the web2py\site-packages directory? -copy_site_packages: Yes - -#include the web2py\scripts directory? -copy_scripts: Yes - -#create a zip file of the build for easy distribution? -make_zip: Yes - -#what should the zip file be named? (leave off the .zip extension) -zip_filename = web2py_win - -#should the build, deposit & dist directories used by py2exe be removed? -#if you created a zip file you likely don't need these directories anymore -remove_build_files = Yes - -#should the build include the gevented webserver (needs gevent) -include_gevent = Yes \ No newline at end of file From 7e30be377d3744cb95f4d3dfe2e7d6c688dcc556 Mon Sep 17 00:00:00 2001 From: Nico Zanferrari Date: Fri, 22 Mar 2019 15:33:44 +0100 Subject: [PATCH 004/111] cleanup --- extras/build_web2py/setup_exe.py | 232 ------------------------------- 1 file changed, 232 deletions(-) delete mode 100755 extras/build_web2py/setup_exe.py diff --git a/extras/build_web2py/setup_exe.py b/extras/build_web2py/setup_exe.py deleted file mode 100755 index ae0d00ee..00000000 --- a/extras/build_web2py/setup_exe.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -#Adapted from http://bazaar.launchpad.net/~flavour/sahana-eden/trunk/view/head:/static/scripts/tools/standalone_exe.py - -USAGE = """ -Usage: - Copy this and setup_exe.conf to web2py root folder - To build with py2exe: - Install py2exe: http://sourceforge.net/projects/py2exe/files/ - run python setup_exe.py py2exe - To build with bbfreeze: - Install bbfreeze: https://pypi.python.org/pypi/bbfreeze/ - run python setup_exe.py bbfreeze -""" - -from distutils.core import setup -from gluon.import_all import base_modules, contributed_modules -from gluon.fileutils import readlines_file -from glob import glob -import fnmatch -import os -import shutil -import sys -import re -import zipfile - -if len(sys.argv) != 2 or not os.path.isfile('web2py.py'): - print USAGE - sys.exit(1) -BUILD_MODE = sys.argv[1] -if not BUILD_MODE in ('py2exe', 'bbfreeze'): - print USAGE - sys.exit(1) - -def unzip(source_filename, dest_dir): - with zipfile.ZipFile(source_filename) as zf: - zf.extractall(dest_dir) - -#borrowed from http://bytes.com/topic/python/answers/851018-how-zip-directory-python-using-zipfile -def recursive_zip(zipf, directory, folder=""): - for item in os.listdir(directory): - if os.path.isfile(os.path.join(directory, item)): - zipf.write(os.path.join(directory, item), folder + os.sep + item) - elif os.path.isdir(os.path.join(directory, item)): - recursive_zip( - zipf, os.path.join(directory, item), folder + os.sep + item) - - -#read web2py version from VERSION file -web2py_version_line = readlines_file('VERSION')[0] -#use regular expression to get just the version number -v_re = re.compile('[0-9]+\.[0-9]+\.[0-9]+') -web2py_version = v_re.search(web2py_version_line).group(0) - -#pull in preferences from config file -import ConfigParser -Config = ConfigParser.ConfigParser() -Config.read('setup_exe.conf') -remove_msft_dlls = Config.getboolean("Setup", "remove_microsoft_dlls") -copy_apps = Config.getboolean("Setup", "copy_apps") -copy_site_packages = Config.getboolean("Setup", "copy_site_packages") -copy_scripts = Config.getboolean("Setup", "copy_scripts") -make_zip = Config.getboolean("Setup", "make_zip") -zip_filename = Config.get("Setup", "zip_filename") -remove_build_files = Config.getboolean("Setup", "remove_build_files") -include_gevent = Config.getboolean("Setup", "include_gevent") - -# Python base version -python_version = sys.version_info[:3] - - - -if BUILD_MODE == 'py2exe': - import py2exe - - setup( - console=[{'script':'web2py.py', - 'icon_resources': [(0, 'extras/icons/web2py.ico')] - }], - windows=[{'script':'web2py.py', - 'icon_resources': [(1, 'extras/icons/web2py.ico')], - 'dest_base':'web2py_no_console' # MUST NOT be just 'web2py' otherwise it overrides the standard web2py.exe - }], - name="web2py", - version=web2py_version, - description="web2py web framework", - author="Massimo DiPierro", - license="LGPL v3", - data_files=[ - 'ABOUT', - 'LICENSE', - 'VERSION' - ], - options={'py2exe': { - 'packages': contributed_modules, - 'includes': base_modules, - }}, - ) - #py2exe packages lots of duplicates in the library.zip, let's save some space - library_temp_dir = os.path.join('dist', 'library_temp') - library_zip_archive = os.path.join('dist', 'library.zip') - os.makedirs(library_temp_dir) - unzip(library_zip_archive, library_temp_dir) - os.unlink(library_zip_archive) - zipl = zipfile.ZipFile(library_zip_archive, "w", compression=zipfile.ZIP_DEFLATED) - recursive_zip(zipl, library_temp_dir) - zipl.close() - shutil.rmtree(library_temp_dir) - print "web2py binary successfully built" - -elif BUILD_MODE == 'bbfreeze': - modules = base_modules + contributed_modules - from bbfreeze import Freezer - f = Freezer(distdir="dist", includes=(modules)) - f.addScript("web2py.py") - #to make executable without GUI we need this trick - shutil.copy("web2py.py", "web2py_no_console.py") - f.addScript("web2py_no_console.py", gui_only=True) - if include_gevent: - #fetch the gevented webserver script and copy to root - gevented_webserver = os.path.join("handlers", "web2py_on_gevent.py") - shutil.copy(gevented_webserver, "web2py_on_gevent.py") - f.addScript("web2py_on_gevent.py") - f.setIcon('extras/icons/web2py.ico') - f() # starts the freezing process - os.unlink("web2py_no_console.py") - if include_gevent: - os.unlink("web2py_on_gevent.py") - #add data_files - for req in ['ABOUT', 'LICENSE', 'VERSION']: - shutil.copy(req, os.path.join('dist', req)) - print "web2py binary successfully built" - -try: - os.unlink('storage.sqlite') -except: - pass - -#This need to happen after bbfreeze is run because Freezer() deletes distdir before starting! -if python_version > (2,5): - # Python26 compatibility: http://www.py2exe.org/index.cgi/Tutorial#Step52 - try: - shutil.copytree('C:\Bin\Microsoft.VC90.CRT', 'dist/Microsoft.VC90.CRT/') - except: - print "You MUST copy Microsoft.VC90.CRT folder into the archive" - -def copy_folders(source, destination): - """Copy files & folders from source to destination (within dist/)""" - if os.path.exists(os.path.join('dist', destination)): - shutil.rmtree(os.path.join('dist', destination)) - shutil.copytree(os.path.join(source), os.path.join('dist', destination)) - -#should we remove Windows OS dlls user is unlikely to be able to distribute -if remove_msft_dlls: - print "Deleted Microsoft files not licensed for open source distribution" - print "You are still responsible for making sure you have the rights to distribute any other included files!" - #delete the API-MS-Win-Core DLLs - for f in glob('dist/API-MS-Win-*.dll'): - os.unlink(f) - #then delete some other files belonging to Microsoft - other_ms_files = ['KERNELBASE.dll', 'MPR.dll', 'MSWSOCK.dll', - 'POWRPROF.dll'] - for f in other_ms_files: - try: - os.unlink(os.path.join('dist', f)) - except: - print "unable to delete dist/" + f - -#Should we include applications? -if copy_apps: - copy_folders('applications', 'applications') - print "Your application(s) have been added" -else: - #only copy web2py's default applications - copy_folders('applications/admin', 'applications/admin') - copy_folders('applications/welcome', 'applications/welcome') - copy_folders('applications/examples', 'applications/examples') - print "Only web2py's admin, examples & welcome applications have been added" - -copy_folders('extras', 'extras') -copy_folders('examples', 'examples') -copy_folders('handlers', 'handlers') - - -#should we copy project's site-packages into dist/site-packages -if copy_site_packages: - #copy site-packages - copy_folders('site-packages', 'site-packages') -else: - #no worries, web2py will create the (empty) folder first run - print "Skipping site-packages" - -#should we copy project's scripts into dist/scripts -if copy_scripts: - #copy scripts - copy_folders('scripts', 'scripts') -else: - #no worries, web2py will create the (empty) folder first run - print "Skipping scripts" - -#should we create a zip file of the build? -if make_zip: - #create a web2py folder & copy dist's files into it - shutil.copytree('dist', 'zip_temp/web2py') - #create zip file - zipf = zipfile.ZipFile(zip_filename + ".zip", - "w", compression=zipfile.ZIP_DEFLATED) - # just temp so the web2py directory is included in our zip file - path = 'zip_temp' - # leave the first folder as None, as path is root. - recursive_zip(zipf, path) - zipf.close() - shutil.rmtree('zip_temp') - print "Your Windows binary version of web2py can be found in " + \ - zip_filename + ".zip" - print "You may extract the archive anywhere and then run web2py/web2py.exe" - -#should py2exe build files be removed? -if remove_build_files: - if BUILD_MODE == 'py2exe': - shutil.rmtree('build') - shutil.rmtree('deposit') - shutil.rmtree('dist') - print "build files removed" - -#final info -if not make_zip and not remove_build_files: - print "Your Windows binary & associated files can also be found in /dist" - -print "Finished!" -print "Enjoy web2py " + web2py_version_line From 7a0e113a5f6910100fce8ccf80ca0dae99daf110 Mon Sep 17 00:00:00 2001 From: Nico Zanferrari Date: Fri, 22 Mar 2019 15:34:50 +0100 Subject: [PATCH 005/111] new PyInstaller script --- extras/build_web2py/build_web2py.py | 145 ++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 extras/build_web2py/build_web2py.py diff --git a/extras/build_web2py/build_web2py.py b/extras/build_web2py/build_web2py.py new file mode 100644 index 00000000..1734ab2c --- /dev/null +++ b/extras/build_web2py/build_web2py.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# up to 2019, we have used py2applet, py2exe and bbfreeze for building web2py binaries +# The original scripts can be found on GitHub for web2py up to version 2.18.4 +# See also Niphlod's work on http://www.web2pyslices.com/slice/show/1726/build-windows-binaries +# Then we switched to Pyinstaller in order to fully support Python 3 + +from distutils.core import setup +from gluon.import_all import base_modules, contributed_modules +from gluon.fileutils import readlines_file +from glob import glob +import os +import shutil +import sys +import re +import zipfile +import subprocess +import platform + + +USAGE = """ +build_web2py - make web2py Windows and MacOS binaries with pyinstaller + +Usage: + Install pyinstaller program, copy this file to web2py root folder and run: + + python build_py3.py + +""" + +if len(sys.argv) != 1 or not os.path.isfile('web2py.py'): + print(USAGE) + sys.exit(1) +os_version = platform.system() +if os_version not in ('Windows', 'Darwin'): + print('Unsupported system: %s' % os_version) + sys.exit(1) + + +def unzip(source_filename, dest_dir): + with zipfile.ZipFile(source_filename) as zf: + zf.extractall(dest_dir) + +# borrowed from http://bytes.com/topic/python/answers/851018-how-zip-directory-python-using-zipfile + + +def recursive_zip(zipf, directory, folder=""): + for item in os.listdir(directory): + if os.path.isfile(os.path.join(directory, item)): + zipf.write(os.path.join(directory, item), folder + os.sep + item) + elif os.path.isdir(os.path.join(directory, item)): + recursive_zip( + zipf, os.path.join(directory, item), folder + os.sep + item) + + +# read web2py version from VERSION file +web2py_version_line = readlines_file('VERSION')[0] +# use regular expression to get just the version number +v_re = re.compile('[0-9]+\.[0-9]+\.[0-9]+') +web2py_version = v_re.search(web2py_version_line).group(0) + +# Python base version +python_version = sys.version_info[:3] + + +if os_version == 'Windows': + print("\nBuilding binary web2py for Windows\n") + # to make executable without GUI we need this trick + shutil.copy("web2py.py", "web2py_no_console.py") + + subprocess.run('pyinstaller --clean --icon=extras/icons/web2py.ico --hidden-import=gluon.packages.dal.pydal \ + --hidden-import=gluon.packages.yatl.yatl --hidden-import=site-packages web2py.py') + subprocess.run('pyinstaller -w --clean --icon=extras/icons/web2py.ico --hidden-import=gluon.packages.dal.pydal \ + --hidden-import=gluon.packages.yatl.yatl --hidden-import=site-packages web2py_no_console.py') + + # cleanup + move binary files to dist folder + os.unlink('web2py_no_console.py') + os.unlink('web2py_no_console.spec') + source = 'dist/web2py/' + for files in os.listdir(source): + shutil.move(os.path.join(source, files), 'dist') + source2 = 'dist/web2py_no_console/' + files = 'web2py_no_console.exe' + shutil.move(os.path.join(source2, files), 'dist') + shutil.rmtree(source) + shutil.rmtree(source2) + os.unlink('dist/web2py.exe.manifest') + + zip_filename = 'web2py_win' + bin_folder = 'dist' + + +elif os_version == 'Darwin': + print("\nBuilding binary web2py for MacOS\n") + + import subprocess + subprocess.call("pyinstaller --clean --windowed --icon=extras/icons/web2py.icns --hidden-import=gluon.packages.dal.pydal --hidden-import=gluon.packages.yatl.yatl \ + --hidden-import=site-packages --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' \ + --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' web2py.py", shell=True) + + # cleanup + move binary files to dist folder + shutil.rmtree(os.path.join('dist', 'web2py')) + shutil.rmtree('build') + + zip_filename = 'web2py_osx' + bin_folder = (os.path.join('dist', 'web2py.app/Contents/MacOS')) + +print("\nWeb2py binary successfully built!\n") + + +# add data_files +for req in ['CHANGELOG', 'LICENSE', 'VERSION']: + shutil.copy(req, os.path.join(bin_folder, req)) +# cleanup unuseful binary cache +for dirpath, dirnames, files in os.walk('.'): + if dirpath.endswith('__pycache__'): + print('Deleting cached binary directory : %s' % dirpath) + shutil.rmtree(dirpath) + +print("\nPreparing package ...") +# misc +for folders in ['gluon', 'extras', 'site-packages', 'scripts', 'applications', 'examples', 'handlers']: + shutil.copytree(folders, os.path.join(bin_folder, folders)) +os.mkdir(os.path.join(bin_folder, 'logs')) +os.unlink('web2py.spec') + + +# create a web2py folder & copy dist's files into it +shutil.copytree('dist', 'zip_temp/web2py') +# create zip file +zipf = zipfile.ZipFile(zip_filename + ".zip", + "w", compression=zipfile.ZIP_DEFLATED) +# just temp so the web2py directory is included in our zip file +path = 'zip_temp' +# leave the first folder as None, as path is root. +recursive_zip(zipf, path) +zipf.close() +shutil.rmtree('zip_temp') +shutil.rmtree('dist') + +print("Your binary version of web2py can be found in " + \ + zip_filename + ".zip") +print("You may extract the archive anywhere and then run web2py without worrying about dependency") +print("\nEnjoy binary web2py " + web2py_version_line + "\n with embedded Python " + sys.version + "\n") From ca7b676591c1203d135a5f5af39a6920b23b4725 Mon Sep 17 00:00:00 2001 From: Nico Zanferrari Date: Fri, 22 Mar 2019 15:37:14 +0100 Subject: [PATCH 006/111] Update README --- extras/build_web2py/README | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extras/build_web2py/README b/extras/build_web2py/README index 0fc35b14..d060dbfd 100644 --- a/extras/build_web2py/README +++ b/extras/build_web2py/README @@ -1,2 +1,4 @@ +# build-web2py + The files in this folder must be run from the main web2py folder. -They are for building windows and osx binary distribution and not meant for the end user. \ No newline at end of file +They are for building windows and osx binary distribution using PyInstaller and not meant for the end user. From c03c962778d77ad80377b2e2cf326593cdc644fc Mon Sep 17 00:00:00 2001 From: Nico Zanferrari Date: Fri, 22 Mar 2019 15:37:44 +0100 Subject: [PATCH 007/111] Rename README to README.md --- extras/build_web2py/{README => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename extras/build_web2py/{README => README.md} (100%) diff --git a/extras/build_web2py/README b/extras/build_web2py/README.md similarity index 100% rename from extras/build_web2py/README rename to extras/build_web2py/README.md From eb07384c23f497878ac974df239ef815f9cf9493 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 23 Mar 2019 21:42:06 -0700 Subject: [PATCH 008/111] moved pluralize logic into sqlhtml for speed reasons, thanks Paolo P. --- gluon/packages/dal | 2 +- gluon/sqlhtml.py | 68 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index c0c8d29b..49ddb26b 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit c0c8d29ba3fc5b5cfd5635acbeabe62475040442 +Subproject commit 49ddb26bf00b8a10eb20413d359155dc5d54d4af diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 774430e1..9423c2bd 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -47,9 +47,8 @@ try: except ImportError: settings = {} -widget_class = re.compile('^\w*') -REGEX_ALIAS_MATCH = re.compile('^(.*) AS (.*)$') +REGEX_WIDGET_CLASS = re.compile(r'^\w*') def add_class(a, b): @@ -143,6 +142,34 @@ def show_if(cond): raise RuntimeError("Not Implemented Error") +PLURALIZE_RULES = None + +def pluralize(singular, rules=None): + if rules is None: + global PLURALIZE_RULES + if PLURALIZE_RULES is None: + PLURALIZE_RULES = [ + (re.compile('child$'), re.compile('child$'), 'children'), + (re.compile('oot$'), re.compile('oot$'), 'eet'), + (re.compile('ooth$'), re.compile('ooth$'), 'eeth'), + (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'), + (re.compile('sis$'), re.compile('sis$'), 'ses'), + (re.compile('man$'), re.compile('man$'), 'men'), + (re.compile('ife$'), re.compile('ife$'), 'ives'), + (re.compile('eau$'), re.compile('eau$'), 'eaux'), + (re.compile('lf$'), re.compile('lf$'), 'lves'), + (re.compile('[sxz]$'), re.compile('$'), 'es'), + (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'), + (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'), + (re.compile('$'), re.compile('$'), 's'), + ] + rules = PLURALIZE_RULES + for line in rules: + re_search, re_sub, replace = line + plural = re_search.search(singular) and re_sub.sub(replace, singular) + if plural: return plural + + class FormWidget(object): """ Helper for SQLFORM to generate form input fields (widget), related to the @@ -165,7 +192,7 @@ class FormWidget(object): attr = dict( _id='%s_%s' % (field.tablename, field.name), _class=cls._class or - widget_class.match(str(field.type)).group(), + REGEX_WIDGET_CLASS.match(str(field.type)).group(), _name=field.name, requires=field.requires, ) @@ -1541,7 +1568,7 @@ class SQLFORM(FORM): # SQLCustomType has a widget, use it inp = field.type.widget(field, default) else: - field_type = widget_class.match(str(field.type)).group() + field_type = REGEX_WIDGET_CLASS.match(str(field.type)).group() field_type = field_type in self.widgets and field_type or 'string' inp = self.widgets[field_type].widget(field, default) @@ -3180,7 +3207,8 @@ class SQLFORM(FORM): return dict(form=form) """ - request, T = current.request, current.T + request = current.request + T = current.T if args is None: args = [] @@ -3198,8 +3226,7 @@ class SQLFORM(FORM): links = {} if constraints is None: constraints = {} - field = None - name = None + field = name = None def format(table, row): if not row: @@ -3210,10 +3237,14 @@ class SQLFORM(FORM): return table._format(row) else: return '#' + str(row.id) + + def plural(table): + return table._plural or pluralize(table._singular.lower()).capitalize() + try: nargs = len(args) + 1 - previous_tablename, previous_fieldname, previous_id = \ - table._tablename, None, None + previous_tablename = table._tablename + previous_fieldname = previous_id = None while len(request.args) > nargs: key = request.args(nargs) if '.' in key: @@ -3234,11 +3265,12 @@ class SQLFORM(FORM): if previous_id: if record[previous_fieldname] != int(previous_id): raise HTTP(400) - previous_tablename, previous_fieldname, previous_id = \ - tablename, fieldname, id + previous_tablename = tablename + previous_fieldname = fieldname + previous_id = id name = format(db[referee], record) breadcrumbs.append( - LI(A(T(db[referee]._plural), + LI(A(T(plural(db[referee])), cid=request.cid, _href=url()), SPAN(divider, _class='divider'), @@ -3319,8 +3351,8 @@ class SQLFORM(FORM): if tb: multiple_links = len(linked_fieldnames) > 1 for fieldname in linked_fieldnames: - t = T(tb._plural) if not multiple_links else \ - T(tb._plural + '(' + fieldname + ')') + t = T(plural(tb)) if not multiple_links else \ + T("%s(%s)" % (plural(tb), fieldname)) args0 = tablename + '.' + fieldname linked.append( lambda row, t=t, nargs=nargs, args0=args0: @@ -3332,7 +3364,7 @@ class SQLFORM(FORM): user_signature=user_signature, **kwargs) if isinstance(grid, DIV): - header = table._plural + header = plural(table) next = grid.create_form or grid.update_form or grid.view_form breadcrumbs.append(LI( A(T(header), cid=request.cid, _href=url()), @@ -3397,6 +3429,8 @@ class SQLTABLE(TABLE): """ + REGEX_ALIAS_MATCH = '^(.*) AS (.*)$' + def __init__(self, sqlrows, linkto=None, @@ -3450,7 +3484,7 @@ class SQLTABLE(TABLE): if isinstance(f, field_types): headers[c] = make_name(f) else: - headers[c] = REGEX_ALIAS_MATCH.sub(r'\2', c) + headers[c] = re.sub(self.REGEX_ALIAS_MATCH, r'\2', c) if colgroup: cols = [COL(_id=c.replace('.', '-'), data={'column': i + 1}) for i, c in enumerate(columns)] @@ -3476,7 +3510,7 @@ class SQLTABLE(TABLE): row.append(TH(A(headers.get(c, c), _href=th_link + '?orderby=' + c, cid=cid))) else: - row.append(TH(headers.get(c, REGEX_ALIAS_MATCH.sub(r'\2', c)))) + row.append(TH(headers.get(c, re.sub(self.REGEX_ALIAS_MATCH, r'\2', c)))) if extracolumns: # new implement dict for c in extracolumns: From 83ca7f20b8d7a2e55c1db783b9a94d04c19aef81 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 23 Mar 2019 21:53:30 -0700 Subject: [PATCH 009/111] no need for Field import, thanks Paolo --- gluon/globals.py | 5 ++--- gluon/packages/dal | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gluon/globals.py b/gluon/globals.py index 0b8f203a..44c9b794 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -836,7 +836,6 @@ class Session(Storage): compression_level(int): 0-9, sets zlib compression on the data before the encryption """ - from gluon.dal import Field request = request or current.request response = response or current.response masterapp = masterapp or request.application @@ -927,7 +926,7 @@ class Session(Storage): elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) - # if had a session on file alreday, close it (yes, can happen) + # if had a session on file already, close it (yes, can happen) if response.session_file: self._close(response) # if on GAE tickets go also in DB @@ -939,7 +938,7 @@ class Session(Storage): table_migrate = False tname = tablename + '_' + masterapp table = db.get(tname, None) - # Field = db.Field + Field = db.Field if table is None: db.define_table( tname, diff --git a/gluon/packages/dal b/gluon/packages/dal index 49ddb26b..540d9c05 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 49ddb26bf00b8a10eb20413d359155dc5d54d4af +Subproject commit 540d9c05dd07946e0d17e310805286293b9c1b09 From ce2ad2d15beb1aaf79a19d9f38c8e70d811785ad Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 23 Mar 2019 22:13:13 -0700 Subject: [PATCH 010/111] syncing --- gluon/packages/dal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index 540d9c05..1f13ac59 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 540d9c05dd07946e0d17e310805286293b9c1b09 +Subproject commit 1f13ac59bbe663fca691c8525e1f6de3baac18e0 From 1a828bf630b584af954e0e7e4e960161ee8ccc64 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 23 Mar 2019 22:27:35 -0700 Subject: [PATCH 011/111] syncing --- gluon/packages/dal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index 1f13ac59..24ea3602 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 1f13ac59bbe663fca691c8525e1f6de3baac18e0 +Subproject commit 24ea3602ba75b1a1692fd0468f837bb2c81ad920 From 12e043c0a29c57455250e108f3ffca10dd2b5b57 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 24 Mar 2019 10:48:58 -0700 Subject: [PATCH 012/111] removed un-necessary code from shell, thanks Paolo Pastori --- gluon/shell.py | 78 ++------------------------------------------------ 1 file changed, 2 insertions(+), 76 deletions(-) diff --git a/gluon/shell.py b/gluon/shell.py index bde0ea51..225ac4e6 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -20,7 +19,6 @@ import copy import logging import types import re -import optparse import glob import traceback import gluon.fileutils as fileutils @@ -278,7 +276,7 @@ def run( if import_models: BaseAdapter.close_all_instances('commit') - except Exception as e: + except: print(traceback.format_exc()) if import_models: BaseAdapter.close_all_instances('rollback') @@ -287,7 +285,7 @@ def run( exec(python_code, _env) if import_models: BaseAdapter.close_all_instances('commit') - except Exception as e: + except: print(traceback.format_exc()) if import_models: BaseAdapter.close_all_instances('rollback') @@ -432,75 +430,3 @@ def test(testpath, import_models=True, verbose=False): for (name, obj) in globs.items(): if name not in ignores and (f is None or f == name): doctest_object(name, obj) - - -def get_usage(): - usage = """ - %prog [options] pythonfile -""" - return usage - - -def execute_from_command_line(argv=None): - if argv is None: - argv = sys.argv - - parser = optparse.OptionParser(usage=get_usage()) - - parser.add_option('-S', '--shell', dest='shell', metavar='APPNAME', - help='run web2py in interactive shell ' + - 'or IPython(if installed) with specified appname') - msg = 'run web2py in interactive shell or bpython (if installed) with' - msg += ' specified appname (if app does not exist it will be created).' - msg += '\n Use combined with --shell' - parser.add_option( - '-B', - '--bpython', - action='store_true', - default=False, - dest='bpython', - help=msg, - ) - parser.add_option( - '-P', - '--plain', - action='store_true', - default=False, - dest='plain', - help='only use plain python shell, should be used with --shell option', - ) - parser.add_option( - '-M', - '--import_models', - action='store_true', - default=False, - dest='import_models', - help='auto import model files, default is False, ' + - ' should be used with --shell option', - ) - parser.add_option( - '-R', - '--run', - dest='run', - metavar='PYTHON_FILE', - default='', - help='run PYTHON_FILE in web2py environment, ' + - 'should be used with --shell option', - ) - - (options, args) = parser.parse_args(argv[1:]) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(0) - - if len(args) > 0: - startfile = args[0] - else: - startfile = '' - run(options.shell, options.plain, startfile=startfile, - bpython=options.bpython) - - -if __name__ == '__main__': - execute_from_command_line() From 565415d4bfb20a5127643115152e402255a4117b Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 24 Mar 2019 10:49:35 -0700 Subject: [PATCH 013/111] code simplificatons in newcron (although deprecated), thanks Paolo Pastori --- gluon/newcron.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/gluon/newcron.py b/gluon/newcron.py index c5a0119f..320af201 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -279,8 +278,7 @@ class cronlauncher(threading.Thread): def crondance(applications_parent, ctype='soft', startup=False, apps=None): apppath = os.path.join(applications_parent, 'applications') - cron_path = os.path.join(applications_parent) - token = Token(cron_path) + token = Token(applications_parent) cronmaster = token.acquire(startup=startup) if not cronmaster: return @@ -314,9 +312,8 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): if not os.path.exists(crontab): continue try: - cronlines = fileutils.readlines_file(crontab, 'rt') - lines = [x.strip() for x in cronlines if x.strip( - ) and not x.strip().startswith('#')] + cronlines = [line.strip() for line in fileutils.readlines_file(crontab, 'rt')] + lines = [line for line in cronlines if line and not line.startswith('#')] tasks = [parsecronline(cline) for cline in lines] except Exception as e: logger.error('WEB2PY CRON: crontab read error %s' % e) From b4e22bf4651903c1203fc25888b47bb17e17501f Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 24 Mar 2019 10:50:02 -0700 Subject: [PATCH 014/111] code simplificaton in widget.py, thanks Paolo Pastori --- gluon/widget.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/gluon/widget.py b/gluon/widget.py index 65b37c1d..68ae84b2 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -808,7 +807,7 @@ def console(): '--run', dest='run', metavar='PYTHON_FILE', - default='', + default='', # NOTE: used for sys.argv[0] if --shell help=msg) msg = ('run scheduled tasks for the specified apps: expects a list of ' @@ -945,7 +944,7 @@ def console(): k = len(sys.argv) sys.argv, other_args = sys.argv[:k], sys.argv[k + 1:] (options, args) = parser.parse_args() - options.args = [options.run] + other_args + options.args = other_args copy_options = copy.deepcopy(options) copy_options.password = '******' @@ -1147,8 +1146,7 @@ def start(cron=True): if options.shell: if options.folder: os.chdir(options.folder) - if not options.args is None: - sys.argv[:] = options.args + sys.argv = [options.run] + options.args run(options.shell, plain=options.plain, bpython=options.bpython, import_models=options.import_models, startfile=options.run, cronjob=options.cronjob) From 8c090954fd64b27aad35596d841c2e53a8551957 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 24 Mar 2019 11:28:28 -0700 Subject: [PATCH 015/111] new custom import works better with recursive imports, thanks Paolo Pastori --- gluon/custom_import.py | 122 ++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 76 deletions(-) diff --git a/gluon/custom_import.py b/gluon/custom_import.py index e12dea42..0eeb5d01 100644 --- a/gluon/custom_import.py +++ b/gluon/custom_import.py @@ -8,7 +8,7 @@ Support for smart import syntax for web2py applications ------------------------------------------------------- """ -from gluon._compat import builtin, unicodeT, PY2, to_native, reload +from gluon._compat import builtin, unicodeT, to_native, reload import os import sys import threading @@ -19,7 +19,6 @@ INVALID_MODULES = set(('', 'gluon', 'applications', 'custom_import')) # backward compatibility API - def custom_import_install(): if builtin.__import__ == NATIVE_IMPORTER: INVALID_MODULES.update(sys.modules.keys()) @@ -35,78 +34,56 @@ def is_tracking_changes(): return current.request._custom_import_track_changes -class CustomImportException(ImportError): - pass +# see https://docs.python.org/3/library/functions.html#__import__ +# Changed in Python 3.3: Negative values for level are no longer supported, +# which also changes the default value to 0 (was -1) +_DEFAULT_LEVEL = 0 if sys.version_info[:2] >= (3, 3) else -1 - -def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1): +def custom_importer(name, globals={}, locals=None, fromlist=(), level=_DEFAULT_LEVEL): """ web2py's custom importer. It behaves like the standard Python importer but it tries to transform import statements as something like "import applications.app_name.modules.x". - If the import fails, it falls back on naive_importer + If the import fails, it falls back on builtin importer. """ + # support for non-ascii name if isinstance(name, unicodeT): name = to_native(name) - globals = globals or {} - locals = locals or {} - fromlist = fromlist or [] - - try: + if hasattr(current, 'request') \ + and level <= 0 \ + and name.partition('.')[0] not in INVALID_MODULES: + # absolute import from application code + try: + return NATIVE_IMPORTER(name, globals, locals, fromlist, level) + except (ImportError, KeyError): + pass if current.request._custom_import_track_changes: base_importer = TRACK_IMPORTER else: base_importer = NATIVE_IMPORTER - except: # there is no current.request (should never happen) - base_importer = NATIVE_IMPORTER - - if not(PY2) and level < 0: - level = 0 - - # if not relative and not from applications: - if hasattr(current, 'request') \ - and level <= 0 \ - and not name.partition('.')[0] in INVALID_MODULES \ - and isinstance(globals, dict): - import_tb = None - try: - try: - oname = name if not name.startswith('.') else '.'+name - return NATIVE_IMPORTER(oname, globals, locals, fromlist, level) - except (ImportError, KeyError): - items = current.request.folder.split(os.path.sep) - if not items[-1]: - items = items[:-1] - modules_prefix = '.'.join(items[-2:]) + '.modules' - if not fromlist: - # import like "import x" or "import x.y" - result = None - for itemname in name.split("."): - new_mod = base_importer( - modules_prefix, globals, locals, [itemname], level) - try: - result = result or sys.modules[modules_prefix+'.'+itemname] - except KeyError as e: - raise ImportError('Cannot import module %s' % str(e)) - modules_prefix += "." + itemname - return result - else: - # import like "from x import a, b, ..." - pname = modules_prefix + "." + name - return base_importer(pname, globals, locals, fromlist, level) - except ImportError as e1: - import_tb = sys.exc_info()[2] - try: - return NATIVE_IMPORTER(name, globals, locals, fromlist, level) - except (ImportError, KeyError) as e3: - raise ImportError(e1, import_tb) # there an import error in the module - except Exception as e2: - raise # there is an error in the module - finally: - if import_tb: - import_tb = None + items = current.request.folder.split(os.path.sep) + # FIXME: why does request.folder endswith(os.path.sep) ? + if not items[-1]: items.pop() + modules_prefix = '.'.join(items[-2:]) + '.modules' + if not fromlist: + # "import x" or "import x.y" + result = None + for itemname in name.split("."): + new_mod = base_importer( + modules_prefix, globals, locals, (itemname,), level) + modules_prefix += "." + itemname + if result is None: + try: + result = sys.modules[modules_prefix] + except KeyError as e: + raise ImportError("No module named %s" % e) + return result + else: + # "from x import a, b, ..." + pname = "%s.%s" % (modules_prefix, name) + return base_importer(pname, globals, locals, fromlist, level) return NATIVE_IMPORTER(name, globals, locals, fromlist, level) @@ -123,30 +100,23 @@ class TrackImporter(object): def __init__(self): self._import_dates = {} # Import dates of the files of the modules - def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1): + def __call__(self, name, globals={}, locals=None, fromlist=(), level=_DEFAULT_LEVEL): """ The import method itself. """ - globals = globals or {} - locals = locals or {} - fromlist = fromlist or [] - try: - # Check the date and reload if needed: - self._update_dates(name, globals, locals, fromlist, level) - # Try to load the module and update the dates if it works: - result = NATIVE_IMPORTER(name, globals, locals, fromlist, level) - # Module maybe loaded for the 1st time so we need to set the date - self._update_dates(name, globals, locals, fromlist, level) - return result - except Exception as e: - raise # Don't hide something that went wrong + # Check the date and reload if needed: + self._update_dates(name, globals, locals, fromlist, level) + # Try to load the module and update the dates if it works: + result = NATIVE_IMPORTER(name, globals, locals, fromlist, level) + # Module maybe loaded for the 1st time so we need to set the date + self._update_dates(name, globals, locals, fromlist, level) + return result def _update_dates(self, name, globals, locals, fromlist, level): """ Update all the dates associated to the statement import. A single import statement may import many modules. """ - self._reload_check(name, globals, locals, level) for fromlist_name in fromlist or []: pname = "%s.%s" % (name, fromlist_name) @@ -169,7 +139,7 @@ class TrackImporter(object): except: self._import_dates.pop(file, None) # Clean up # Handle module changing in package and - #package changing in module: + # package changing in module: if file.endswith(".py"): # Get path without file ext: file = os.path.splitext(file)[0] From 44b93929e2b7d2c03bf94d46a115ae5b07000c12 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 30 Mar 2019 10:34:49 -0700 Subject: [PATCH 016/111] better regex, thanks Paolo --- gluon/newcron.py | 3 +-- gluon/packages/dal | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gluon/newcron.py b/gluon/newcron.py index 320af201..4affed93 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -190,8 +190,7 @@ def rangetolist(s, period='min'): s = s.replace('*', '1-12', 1) elif period == 'dow': s = s.replace('*', '0-6', 1) - m = re.compile(r'(\d+)-(\d+)/(\d+)') - match = m.match(s) + match = re.match(r'(\d+)-(\d+)/(\d+)', s) if match: for i in range(int(match.group(1)), int(match.group(2)) + 1): if i % int(match.group(3)) == 0: diff --git a/gluon/packages/dal b/gluon/packages/dal index 24ea3602..7d9adad3 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 24ea3602ba75b1a1692fd0468f837bb2c81ad920 +Subproject commit 7d9adad364603bcf2368c4695b46924dbce95d48 From cfdee6e065050130d7c34523e7587bfd5f16f1c7 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 30 Mar 2019 10:42:01 -0700 Subject: [PATCH 017/111] better logic in running system tests, thanks Paolo --- gluon/packages/dal | 2 +- gluon/widget.py | 45 +++++++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index 7d9adad3..49c5ec28 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 7d9adad364603bcf2368c4695b46924dbce95d48 +Subproject commit 49c5ec284d2af7593f8e3f62f0cd95b043b3ccee diff --git a/gluon/widget.py b/gluon/widget.py index 68ae84b2..b1aa33d6 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -54,33 +54,38 @@ def run_system_tests(options): """ Runs unittests for gluon.tests """ - import subprocess - major_version = sys.version_info[0] - call_args = [sys.executable, '-m', 'unittest', '-v', 'gluon.tests'] - if major_version == 2: - sys.stderr.write("Python 2.7\n") - else: - sys.stderr.write("Experimental Python 3.x.\n") + # see "python -m unittest -h" for unittest options help + # NOTE: someone might be interested either in using the + # -f (--failfast) option to stop testing on first failure, or + # in customizing the test selection, for example to run only + # 'gluon.tests.', 'gluon.tests..' (this + # could be shortened as 'gluon.tests.'), or even + # 'gluon.tests...' (or + # the shorter 'gluon.tests..') + call_args = ['-m', 'unittest', '-c', 'gluon.tests'] + if options.verbose: + call_args.insert(-1, '-v') if options.with_coverage: - has_coverage = False - coverage_exec = 'coverage2' if major_version == 2 else 'coverage3' try: import coverage - has_coverage = True except: - sys.stderr.write('Coverage was not installed, skipping\n') + sys.stderr.write('Coverage was not installed\n') + sys.exit(256) + if not PY2: + sys.stderr.write('Experimental ') + sys.stderr.write("Python %s\n" % sys.version) + if options.with_coverage: + coverage_exec = 'coverage2' if PY2 else 'coverage3' coverage_config_file = os.path.join('gluon', 'tests', 'coverage.ini') coverage_config = os.environ.setdefault("COVERAGE_PROCESS_START", coverage_config_file) - call_args = [coverage_exec, 'run', '--rcfile=%s' % - coverage_config, '-m', 'unittest', '-v', 'gluon.tests'] - if has_coverage: - ret = subprocess.call(call_args) - else: - ret = 256 + run_args = [coverage_exec, 'run', '--rcfile=%s' % coverage_config] + # replace the current process + os.execvpe(run_args[0], run_args + call_args, os.environ) else: - ret = subprocess.call(call_args) - sys.exit(ret and 1) + run_args = [sys.executable] + # replace the current process + os.execv(run_args[0], run_args + call_args) class IO(object): @@ -1012,7 +1017,7 @@ def console(): options.interfaces.append(tuple(interface)) # accepts --scheduler in the form - # "app:group1,group2,app2:group1" + # "app:group1:group2,app2:group1" scheduler = [] options.scheduler_groups = None if isinstance(options.scheduler, str): From 66d5faf78fe8c2044c6460039ecf450cbfc80ebf Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 30 Mar 2019 11:35:34 -0700 Subject: [PATCH 018/111] fixed sys.exit value, thanks Paolo --- gluon/widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/widget.py b/gluon/widget.py index b1aa33d6..2c1dde9c 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -70,7 +70,7 @@ def run_system_tests(options): import coverage except: sys.stderr.write('Coverage was not installed\n') - sys.exit(256) + sys.exit(1) if not PY2: sys.stderr.write('Experimental ') sys.stderr.write("Python %s\n" % sys.version) From 18da4fa7fdf96526139851105fb137a61f5fb9f7 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 31 Mar 2019 19:46:55 -0700 Subject: [PATCH 019/111] request.folder no longer ends with os.sep --- examples/app.example.yaml | 4 ++-- gluon/main.py | 2 +- gluon/packages/dal | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/app.example.yaml b/examples/app.example.yaml index b0fe2879..60a8add5 100644 --- a/examples/app.example.yaml +++ b/examples/app.example.yaml @@ -1,7 +1,7 @@ # For Google App Engine deployment, copy this file to app.yaml # and edit as required -# See http://code.google.com/appengine/docs/python/config/appconfig.html -# and http://web2py.com/book/default/chapter/11?search=app.yaml +# See https://cloud.google.com/appengine/docs/standard/python/config/appref +# and http://www.web2py.com/book/default/chapter/13#Deploying-on-Google-App-Engine application: yourappname version: 1 diff --git a/gluon/main.py b/gluon/main.py index b6c03fcd..aeb7f76e 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -357,7 +357,7 @@ def wsgibase(environ, responder): request.update( client=client, - folder=abspath('applications', app) + os.sep, + folder=abspath('applications', app), ajax=x_req_with == 'xmlhttprequest', cid=env.http_web2py_component_element, is_local=(env.remote_addr in local_hosts and client == env.remote_addr), diff --git a/gluon/packages/dal b/gluon/packages/dal index 49c5ec28..e973f27b 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 49c5ec284d2af7593f8e3f62f0cd95b043b3ccee +Subproject commit e973f27b69a8499e4e30e5663f3fba74bf87364c From e637b6b58ab3dfc6a9cc79ca8618c37a434ec31c Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 31 Mar 2019 19:50:02 -0700 Subject: [PATCH 020/111] removed un-needed IO class in widgets and som refactoring, thanks Paolo --- gluon/widget.py | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/gluon/widget.py b/gluon/widget.py index 2c1dde9c..238a5674 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -9,7 +9,6 @@ The widget is called from web2py ---------------------------------- """ -import datetime import sys from gluon._compat import StringIO, thread, xrange, PY2 import time @@ -23,7 +22,6 @@ import logging import getpass from gluon import main, newcron - from gluon.fileutils import read_file, write_file, create_welcome_w2p from gluon.settings import global_settings from gluon.shell import run, test @@ -35,17 +33,13 @@ if PY2: ProgramName = 'web2py Web Framework' ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str( - datetime.datetime.now().year) -ProgramVersion = read_file('VERSION').strip() + time.localtime().tm_year) +ProgramVersion = read_file('VERSION').rstrip() -ProgramInfo = '''%s - %s - %s''' % (ProgramName, ProgramAuthor, ProgramVersion) - -if sys.version_info < (2, 7) and (3, 0) < sys.version_info < (3, 5): - msg = 'Warning: web2py requires at least Python 2.7/3.5 but you are running:\n%s' - msg = msg % sys.version - sys.stderr.write(msg) +if sys.version_info < (2, 7) or (3, 0) < sys.version_info < (3, 5): + from platform import python_version + sys.stderr.write("Warning: web2py requires at least Python 2.7/3.5" + " but you are running %s\n" % python_version()) logger = logging.getLogger("web2py") @@ -88,24 +82,6 @@ def run_system_tests(options): os.execv(run_args[0], run_args + call_args) -class IO(object): - """ """ - - def __init__(self): - """ """ - - self.buffer = StringIO() - - def write(self, data): - """ """ - - sys.__stdout__.write(data) - if hasattr(self, 'callback'): - self.callback(data) - else: - self.buffer.write(data) - - def get_url(host, path='/', proto='http', port=80): if ':' in host: host = '[%s]' % host @@ -191,6 +167,9 @@ class web2pyDialog(object): command=item) # About + ProgramInfo = """%s + %s + %s""" % (ProgramName, ProgramAuthor, ProgramVersion) item = lambda: messagebox.showinfo('About web2py', ProgramInfo) helpmenu.add_command(label='About', command=item) From 39b965be7b5797332470813f2e62aa7fdcaa5b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leonel=20C=C3=A2mara?= Date: Mon, 1 Apr 2019 15:24:46 +0100 Subject: [PATCH 021/111] Revert "don't use a regex to find exposed functions" --- gluon/compileapp.py | 6 +++--- gluon/myregex.py | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 6a4f2f72..e7c2b7e8 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -23,6 +23,7 @@ from gluon.storage import Storage, List from gluon.template import parse_template from gluon.restricted import restricted, compile2 from gluon.fileutils import mktree, listdir, read_file, write_file +from gluon.myregex import regex_expose, regex_longcomments from gluon.languages import TranslatorFactory from gluon.dal import DAL, Field from gluon.validators import Validator @@ -44,7 +45,6 @@ from functools import reduce from gluon import rewrite from gluon.custom_import import custom_import_install import py_compile -import ast logger = logging.getLogger("web2py") @@ -517,8 +517,8 @@ def compile_models(folder): def find_exposed_functions(data): - parsed = ast.parse(data) - return [n.name for n in ast.walk(parsed) if type(n) == ast.FunctionDef and len(n.args.args) == 0 and n.args.kwarg is None] + data = regex_longcomments.sub('', data) + return regex_expose.findall(data) def compile_controllers(folder): diff --git a/gluon/myregex.py b/gluon/myregex.py index 0a015a39..5c3a7c99 100644 --- a/gluon/myregex.py +++ b/gluon/myregex.py @@ -18,7 +18,13 @@ regex_tables = re.compile( """^[\w]+\.define_table\(\s*[\'\"](?P\w+)[\'\"]""", flags=re.M) -# patterns to find includes and extends in views +# pattern to find exposed functions in controller + +regex_expose = re.compile( + '^def\s+(?P_?[a-zA-Z0-9]\w*)\( *\)\s*:', + flags=re.M) + +regex_longcomments = re.compile('(""".*?"""|'+"'''.*?''')", re.DOTALL) regex_include = re.compile( '(?P\{\{\s*include\s+[\'"](?P[^\'"]*)[\'"]\s*\}\})') From f2dcc53a1895d5faaee90e68bcad982f7bf76b65 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 6 Apr 2019 19:51:57 -0700 Subject: [PATCH 022/111] cleanup in custom import, thanks Paolo --- gluon/custom_import.py | 9 ++++----- gluon/packages/yatl | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gluon/custom_import.py b/gluon/custom_import.py index 0eeb5d01..e9c9f401 100644 --- a/gluon/custom_import.py +++ b/gluon/custom_import.py @@ -63,9 +63,8 @@ def custom_importer(name, globals={}, locals=None, fromlist=(), level=_DEFAULT_L base_importer = TRACK_IMPORTER else: base_importer = NATIVE_IMPORTER - items = current.request.folder.split(os.path.sep) - # FIXME: why does request.folder endswith(os.path.sep) ? - if not items[-1]: items.pop() + # rstrip for backward compatibility + items = current.request.folder.rstrip(os.sep).split(os.sep) modules_prefix = '.'.join(items[-2:]) + '.modules' if not fromlist: # "import x" or "import x.y" @@ -77,8 +76,8 @@ def custom_importer(name, globals={}, locals=None, fromlist=(), level=_DEFAULT_L if result is None: try: result = sys.modules[modules_prefix] - except KeyError as e: - raise ImportError("No module named %s" % e) + except KeyError: + raise ImportError("No module named %s" % modules_prefix) return result else: # "from x import a, b, ..." diff --git a/gluon/packages/yatl b/gluon/packages/yatl index 7e905158..694f630f 160000 --- a/gluon/packages/yatl +++ b/gluon/packages/yatl @@ -1 +1 @@ -Subproject commit 7e905158ff713fb10bb5395b438bd7fbbceb6180 +Subproject commit 694f630f793945e70249ba54ff4341454200ff72 From 33fe831287074eed6987f979621ebed3c8cd93d9 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 6 Apr 2019 20:05:39 -0700 Subject: [PATCH 023/111] support web2py.py --GAE {app-name}, thanks Paolo --- gluon/packages/dal | 2 +- gluon/widget.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index e973f27b..0c2ba9d7 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit e973f27b69a8499e4e30e5663f3fba74bf87364c +Subproject commit 0c2ba9d71f426bc6cc2347bd4ce111b18308fbe6 diff --git a/gluon/widget.py b/gluon/widget.py index 238a5674..0cbe6567 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -618,7 +618,7 @@ def console(): '--GAE', default=None, dest='gae', - help="'-G configure' will create app.yaml and gaehandler.py") + help="'-G {gae app name}' will create app.yaml and gaehandler.py") msg = ('password to be used for administration ' '(use -a "" to reuse the last password))') @@ -937,7 +937,9 @@ def console(): if options.gae: if not os.path.exists('app.yaml'): - name = input("Your GAE app name: ") + name = options.gae + if name == 'configure': + name = input("Your GAE app name: ") content = open(os.path.join('examples', 'app.example.yaml'), 'rb').read() open('app.yaml', 'wb').write(content.replace("yourappname", name)) else: From c9a42c46385e81f321f10042ef28b749d7d79e30 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 6 Apr 2019 20:54:16 -0700 Subject: [PATCH 024/111] refactoring of widget.py, thanks Paolo --- gluon/widget.py | 546 ++++++++++++++++++++---------------------------- 1 file changed, 227 insertions(+), 319 deletions(-) diff --git a/gluon/widget.py b/gluon/widget.py index 0cbe6567..2f2f7875 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -85,8 +85,8 @@ def run_system_tests(options): def get_url(host, path='/', proto='http', port=80): if ':' in host: host = '[%s]' % host - else: - host = host.replace('0.0.0.0', '127.0.0.1') + elif host == '0.0.0.0': + host = '127.0.0.1' if path.startswith('/'): path = path[1:] if proto.endswith(':'): @@ -469,9 +469,7 @@ class web2pyDialog(object): except: return self.error('invalid port number') - # Check for non default value for ssl inputs - if (len(self.options.ssl_certificate) > 0 or - len(self.options.ssl_private_key) > 0): + if self.options.ssl_certificate or self.options.ssl_private_key: proto = 'https' else: proto = 'http' @@ -583,342 +581,252 @@ class web2pyDialog(object): def console(): """ Defines the behavior of the console web2py execution """ import optparse - import textwrap - - usage = "python web2py.py" - - description = """\ - web2py Web Framework startup script. - ATTENTION: unless a password is specified (-a 'passwd') web2py will - attempt to run a GUI. In this case command line options are ignored.""" - - description = textwrap.dedent(description) parser = optparse.OptionParser( - usage, None, optparse.Option, ProgramVersion) + usage='python %prog [options]', + version=ProgramVersion, + description='web2py Web Framework startup script.', + epilog='''NOTE: unless a password is specified (-a 'passwd') +web2py will attempt to run a GUI to ask for it +(if not disabled with --nogui).''') - parser.description = description - - msg = ('IP address of the server (e.g., 127.0.0.1 or ::1); ' - 'Note: This value is ignored when using the \'interfaces\' option.') - parser.add_option('-i', - '--ip', + parser.add_option('-i', '--ip', default='127.0.0.1', - dest='ip', - help=msg) + help=\ + 'IP address of the server (e.g., 127.0.0.1 or ::1); ' \ + 'Note: This value is ignored when using the --interfaces option') - parser.add_option('-p', - '--port', + parser.add_option('-p', '--port', default='8000', - dest='port', type='int', - help='port of server (8000)') + help='port of server (%default)') - parser.add_option('-G', - '--GAE', + parser.add_option('-G', '--GAE', dest='gae', default=None, - dest='gae', - help="'-G {gae app name}' will create app.yaml and gaehandler.py") + help=\ + "'-G {app-name}' will create app.yaml and gaehandler.py") - msg = ('password to be used for administration ' - '(use -a "" to reuse the last password))') - parser.add_option('-a', - '--password', + parser.add_option('-a', '--password', default='', - dest='password', - help=msg) + help=\ + 'password to be used for administration ' \ + '(use -a "" to reuse the last password))') - parser.add_option('-c', - '--ssl_certificate', + parser.add_option('-c', '--ssl_certificate', default='', - dest='ssl_certificate', help='file that contains ssl certificate') - parser.add_option('-k', - '--ssl_private_key', + parser.add_option('-k', '--ssl_private_key', default='', - dest='ssl_private_key', help='file that contains ssl private key') - msg = ('Use this file containing the CA certificate to validate X509 ' - 'certificates from clients') - parser.add_option('--ca-cert', - action='store', - dest='ssl_ca_certificate', + parser.add_option('--ca-cert', dest='ssl_ca_certificate', default=None, - help=msg) + help=\ + 'use this file containing the CA certificate to validate X509 ' \ + 'certificates from clients') - parser.add_option('-d', - '--pid_filename', + parser.add_option('-d', '--pid_filename', default='httpserver.pid', - dest='pid_filename', help='file to store the pid of the server') - parser.add_option('-l', - '--log_filename', + parser.add_option('-l', '--log_filename', default='httpserver.log', - dest='log_filename', - help='file to log connections') + help='name for the server log file') - parser.add_option('-n', - '--numthreads', + parser.add_option('-n', '--numthreads', default=None, type='int', - dest='numthreads', help='number of threads (deprecated)') parser.add_option('--minthreads', default=None, type='int', - dest='minthreads', help='minimum number of server threads') parser.add_option('--maxthreads', default=None, type='int', - dest='maxthreads', help='maximum number of server threads') - parser.add_option('-s', - '--server_name', + parser.add_option('-s', '--server_name', default=socket.gethostname(), - dest='server_name', - help='server name for the web server') + help='web server name (%default)') - msg = 'max number of queued requests when server unavailable' - parser.add_option('-q', - '--request_queue_size', + parser.add_option('-q', '--request_queue_size', default='5', type='int', - dest='request_queue_size', - help=msg) + help=\ + 'max number of queued requests when server unavailable') - parser.add_option('-o', - '--timeout', + parser.add_option('-o', '--timeout', default='10', type='int', - dest='timeout', - help='timeout for individual request (10 seconds)') + help='timeout for individual request (%default seconds)') - parser.add_option('-z', - '--shutdown_timeout', + parser.add_option('-z', '--shutdown_timeout', default='5', type='int', - dest='shutdown_timeout', - help='timeout on shutdown of server (5 seconds)') + help='timeout on shutdown of server (%default seconds)') - parser.add_option('--socket-timeout', + parser.add_option('--socket-timeout', dest='socket_timeout', # not needed default=5, type='int', - dest='socket_timeout', - help='timeout for socket (5 second)') + help='timeout for socket (%default second)') - parser.add_option('-f', - '--folder', + parser.add_option('-f', '--folder', default=os.getcwd(), - dest='folder', help='folder from which to run web2py') - parser.add_option('-v', - '--verbose', - action='store_true', - dest='verbose', + parser.add_option('-v', '--verbose', default=False, - help='increase --test verbosity') + action='store_true', + help='increase --test and --run_system_tests verbosity') - parser.add_option('-Q', - '--quiet', - action='store_true', - dest='quiet', + parser.add_option('-Q', '--quiet', default=False, + action='store_true', help='disable all output') - parser.add_option('-e', - '--errors_to_console', - action='store_true', - dest='print_errors', + parser.add_option('-e', '--errors_to_console', dest='print_errors', default=False, + action='store_true', help='log all errors to console') - msg = ('set debug output level (0-100, 0 means all, 100 means none; ' - 'default is 30)') - parser.add_option('-D', - '--debug', - dest='debuglevel', + parser.add_option('-D', '--debug', dest='debuglevel', default=30, type='int', - help=msg) + help=\ + 'set debug output level (0-100, 0 means all, 100 means none; ' \ + 'default is %default)') - msg = ('run web2py in interactive shell or IPython (if installed) with ' - 'specified appname (if app does not exist it will be created). ' - 'APPNAME like a/c/f?x=y (c,f and vars x,y optional)') - parser.add_option('-S', - '--shell', - dest='shell', - metavar='APPNAME', - help=msg) + parser.add_option('-S', '--shell', + default=None, + metavar='APPNAME', help=\ + 'run web2py in interactive shell or IPython (if installed) with ' \ + 'specified appname (if app does not exist it will be created). ' \ + 'APPNAME like a/c/f?x=y (c,f and vars x,y optional)') - msg = ('run web2py in interactive shell or bpython (if installed) with ' - 'specified appname (if app does not exist it will be created).\n' - 'Use combined with --shell') - parser.add_option('-B', - '--bpython', - action='store_true', + parser.add_option('-B', '--bpython', default=False, - dest='bpython', - help=msg) - - msg = 'only use plain python shell; should be used with --shell option' - parser.add_option('-P', - '--plain', action='store_true', - default=False, - dest='plain', - help=msg) + help=\ + 'run web2py in interactive shell or bpython (if installed) with ' \ + 'specified appname (if app does not exist it will be created). ' \ + 'Use combined with --shell') - msg = ('auto import model files; default is False; should be used ' - 'with --shell option') - parser.add_option('-M', - '--import_models', + parser.add_option('-P', '--plain', + default=False, action='store_true', - default=False, - dest='import_models', - help=msg) + help=\ + 'only use plain python shell; should be used with --shell option') - msg = ('run PYTHON_FILE in web2py environment; ' - 'should be used with --shell option') - parser.add_option('-R', - '--run', - dest='run', - metavar='PYTHON_FILE', + parser.add_option('-M', '--import_models', + default=False, + action='store_true', + help=\ + 'auto import model files; default is %default; should be used ' \ + 'with --shell option') + + parser.add_option('-R', '--run', default='', # NOTE: used for sys.argv[0] if --shell - help=msg) + metavar='PYTHON_FILE', help=\ + 'run PYTHON_FILE in web2py environment; ' \ + 'should be used with --shell option') - msg = ('run scheduled tasks for the specified apps: expects a list of ' - 'app names as -K app1,app2,app3 ' - 'or a list of app:groups as -K app1:group1:group2,app2:group1 ' - 'to override specific group_names. (only strings, no spaces ' - 'allowed. Requires a scheduler defined in the models') - parser.add_option('-K', - '--scheduler', - dest='scheduler', + parser.add_option('-K', '--scheduler', default=None, - help=msg) + help=\ + 'run scheduled tasks for the specified apps: expects a list of ' \ + 'app names as -K app1,app2,app3 ' \ + 'or a list of app:groups as -K app1:group1:group2,app2:group1 ' \ + 'to override specific group_names. (only strings, no spaces ' \ + 'allowed. Requires a scheduler defined in the models') - msg = 'run schedulers alongside webserver, needs -K app1 and -a too' - parser.add_option('-X', - '--with-scheduler', + parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed + default=False, + action='store_true', + help=\ + 'run schedulers alongside webserver, needs -K app1 and -a too') + + parser.add_option('-T', '--test', + default=None, + metavar='TEST_PATH', help=\ + 'run doctests in web2py environment; ' \ + 'TEST_PATH like a/c/f (c,f optional)') + + parser.add_option('-C', '--cron', dest='extcron', action='store_true', default=False, - dest='with_scheduler', - help=msg) + help=\ + 'trigger a cron run manually; usually invoked from a system crontab') - msg = ('run doctests in web2py environment; ' - 'TEST_PATH like a/c/f (c,f optional)') - parser.add_option('-T', - '--test', - dest='test', - metavar='TEST_PATH', - default=None, - help=msg) - - msg = 'trigger a cron run manually; usually invoked from a system crontab' - parser.add_option('-C', - '--cron', - action='store_true', - dest='extcron', - default=False, - help=msg) - - msg = 'triggers the use of softcron' parser.add_option('--softcron', - action='store_true', - dest='softcron', default=False, - help=msg) + action='store_true', + help='triggers the use of softcron') - parser.add_option('-Y', - '--run-cron', - action='store_true', - dest='runcron', + parser.add_option('-Y', '--run-cron', dest='runcron', default=False, + action='store_true', help='start the background cron process') - parser.add_option('-J', - '--cronjob', - action='store_true', - dest='cronjob', + parser.add_option('-J', '--cronjob', default=False, + action='store_true', help='identify cron-initiated command') - parser.add_option('-L', - '--config', - dest='config', + parser.add_option('-L', '--config', default='', help='config file') - parser.add_option('-F', - '--profiler', - dest='profiler_dir', + parser.add_option('-F', '--profiler', dest='profiler_dir', default=None, help='profiler dir') - parser.add_option('-t', - '--taskbar', - action='store_true', - dest='taskbar', + parser.add_option('-t', '--taskbar', default=False, - help='use web2py gui and run in taskbar (system tray)') - - parser.add_option('', - '--nogui', action='store_true', - default=False, - dest='nogui', - help='text-only, no GUI') + help='use web2py GUI and run in taskbar (system tray)') - msg = ('should be followed by a list of arguments to be passed to script, ' - 'to be used with -S, -A must be the last option') - parser.add_option('-A', - '--args', - action='store', - dest='args', + parser.add_option('--nogui', + default=False, + action='store_true', + help='do not run GUI') + + parser.add_option('-A', '--args', default=None, - help=msg) + help=\ + 'should be followed by a list of arguments to be passed to script, ' \ + 'to be used with -S, -A must be the last option') - parser.add_option('--no-banner', - action='store_true', + parser.add_option('--no-banner', dest='nobanner', default=False, - dest='nobanner', - help='Do not print header banner') + action='store_true', + help='do not print header banner') - msg = ('listen on multiple addresses: ' - '"ip1:port1:key1:cert1:ca_cert1;ip2:port2:key2:cert2:ca_cert2;..." ' - '(:key:cert:ca_cert optional; no spaces; IPv6 addresses must be in ' - 'square [] brackets)') parser.add_option('--interfaces', - action='store', - dest='interfaces', default=None, - help=msg) + help=\ + 'listen on multiple addresses: ' \ + '"ip1:port1:key1:cert1:ca_cert1;ip2:port2:key2:cert2:ca_cert2;..." ' \ + '(:key:cert:ca_cert optional; no spaces; IPv6 addresses must be in ' \ + 'square [] brackets)') - msg = 'runs web2py tests' parser.add_option('--run_system_tests', - action='store_true', - dest='run_system_tests', default=False, - help=msg) + action='store_true', + help='run web2py tests') - msg = ('adds coverage reporting (needs --run_system_tests), ' - 'python 2.7 and the coverage module installed. ' - 'You can alter the default path setting the environmental ' - 'var "COVERAGE_PROCESS_START". ' - 'By default it takes gluon/tests/coverage.ini') parser.add_option('--with_coverage', - action='store_true', - dest='with_coverage', default=False, - help=msg) + action='store_true', + help=\ + 'adds coverage reporting (needs --run_system_tests), ' \ + 'python 2.7 and the coverage module installed. ' \ + 'You can alter the default path setting the environment ' \ + 'variable "COVERAGE_PROCESS_START" ' \ + '(by default it takes gluon/tests/coverage.ini)') if '-A' in sys.argv: k = sys.argv.index('-A') @@ -930,6 +838,11 @@ def console(): (options, args) = parser.parse_args() options.args = other_args + if options.config.endswith('.py'): + options.config = options.config[:-3] + + # TODO: process --config here; now is done in start function, too late + copy_options = copy.deepcopy(options) copy_options.password = '******' global_settings.cmd_options = copy_options @@ -938,6 +851,7 @@ def console(): if options.gae: if not os.path.exists('app.yaml'): name = options.gae + # for backward compatibility if name == 'configure': name = input("Your GAE app name: ") content = open(os.path.join('examples', 'app.example.yaml'), 'rb').read() @@ -958,7 +872,9 @@ def console(): except socket.gaierror: options.ips = [] + # FIXME: this should be done after create_welcome_w2p if options.run_system_tests: + # run system test and exit run_system_tests(options) if options.quiet: @@ -968,9 +884,6 @@ def console(): else: logger.setLevel(options.debuglevel) - if options.config[-3:] == '.py': - options.config = options.config[:-3] - if options.cronjob: global_settings.cronjob = True # tell the world options.plain = True # cronjobs use a plain shell @@ -986,16 +899,17 @@ def console(): interfaces = options.interfaces.split(';') options.interfaces = [] for interface in interfaces: - if interface.startswith('['): # IPv6 + if interface.startswith('['): + # IPv6 ip, if_remainder = interface.split(']', 1) ip = ip[1:] - if_remainder = if_remainder[1:].split(':') - if_remainder[0] = int(if_remainder[0]) # numeric port - options.interfaces.append(tuple([ip] + if_remainder)) - else: # IPv4 + interface = if_remainder[1:].split(':') + interface.insert(0, ip) + else: + # IPv4 interface = interface.split(':') - interface[1] = int(interface[1]) # numeric port - options.interfaces.append(tuple(interface)) + interface[1] = int(interface[1]) # numeric port + options.interfaces.append(tuple(interface)) # accepts --scheduler in the form # "app:group1:group2,app2:group1" @@ -1013,6 +927,7 @@ def console(): create_welcome_w2p() + # FIXME: do we still really need this? if not options.cronjob: # If we have the applications package or if we should upgrade if not os.path.exists('applications/__init__.py'): @@ -1093,43 +1008,40 @@ def start_schedulers(options): def start(cron=True): - """ Starts server """ - - # ## get command line arguments + """ Starts server and other services """ + # get command line arguments (options, args) = console() - if not options.nobanner: - print(ProgramName) - print(ProgramAuthor) - print(ProgramVersion) - - from pydal.drivers import DRIVERS - if not options.nobanner: - print('Database drivers available: %s' % ', '.join(DRIVERS)) - - # ## if -L load options from options.config file + # FIXME: this should be anticipated in console() if options.config: + # import options from options.config file try: - options2 = __import__(options.config, {}, {}, '') - except Exception: - try: - # Jython doesn't like the extra stuff - options2 = __import__(options.config) - except Exception: - print('Cannot import config file [%s]' % options.config) - sys.exit(1) + options2 = __import__(options.config) + except: + sys.stderr.write("Cannot import config file %s\n" % options.config) + sys.exit(1) for key in dir(options2): + # FIXME: better import condition, not all options attributes + # should be sourced from config file if hasattr(options, key): setattr(options, key, getattr(options2, key)) - # ## if -T run doctests (no cron) - if hasattr(options, 'test') and options.test: + if not options.nobanner: + # banner + print(ProgramName) + print(ProgramAuthor) + print(ProgramVersion) + from pydal.drivers import DRIVERS + print('Database drivers available: %s' % ', '.join(DRIVERS)) + + if options.test: + # run doctests and exit test(options.test, verbose=options.verbose) return - # ## if -S start interactive shell (also no cron) if options.shell: + # run interactive shell and exit if options.folder: os.chdir(options.folder) sys.argv = [options.run] + options.args @@ -1138,12 +1050,12 @@ def start(cron=True): cronjob=options.cronjob) return - # ## if -C start cron run (extcron) and exit - # ## -K specifies optional apps list (overloading scheduler) if options.extcron: + # run cron (extcron) and exit logger.debug('Starting extcron...') global_settings.web2py_crontype = 'external' - if options.scheduler: # -K + if options.scheduler: + # run cron for applications listed with --scheduler (-K) apps = [app.strip() for app in options.scheduler.split( ',') if check_existent_app(options, app.strip())] else: @@ -1153,57 +1065,52 @@ def start(cron=True): extcron.join() return - # ## if -K if options.scheduler and not options.with_scheduler: + # run schedulers and exit try: start_schedulers(options) except KeyboardInterrupt: pass return - # ## if -H cron is enabled in this *process* - # ## if --softcron use softcron - # ## use hardcron in all other cases - if cron and options.runcron and options.softcron: - print('Using softcron (but this is not very efficient)') - global_settings.web2py_crontype = 'soft' - elif cron and options.runcron: - logger.debug('Starting hardcron...') - global_settings.web2py_crontype = 'hard' - newcron.hardcron(options.folder).start() + if cron and options.runcron: + if options.softcron: + print('Using softcron (but this is not very efficient)') + global_settings.web2py_crontype = 'soft' + else: + # start hardcron thread + logger.debug('Starting hardcron...') + global_settings.web2py_crontype = 'hard' + newcron.hardcron(options.folder).start() - # ## if no password provided and havetk start Tk interface - # ## or start interface if we want to put in taskbar (system tray) - - try: - options.taskbar - except: - options.taskbar = False + # if no password provided and have Tk library start GUI (when not + # explicitly disabled), we also need a GUI to put in taskbar (system tray) + # when requested + # FIXME: this check should be done first if options.taskbar and os.name != 'nt': - print('Error: taskbar not supported on this platform') + sys.stderr.write('Error: taskbar not supported on this platform\n') sys.exit(1) root = None - if not options.nogui and options.password == '': + if (not options.nogui and options.password == '') or options.taskbar: try: if PY2: import Tkinter as tkinter else: import tkinter - havetk = True - try: - root = tkinter.Tk() - except: - pass + root = tkinter.Tk() except (ImportError, OSError): logger.warn( 'GUI not available because Tk library is not installed') - havetk = False + options.nogui = True + except: + logger.exception('cannot get Tk root window, GUI disabled') options.nogui = True if root: + # run GUI and exit root.focus_force() # Mac OS X - make the GUI window rise to the top @@ -1216,6 +1123,7 @@ end tell """ % (os.getpid()) os.system("/usr/bin/osascript -e '%s'" % applescript) + # web2pyDialog takes care of schedulers master = web2pyDialog(root, options) signal.signal(signal.SIGTERM, lambda a, b: master.quit()) @@ -1226,31 +1134,32 @@ end tell sys.exit() - # ## if no tk and no password, ask for a password - - if not root and options.password == '': + if options.password == '': options.password = getpass.getpass('choose a password:') if not options.password and not options.nobanner: - print('no password, no admin interface') + print('no password, disable admin interface') - # ##-X (if no tk, the widget takes care of it himself) - if not root and options.scheduler and options.with_scheduler: - t = threading.Thread(target=start_schedulers, args=(options,)) - t.start() + spt = None - # ## start server + if options.scheduler and options.with_scheduler: + # start schedulers in a separate thread + spt = threading.Thread(target=start_schedulers, args=(options,)) + spt.start() + + # start server # Use first interface IP and port if interfaces specified, since the # interfaces option overrides the IP (and related) options. if not options.interfaces: - (ip, port) = (options.ip, int(options.port)) + ip = options.ip + port = int(options.port) else: first_if = options.interfaces[0] - (ip, port) = first_if[0], first_if[1] + ip = first_if[0] + port = first_if[1] - # Check for non default value for ssl inputs - if (len(options.ssl_certificate) > 0) or (len(options.ssl_private_key) > 0): + if options.ssl_certificate or options.ssl_private_key: proto = 'https' else: proto = 'http' @@ -1258,12 +1167,12 @@ end tell url = get_url(ip, proto=proto, port=port) if not options.nobanner: - message = '\nplease visit:\n\t%s\n' % url + message = '\nplease visit:\n\t%s\n' if sys.platform.startswith('win'): - message += 'use "taskkill /f /pid %i" to shutdown the web2py server\n\n' % os.getpid() + message += 'use "taskkill /f /pid %i" to shutdown the web2py server\n\n' else: - message += 'use "kill -SIGTERM %i" to shutdown the web2py server\n\n' % os.getpid() - print(message) + message += 'use "kill -SIGTERM %i" to shutdown the web2py server\n\n' + print(message % (url, os.getpid())) # enhance linecache.getline (used by debugger) to look at the source file # if the line was not found (under py2exe & when file was modified) @@ -1274,16 +1183,13 @@ end tell line = py2exe_getline(filename, lineno, *args, **kwargs) if not line: try: - f = open(filename, "rb") - try: + with open(filename, "rb") as f: for i, line in enumerate(f): line = line.decode('utf-8') if lineno == i + 1: break else: line = '' - finally: - f.close() except (IOError, OSError): line = '' return line @@ -1312,8 +1218,10 @@ end tell server.start() except KeyboardInterrupt: server.stop() - try: - t.join() - except: - pass + if spt is not None: + try: + spt.join() + except: + logger.exception('error terminating schedulers') + pass logging.shutdown() From 5667149f8d94a9c9804c36c270916aec7a3ef542 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 6 Apr 2019 20:59:23 -0700 Subject: [PATCH 025/111] syncing --- gluon/packages/dal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index 0c2ba9d7..0d6db1c1 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 0c2ba9d71f426bc6cc2347bd4ce111b18308fbe6 +Subproject commit 0d6db1c19166384f787fc5502be70136f2492e6a From d13a003475dbe01348b9556420cf87e32cb84fbd Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 7 Apr 2019 09:25:08 -0700 Subject: [PATCH 026/111] use metavar, thanks Paolo --- gluon/packages/dal | 2 +- gluon/widget.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index 0d6db1c1..ce5105f9 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 0d6db1c19166384f787fc5502be70136f2492e6a +Subproject commit ce5105f9ef4114dd3f6162bfe6ba7d650e592075 diff --git a/gluon/widget.py b/gluon/widget.py index 2f2f7875..6e27f8f4 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -603,8 +603,8 @@ web2py will attempt to run a GUI to ask for it parser.add_option('-G', '--GAE', dest='gae', default=None, - help=\ - "'-G {app-name}' will create app.yaml and gaehandler.py") + metavar='APP_NAME', help=\ + "will create app.yaml and gaehandler.py") parser.add_option('-a', '--password', default='', @@ -672,7 +672,7 @@ web2py will attempt to run a GUI to ask for it parser.add_option('--socket-timeout', dest='socket_timeout', # not needed default=5, type='int', - help='timeout for socket (%default second)') + help='timeout for socket (%default seconds)') parser.add_option('-f', '--folder', default=os.getcwd(), @@ -756,8 +756,8 @@ web2py will attempt to run a GUI to ask for it 'TEST_PATH like a/c/f (c,f optional)') parser.add_option('-C', '--cron', dest='extcron', - action='store_true', default=False, + action='store_true', help=\ 'trigger a cron run manually; usually invoked from a system crontab') From 455d188da894a2d9b13c25c6a3378efb08647460 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 7 Apr 2019 16:26:28 -0700 Subject: [PATCH 027/111] removed un-necessary sort exception is sqlform.grid --- gluon/packages/dal | 2 +- gluon/sqlhtml.py | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index ce5105f9..39c8463f 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit ce5105f9ef4114dd3f6162bfe6ba7d650e592075 +Subproject commit 39c8463f959c42173f7bc5575652595bc6b22c02 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 5c1c59f6..19d333d9 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -2665,11 +2665,7 @@ class SQLFORM(FORM): if order and not order == 'None': otablename, ofieldname = order.split('~')[-1].split('.', 1) sort_field = db[otablename][ofieldname] - exception = sort_field.type in ('date', 'datetime', 'time') - if exception: - orderby = (order[:1] == '~' and sort_field) or ~sort_field - else: - orderby = (order[:1] == '~' and ~sort_field) or sort_field + orderby = sort_field if order[:1] != '~' else ~sort_field orderby = fix_orderby(orderby) @@ -2803,12 +2799,7 @@ class SQLFORM(FORM): otablename, ofieldname = order.split('~')[-1].split('.', 1) sort_field = db[otablename][ofieldname] # invert order direction on date/time fields - exception = sort_field.type in ('date', 'datetime', 'time') - if exception: - desc_icon, asc_icon = sorter_icons - orderby = (order[:1] == '~' and sort_field) or ~sort_field - else: - orderby = (order[:1] == '~' and ~sort_field) or sort_field + orderby = sort_field if order[:1] != '~' else ~sort_field headcols = [] if selectable: From 59700b8d06507a0b808efb3e4a5962f6af3992c7 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 7 Apr 2019 21:16:13 -0700 Subject: [PATCH 028/111] R-2.18.5 --- CHANGELOG | 2 +- Makefile | 2 +- VERSION | 2 +- gluon/packages/dal | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index e27d5bdc..c0735936 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -## 2.18.1-2.18.4 +## 2.18.1-2.18.5 - pydal 19.02 - made template its own module (Yet Another Template Language) - improved python 3.4-3.7 support diff --git a/Makefile b/Makefile index fba4fbd3..2e33adf9 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ rmfiles: rm -rf applications/examples/uploads/* src: ### Use semantic versioning - echo 'Version 2.18.4-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION + echo 'Version 2.18.5-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION ### rm -f all junk files make clean # make rmfiles diff --git a/VERSION b/VERSION index ceadb05f..67432273 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.18.4-stable+timestamp.2019.03.12.22.20.22 +Version 2.18.5-stable+timestamp.2019.04.07.21.13.59 diff --git a/gluon/packages/dal b/gluon/packages/dal index 39c8463f..55a98df7 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 39c8463f959c42173f7bc5575652595bc6b22c02 +Subproject commit 55a98df73ef3460e8b57e9494fe555da7474b869 From b96c54cef94d96b113cf90af690d53e19397bd16 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 9 Apr 2019 21:49:04 -0700 Subject: [PATCH 029/111] some cleanup and better use of die() function, thanks paolo --- CHANGELOG | 2 +- gluon/packages/dal | 2 +- gluon/shell.py | 16 +++--- gluon/widget.py | 131 ++++++++++++++++++++------------------------- web2py.py | 32 +++++++++-- 5 files changed, 96 insertions(+), 87 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c0735936..b5909b91 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,5 @@ ## 2.18.1-2.18.5 -- pydal 19.02 +- pydal 19.04 - made template its own module (Yet Another Template Language) - improved python 3.4-3.7 support - better regular expressions diff --git a/gluon/packages/dal b/gluon/packages/dal index 55a98df7..cecd7712 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 55a98df73ef3460e8b57e9494fe555da7474b869 +Subproject commit cecd77127c122404c1aee7f6377c6a0150d86d84 diff --git a/gluon/shell.py b/gluon/shell.py index 225ac4e6..bcac912e 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -193,6 +193,13 @@ def exec_pythonrc(): return dict() +def die(msg, exit_status=1, error_preamble=True): + if error_preamble: + msg = "%s: error: %s" % (sys.argv[0], msg) + print(msg, file=sys.stderr) + sys.exit(exit_status) + + def run( appname, plain=False, @@ -212,7 +219,7 @@ def run( (a, c, f, args, vars) = parse_path_info(appname, av=True) errmsg = 'invalid application name: %s' % appname if not a: - die(errmsg) + die(errmsg, error_preamble=False) adir = os.path.join('applications', a) if not os.path.exists(adir): @@ -258,7 +265,7 @@ def run( elif os.path.isfile(pyfile): execfile(pyfile, _env) else: - die(errmsg) + die(errmsg, error_preamble=False) if f: exec('print( %s())' % f, _env) @@ -358,11 +365,6 @@ def parse_path_info(path_info, av=False): return (None, None, None) -def die(msg): - print(msg, file=sys.stderr) - sys.exit(1) - - def test(testpath, import_models=True, verbose=False): """ Run doctests in web2py environment. testpath is formatted like: diff --git a/gluon/widget.py b/gluon/widget.py index 6e27f8f4..33ef31df 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -24,12 +24,9 @@ from gluon import main, newcron from gluon.fileutils import read_file, write_file, create_welcome_w2p from gluon.settings import global_settings -from gluon.shell import run, test +from gluon.shell import die, run, test from gluon.utils import is_valid_ip_address, is_loopback_ip_address, getipaddrinfo -if PY2: - input = raw_input - ProgramName = 'web2py Web Framework' ProgramAuthor = 'Created by Massimo Di Pierro, Copyright 2007-' + str( @@ -63,8 +60,7 @@ def run_system_tests(options): try: import coverage except: - sys.stderr.write('Coverage was not installed\n') - sys.exit(1) + die('Coverage not installed') if not PY2: sys.stderr.write('Experimental ') sys.stderr.write("Python %s\n" % sys.version) @@ -342,7 +338,7 @@ class web2pyDialog(object): try: from multiprocessing import Process except: - sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n') + sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') return code = "from gluon.globals import current;current._scheduler.loop()" print('starting scheduler from widget for "%s"...' % app) @@ -597,14 +593,15 @@ web2py will attempt to run a GUI to ask for it 'Note: This value is ignored when using the --interfaces option') parser.add_option('-p', '--port', - default='8000', - type='int', - help='port of server (%default)') + default=8000, + type='int', help=\ + 'port of server (%default); ' \ + 'Note: This value is ignored when using the --interfaces option') parser.add_option('-G', '--GAE', dest='gae', default=None, metavar='APP_NAME', help=\ - "will create app.yaml and gaehandler.py") + 'will create app.yaml and gaehandler.py and exit') parser.add_option('-a', '--password', default='', @@ -836,34 +833,21 @@ web2py will attempt to run a GUI to ask for it k = len(sys.argv) sys.argv, other_args = sys.argv[:k], sys.argv[k + 1:] (options, args) = parser.parse_args() + # TODO: warn or error if args (should be no unparsed arguments) options.args = other_args if options.config.endswith('.py'): options.config = options.config[:-3] - - # TODO: process --config here; now is done in start function, too late - - copy_options = copy.deepcopy(options) - copy_options.password = '******' - global_settings.cmd_options = copy_options - global_settings.cmd_args = args - - if options.gae: - if not os.path.exists('app.yaml'): - name = options.gae - # for backward compatibility - if name == 'configure': - name = input("Your GAE app name: ") - content = open(os.path.join('examples', 'app.example.yaml'), 'rb').read() - open('app.yaml', 'wb').write(content.replace("yourappname", name)) - else: - print("app.yaml alreday exists in the web2py folder") - if not os.path.exists('gaehandler.py'): - content = open(os.path.join('handlers', 'gaehandler.py'), 'rb').read() - open('gaehandler.py', 'wb').write(content) - else: - print("gaehandler.py alreday exists in the web2py folder") - sys.exit(0) + if options.config: + # import options from options.config file + try: + # FIXME: avoid __import__ + options2 = __import__(options.config) + except: + die("cannot import config file %s" % options.config) + for key in dir(options2): + if hasattr(options, key): + setattr(options, key, getattr(options2, key)) try: options.ips = list(set( # no duplicates @@ -872,29 +856,15 @@ web2py will attempt to run a GUI to ask for it except socket.gaierror: options.ips = [] - # FIXME: this should be done after create_welcome_w2p - if options.run_system_tests: - # run system test and exit - run_system_tests(options) - - if options.quiet: - capture = StringIO() - sys.stdout = capture - logger.setLevel(logging.CRITICAL + 1) - else: - logger.setLevel(options.debuglevel) - if options.cronjob: global_settings.cronjob = True # tell the world options.plain = True # cronjobs use a plain shell options.nobanner = True options.nogui = True - options.folder = os.path.abspath(options.folder) - # accept --interfaces in the form # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" - # (no spaces; optional key:cert indicate SSL) + # (no spaces; optional key:cert:ca_cert indicate SSL) if isinstance(options.interfaces, str): interfaces = options.interfaces.split(';') options.interfaces = [] @@ -925,13 +895,11 @@ web2py will attempt to run a GUI to ask for it if options.numthreads is not None and options.minthreads is None: options.minthreads = options.numthreads # legacy - create_welcome_w2p() - + copy_options = copy.deepcopy(options) + copy_options.password = '******' + global_settings.cmd_options = copy_options # FIXME: do we still really need this? - if not options.cronjob: - # If we have the applications package or if we should upgrade - if not os.path.exists('applications/__init__.py'): - write_file('applications/__init__.py', '') + global_settings.cmd_args = args return options, args @@ -959,7 +927,7 @@ def start_schedulers(options): try: from multiprocessing import Process except: - sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n') + sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') return processes = [] apps = [(app.strip(), None) for app in options.scheduler.split(',')] @@ -1013,19 +981,37 @@ def start(cron=True): # get command line arguments (options, args) = console() - # FIXME: this should be anticipated in console() - if options.config: - # import options from options.config file - try: - options2 = __import__(options.config) - except: - sys.stderr.write("Cannot import config file %s\n" % options.config) - sys.exit(1) - for key in dir(options2): - # FIXME: better import condition, not all options attributes - # should be sourced from config file - if hasattr(options, key): - setattr(options, key, getattr(options2, key)) + if options.gae: + # write app.yaml, gaehandler.py, and exit + if not os.path.exists('app.yaml'): + name = options.gae + # for backward compatibility + if name == 'configure': + if PY2: input = raw_input + name = input("Your GAE app name: ") + content = open(os.path.join('examples', 'app.example.yaml'), 'rb').read() + open('app.yaml', 'wb').write(content.replace("yourappname", name)) + else: + print("app.yaml alreday exists in the web2py folder") + if not os.path.exists('gaehandler.py'): + content = open(os.path.join('handlers', 'gaehandler.py'), 'rb').read() + open('gaehandler.py', 'wb').write(content) + else: + print("gaehandler.py alreday exists in the web2py folder") + return + + create_welcome_w2p() + + if options.run_system_tests: + # run system test and exit + run_system_tests(options) + + if options.quiet: + capture = StringIO() + sys.stdout = capture + logger.setLevel(logging.CRITICAL + 1) + else: + logger.setLevel(options.debuglevel) if not options.nobanner: # banner @@ -1089,8 +1075,7 @@ def start(cron=True): # FIXME: this check should be done first if options.taskbar and os.name != 'nt': - sys.stderr.write('Error: taskbar not supported on this platform\n') - sys.exit(1) + die('taskbar not supported on this platform') root = None @@ -1153,7 +1138,7 @@ end tell # interfaces option overrides the IP (and related) options. if not options.interfaces: ip = options.ip - port = int(options.port) + port = options.port else: first_if = options.interfaces[0] ip = first_if[0] diff --git a/web2py.py b/web2py.py index ced8ff22..875642da 100755 --- a/web2py.py +++ b/web2py.py @@ -1,28 +1,49 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function + import os import sys from multiprocessing import freeze_support -# import gluon.import_all ##### This should be uncommented for py2exe.py if hasattr(sys, 'frozen'): - path = os.path.dirname(os.path.abspath(sys.executable)) # for py2exe + # py2exe + path = os.path.dirname(os.path.abspath(sys.executable)) elif '__file__' in globals(): path = os.path.dirname(os.path.abspath(__file__)) -else: # should never happen +else: + # should never happen path = os.getcwd() + +# process -f (--folder) option +if '-f' in sys.argv: + fi = sys.argv.index('-f') +elif '--folder' in sys.argv: + fi = sys.argv.index('--folder') +else: + fi = None +if fi and fi < len(sys.argv): + fi += 1 + folder = sys.argv[fi] + if not os.path.isdir(os.path.join(folder, 'gluon')): + print("%s: error: bad folder %s" % (sys.argv[0], folder), file=sys.stderr) + sys.exit(1) + path = sys.argv[fi] = os.path.abspath(folder) + os.chdir(path) sys.path = [path] + [p for p in sys.path if not p == path] # important that this import is after the os.chdir +# import gluon.import_all # NOTE: should this be uncommented for py2exe.py ? import gluon.widget -# Start Web2py and Web2py cron service! if __name__ == '__main__': freeze_support() + # support for sub-process coverage, + # see https://coverage.readthedocs.io/en/coverage-4.3.4/subprocess.html if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage @@ -30,4 +51,5 @@ if __name__ == '__main__': except: print('Coverage is not available') pass - gluon.widget.start(cron=True) + # start services + gluon.widget.start() From 066d9c9ab54e9e88e42e58f7effb19afaade4f88 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 11 Apr 2019 21:15:48 -0700 Subject: [PATCH 030/111] better newcrow and widget, fixed some ssl related bugs, thanks Paolo --- gluon/main.py | 5 +- gluon/newcron.py | 106 ++++++++++++++----------------- gluon/packages/dal | 2 +- gluon/widget.py | 155 +++++++++++++++++++++++++-------------------- 4 files changed, 140 insertions(+), 128 deletions(-) diff --git a/gluon/main.py b/gluon/main.py index aeb7f76e..89ef709a 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -55,7 +55,6 @@ web2py_path = global_settings.applications_parent # backward compatibility create_missing_folders() # set up logging for subsequent imports -import logging import logging.config # This needed to prevent exception on Python 2.5: @@ -765,8 +764,8 @@ class HttpServer(object): app_info=app_info, min_threads=min_threads, max_threads=max_threads, - queue_size=int(request_queue_size), - timeout=int(timeout), + queue_size=request_queue_size, + timeout=timeout, handle_signals=False, ) diff --git a/gluon/newcron.py b/gluon/newcron.py index 4affed93..a415b530 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -17,7 +17,6 @@ import time import sched import re import datetime -import platform from functools import reduce from gluon.settings import global_settings from gluon import fileutils @@ -87,7 +86,7 @@ class hardcron(threading.Thread): def run(self): s = sched.scheduler(time.time, time.sleep) - logger.info('Hard cron daemon started') + logger.info('hard cron daemon started') while not _cron_stopping: now = time.time() s.enter(60 - now % 60, 1, self.launch, ()) @@ -133,7 +132,7 @@ class Token(object): else: locktime = 59.99 if portalocker.LOCK_EX is None: - logger.warning('WEB2PY CRON: Disabled because no file locking') + logger.warning('cron disabled because no file locking') return None self.master = fileutils.open_file(self.path, 'rb+') try: @@ -142,13 +141,14 @@ class Token(object): try: (start, stop) = pickle.load(self.master) except: - (start, stop) = (0, 1) + start = 0 + stop = 1 if startup or self.now - start > locktime: ret = self.now if not stop: # this happens if previous cron job longer than 1 minute - logger.warning('WEB2PY CRON: Stale cron.master detected') - logger.debug('WEB2PY CRON: Acquiring lock') + logger.warning('stale cron.master detected') + logger.debug('acquiring lock') self.master.seek(0) pickle.dump((self.now, 0), self.master) self.master.flush() @@ -166,7 +166,7 @@ class Token(object): ret = self.master.closed if not self.master.closed: portalocker.lock(self.master, portalocker.LOCK_EX) - logger.debug('WEB2PY CRON: Releasing cron lock') + logger.debug('releasing cron lock') self.master.seek(0) (start, stop) = pickle.load(self.master) if start == self.now: # if this is my lock @@ -241,12 +241,9 @@ def parsecronline(line): class cronlauncher(threading.Thread): - def __init__(self, cmd, shell=True): + def __init__(self, cmd): threading.Thread.__init__(self) - if platform.system() == 'Windows': - shell = False self.cmd = cmd - self.shell = shell def run(self): import subprocess @@ -258,8 +255,7 @@ class cronlauncher(threading.Thread): proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=self.shell) + stderr=subprocess.PIPE) _cron_subprocs.append(proc) (stdoutdata, stderrdata) = proc.communicate() try: @@ -267,15 +263,14 @@ class cronlauncher(threading.Thread): except ValueError: pass if proc.returncode != 0: - logger.warning( - 'WEB2PY CRON Call returned code %s:\n%s' % - (proc.returncode, stdoutdata + stderrdata)) + logger.warning('call returned code %s:\n%s\n%s', + proc.returncode, stdoutdata, stderrdata) else: - logger.debug('WEB2PY CRON Call returned success:\n%s' - % stdoutdata) + logger.debug('call returned success:\n%s', stdoutdata) def crondance(applications_parent, ctype='soft', startup=False, apps=None): + # TODO: docstring apppath = os.path.join(applications_parent, 'applications') token = Token(applications_parent) cronmaster = token.acquire(startup=startup) @@ -294,6 +289,21 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): full_apath_links = set() + if sys.executable.lower().endswith('pythonservice.exe'): + _python_exe = os.path.join(sys.exec_prefix, 'python.exe') + else: + _python_exe = sys.executable + base_commands = [_python_exe] + w2p_path = fileutils.abspath('web2py.py', gluon=True) + if os.path.exists(w2p_path): + base_commands.append(w2p_path) + if applications_parent != global_settings.gluon_parent: + base_commands.extend(('-f', applications_parent)) + base_commands.extend(('-J', + # FIXME: this should not be needed since we are + # not launching the web server + '-a', '""')) + for app in apps: if _cron_stopping: break @@ -315,22 +325,12 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): lines = [line for line in cronlines if line and not line.startswith('#')] tasks = [parsecronline(cline) for cline in lines] except Exception as e: - logger.error('WEB2PY CRON: crontab read error %s' % e) + logger.error('crontab read error %s', e) continue for task in tasks: if _cron_stopping: break - if sys.executable.lower().endswith('pythonservice.exe'): - _python_exe = os.path.join(sys.exec_prefix, 'python.exe') - else: - _python_exe = sys.executable - commands = [_python_exe] - w2p_path = fileutils.abspath('web2py.py', gluon=True) - if os.path.exists(w2p_path): - commands.append(w2p_path) - if applications_parent != global_settings.gluon_parent: - commands.extend(('-f', applications_parent)) citems = [(k in task and not v in task[k]) for k, v in checks] task_min = task.get('min', []) if not task: @@ -339,40 +339,32 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): continue elif task_min != [-1] and reduce(lambda a, b: a or b, citems): continue - logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' - % (ctype, app, task.get('cmd'), - os.getcwd(), datetime.datetime.now())) - action, command, models = False, task['cmd'], '' + logger.info('%s cron: %s executing %s in %s at %s', + ctype, app, task.get('cmd'), + os.getcwd(), datetime.datetime.now()) + action = models = False + command = task['cmd'] if command.startswith('**'): - (action, models, command) = (True, '', command[2:]) + action = True + command = command[2:] elif command.startswith('*'): - (action, models, command) = (True, '-M', command[1:]) - else: - action = False + action = models = True + command = command[1:] - if action and command.endswith('.py'): - commands.extend(('-J', # cron job - models, # import models? - '-S', app, # app name - '-a', '""', # password - '-R', command)) # command - elif action: - commands.extend(('-J', # cron job - models, # import models? - '-S', app + '/' + command, # app name - '-a', '""')) # password + if action: + commands = base_commands[:] + if command.endswith('.py'): + commands.extend(('-S', app, '-R', command)) + else: + commands.extend(('-S', app + '/' + command)) + if models: + commands.append('-M') else: commands = command - # from python docs: - # You do not need shell=True to run a batch file or - # console-based executable. - shell = False - try: - cronlauncher(commands, shell=shell).start() + cronlauncher(commands).start() except Exception as e: - logger.warning( - 'WEB2PY CRON: Execution error for %s: %s' - % (task.get('cmd'), e)) + logger.warning('execution error for %s: %s', + task.get('cmd'), e) token.release() diff --git a/gluon/packages/dal b/gluon/packages/dal index cecd7712..37784cb6 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit cecd77127c122404c1aee7f6377c6a0150d86d84 +Subproject commit 37784cb6aaa37340eb706eb550164a3f56be4186 diff --git a/gluon/widget.py b/gluon/widget.py index 33ef31df..c87028e9 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -10,7 +10,7 @@ The widget is called from web2py """ import sys -from gluon._compat import StringIO, thread, xrange, PY2 +from gluon._compat import thread, xrange, PY2 import time import threading import os @@ -465,7 +465,7 @@ class web2pyDialog(object): except: return self.error('invalid port number') - if self.options.ssl_certificate or self.options.ssl_private_key: + if self.options.ssl_certificate and self.options.ssl_private_key: proto = 'https' else: proto = 'http' @@ -583,18 +583,18 @@ def console(): version=ProgramVersion, description='web2py Web Framework startup script.', epilog='''NOTE: unless a password is specified (-a 'passwd') -web2py will attempt to run a GUI to ask for it +web2py will attempt to run a GUI to ask for it when starting the web server (if not disabled with --nogui).''') parser.add_option('-i', '--ip', default='127.0.0.1', - help=\ + metavar='IP_ADDR', help=\ 'IP address of the server (e.g., 127.0.0.1 or ::1); ' \ 'Note: This value is ignored when using the --interfaces option') parser.add_option('-p', '--port', default=8000, - type='int', help=\ + type='int', metavar='NUM', help=\ 'port of server (%default); ' \ 'Note: This value is ignored when using the --interfaces option') @@ -607,43 +607,43 @@ web2py will attempt to run a GUI to ask for it default='', help=\ 'password to be used for administration ' \ - '(use -a "" to reuse the last password))') + '(use "" to reuse the last password), ' \ + 'when no password is available the administrative ' \ + 'interface will be disabled') parser.add_option('-c', '--ssl_certificate', - default='', - help='file that contains ssl certificate') + default=None, + metavar='FILE', help='server certificate file') parser.add_option('-k', '--ssl_private_key', - default='', - help='file that contains ssl private key') + default=None, + metavar='FILE', help='server private key file') parser.add_option('--ca-cert', dest='ssl_ca_certificate', default=None, - help=\ - 'use this file containing the CA certificate to validate X509 ' \ - 'certificates from clients') + metavar='FILE', help='CA certificate file') parser.add_option('-d', '--pid_filename', default='httpserver.pid', - help='file to store the pid of the server') + metavar='FILE', help='server pid file (%default)') parser.add_option('-l', '--log_filename', default='httpserver.log', - help='name for the server log file') + metavar='FILE', help='server log file (%default)') parser.add_option('-n', '--numthreads', default=None, - type='int', + type='int', metavar='NUM', help='number of threads (deprecated)') parser.add_option('--minthreads', default=None, - type='int', + type='int', metavar='NUM', help='minimum number of server threads') parser.add_option('--maxthreads', default=None, - type='int', + type='int', metavar='NUM', help='maximum number of server threads') parser.add_option('-s', '--server_name', @@ -651,28 +651,30 @@ web2py will attempt to run a GUI to ask for it help='web server name (%default)') parser.add_option('-q', '--request_queue_size', - default='5', - type='int', + default=5, + type='int', metavar='NUM', help=\ - 'max number of queued requests when server unavailable') + 'max number of queued requests when server unavailable (%default)') parser.add_option('-o', '--timeout', - default='10', - type='int', + default=10, + type='int', metavar='SECONDS', help='timeout for individual request (%default seconds)') parser.add_option('-z', '--shutdown_timeout', - default='5', - type='int', - help='timeout on shutdown of server (%default seconds)') + default=None, + type='int', metavar='SECONDS', + help=\ + 'timeout on server shutdown; this value is not used by ' \ + 'Rocket web server') parser.add_option('--socket-timeout', dest='socket_timeout', # not needed default=5, - type='int', + type='int', metavar='SECONDS', help='timeout for socket (%default seconds)') parser.add_option('-f', '--folder', - default=os.getcwd(), + default=os.getcwd(), metavar='WEB2PY_DIR', help='folder from which to run web2py') parser.add_option('-v', '--verbose', @@ -693,8 +695,8 @@ web2py will attempt to run a GUI to ask for it parser.add_option('-D', '--debug', dest='debuglevel', default=30, type='int', - help=\ - 'set debug output level (0-100, 0 means all, 100 means none; ' \ + metavar='LOG_LEVEL', help=\ + 'set log level (0-100, 0 means all, 100 means none; ' \ 'default is %default)') parser.add_option('-S', '--shell', @@ -722,7 +724,7 @@ web2py will attempt to run a GUI to ask for it default=False, action='store_true', help=\ - 'auto import model files; default is %default; should be used ' \ + 'auto import model files (default is %default); should be used ' \ 'with --shell option') parser.add_option('-R', '--run', @@ -733,18 +735,18 @@ web2py will attempt to run a GUI to ask for it parser.add_option('-K', '--scheduler', default=None, - help=\ + metavar='APP_LIST', help=\ 'run scheduled tasks for the specified apps: expects a list of ' \ - 'app names as -K app1,app2,app3 ' \ - 'or a list of app:groups as -K app1:group1:group2,app2:group1 ' \ - 'to override specific group_names. (only strings, no spaces ' \ - 'allowed. Requires a scheduler defined in the models') + 'app names as app1,app2,app3 ' \ + 'or a list of app:groups as app1:group1:group2,app2:group1 ' \ + '(only strings, no spaces allowed). NOTE: ' \ + 'Requires a scheduler defined in the models') parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed default=False, action='store_true', help=\ - 'run schedulers alongside webserver, needs -K app1 and -a too') + 'run schedulers alongside webserver, needs -K') parser.add_option('-T', '--test', default=None, @@ -756,12 +758,16 @@ web2py will attempt to run a GUI to ask for it default=False, action='store_true', help=\ - 'trigger a cron run manually; usually invoked from a system crontab') + 'trigger a cron run and exit; usually used when invoked ' \ + 'from a system crontab') parser.add_option('--softcron', default=False, action='store_true', - help='triggers the use of softcron') + help=\ + 'use software cron emulation instead of separate cron process, '\ + 'needs -Y; NOTE: use of software cron emulation is strongly ' + 'discouraged') parser.add_option('-Y', '--run-cron', dest='runcron', default=False, @@ -795,7 +801,8 @@ web2py will attempt to run a GUI to ask for it default=None, help=\ 'should be followed by a list of arguments to be passed to script, ' \ - 'to be used with -S, -A must be the last option') + 'to be used with -S; NOTE: must be the last option because eat all ' \ + 'remaining arguments') parser.add_option('--no-banner', dest='nobanner', default=False, @@ -819,8 +826,8 @@ web2py will attempt to run a GUI to ask for it default=False, action='store_true', help=\ - 'adds coverage reporting (needs --run_system_tests), ' \ - 'python 2.7 and the coverage module installed. ' \ + 'adds coverage reporting (should be used with --run_system_tests), ' \ + 'needs Python 2.7+ and the coverage module installed. ' \ 'You can alter the default path setting the environment ' \ 'variable "COVERAGE_PROCESS_START" ' \ '(by default it takes gluon/tests/coverage.ini)') @@ -849,6 +856,7 @@ web2py will attempt to run a GUI to ask for it if hasattr(options, key): setattr(options, key, getattr(options2, key)) + # store in options.ips the list of server IP addresses try: options.ips = list(set( # no duplicates [addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn()) @@ -862,10 +870,11 @@ web2py will attempt to run a GUI to ask for it options.nobanner = True options.nogui = True - # accept --interfaces in the form - # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" - # (no spaces; optional key:cert:ca_cert indicate SSL) - if isinstance(options.interfaces, str): + # transform options.interfaces, in the form + # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" + # (no spaces; optional key:cert:ca_cert indicate SSL), into + # a list of tuples + if options.interfaces: interfaces = options.interfaces.split(';') options.interfaces = [] for interface in interfaces: @@ -881,16 +890,16 @@ web2py will attempt to run a GUI to ask for it interface[1] = int(interface[1]) # numeric port options.interfaces.append(tuple(interface)) - # accepts --scheduler in the form - # "app:group1:group2,app2:group1" - scheduler = [] - options.scheduler_groups = None - if isinstance(options.scheduler, str): - if ':' in options.scheduler: - for opt in options.scheduler.split(','): - scheduler.append(opt.split(':')) - options.scheduler = ','.join([app[0] for app in scheduler]) - options.scheduler_groups = scheduler + # strip group infos from options.scheduler, in the form + # "app:group1:group2,app2:group1", and put into a list of lists + # in options.scheduler_groups + if options.scheduler and ':' in options.scheduler: + sg = options.scheduler_groups = [] + for awg in options.scheduler.split(','): + sg.append(awg.split(':')) + options.scheduler = ','.join([app[0] for app in sg]) + else: + options.scheduler_groups = None if options.numthreads is not None and options.minthreads is None: options.minthreads = options.numthreads # legacy @@ -898,8 +907,6 @@ web2py will attempt to run a GUI to ask for it copy_options = copy.deepcopy(options) copy_options.password = '******' global_settings.cmd_options = copy_options - # FIXME: do we still really need this? - global_settings.cmd_args = args return options, args @@ -930,9 +937,8 @@ def start_schedulers(options): sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') return processes = [] - apps = [(app.strip(), None) for app in options.scheduler.split(',')] - if options.scheduler_groups: - apps = options.scheduler_groups + apps = options.scheduler_groups or \ + [(app.strip(), None) for app in options.scheduler.split(',')] code = "from gluon.globals import current;current._scheduler.loop()" logging.getLogger().setLevel(options.debuglevel) if options.folder: @@ -1007,11 +1013,26 @@ def start(cron=True): run_system_tests(options) if options.quiet: - capture = StringIO() - sys.stdout = capture - logger.setLevel(logging.CRITICAL + 1) - else: - logger.setLevel(options.debuglevel) + # to prevent writes on stdout set a null stream + class NullFile(object): + def write(self, x): + pass + sys.stdout = NullFile() + # but still has to mute existing loggers, to do that iterate + # over all existing loggers (root logger included) and remove + # all attached logging.StreamHandler instances currently + # streaming on sys.stdout or sys.stderr + loggers = [logging.getLogger()] + loggers.extend(logging.Logger.manager.loggerDict.values()) + for logger in loggers: + if isinstance(logger, logging.PlaceHolder): continue + for h in logger.handlers[:]: + if isinstance(h, logging.StreamHandler) and \ + h.stream in (sys.stdout, sys.stderr): + logger.removeHandler(h) + # NOTE: stderr.write() is still working + + logger.setLevel(options.debuglevel) if not options.nobanner: # banner @@ -1144,7 +1165,7 @@ end tell ip = first_if[0] port = first_if[1] - if options.ssl_certificate or options.ssl_private_key: + if options.ssl_certificate and options.ssl_private_key: proto = 'https' else: proto = 'http' From 33335ec3e17af5e13435c1749cb21d762a6aaed7 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 11 Apr 2019 21:16:54 -0700 Subject: [PATCH 031/111] removed redundant os.chdir, thanks paolo --- gluon/widget.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gluon/widget.py b/gluon/widget.py index c87028e9..093ce041 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -941,8 +941,6 @@ def start_schedulers(options): [(app.strip(), None) for app in options.scheduler.split(',')] code = "from gluon.globals import current;current._scheduler.loop()" logging.getLogger().setLevel(options.debuglevel) - if options.folder: - os.chdir(options.folder) if len(apps) == 1 and not options.with_scheduler: app_, code = get_code_for_scheduler(apps[0], options) if not app_: @@ -1049,8 +1047,6 @@ def start(cron=True): if options.shell: # run interactive shell and exit - if options.folder: - os.chdir(options.folder) sys.argv = [options.run] + options.args run(options.shell, plain=options.plain, bpython=options.bpython, import_models=options.import_models, startfile=options.run, From 2fbaced689e0871952423ea55591990919a80bdc Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 11 Apr 2019 21:18:36 -0700 Subject: [PATCH 032/111] removed naming conflict from previous commit, thanks Paolo --- gluon/widget.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gluon/widget.py b/gluon/widget.py index 093ce041..87558e18 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -1022,12 +1022,12 @@ def start(cron=True): # streaming on sys.stdout or sys.stderr loggers = [logging.getLogger()] loggers.extend(logging.Logger.manager.loggerDict.values()) - for logger in loggers: - if isinstance(logger, logging.PlaceHolder): continue - for h in logger.handlers[:]: + for l in loggers: + if isinstance(l, logging.PlaceHolder): continue + for h in l.handlers[:]: if isinstance(h, logging.StreamHandler) and \ h.stream in (sys.stdout, sys.stderr): - logger.removeHandler(h) + l.removeHandler(h) # NOTE: stderr.write() is still working logger.setLevel(options.debuglevel) From 8c29f8b12a60b5f42040870b51a3500837835855 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 13 Apr 2019 18:04:43 -0700 Subject: [PATCH 033/111] fixed order or sorting in grid --- gluon/sqlhtml.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 19d333d9..caa1249a 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -2666,7 +2666,7 @@ class SQLFORM(FORM): otablename, ofieldname = order.split('~')[-1].split('.', 1) sort_field = db[otablename][ofieldname] orderby = sort_field if order[:1] != '~' else ~sort_field - + orderby = fix_orderby(orderby) # expcolumns start with the visible columns, which @@ -2798,20 +2798,12 @@ class SQLFORM(FORM): if order and not order == 'None': otablename, ofieldname = order.split('~')[-1].split('.', 1) sort_field = db[otablename][ofieldname] - # invert order direction on date/time fields orderby = sort_field if order[:1] != '~' else ~sort_field headcols = [] if selectable: headcols.append(TH(_class=ui.get('default'))) - ordermatch = orderby; marker = '' - if orderby: - # if orderby is a single column, remember to put the marker - if isinstance(orderby, Expression): - if orderby.first and not orderby.second: - ordermatch = orderby.first; marker = '~' - ordermatch = marker + str(ordermatch) for field in columns: if not field.readable: continue @@ -2819,19 +2811,23 @@ class SQLFORM(FORM): header = headers.get(key, field.label or key) if sortable and not isinstance(field, Field.Virtual): marker = '' - if order: - if key == order: - key = '~' + order; marker = asc_icon - elif key == order[1:]: - key = 'None'; marker = desc_icon - else: - if key == ordermatch: - key = '~' + ordermatch; marker = asc_icon - elif key == ordermatch[1:]: - marker = desc_icon + inverted = field.type in ('date', 'datetime', 'time') + if key == order.lstrip('~'): + if inverted: + if key == order: + key, marker = 'None', asc_icon + else: + key, marker = order[1:], desc_icon + else: + if key == order: + key, marker = '~' + order, asc_icon + else: + key, marker = 'None', desc_icon + elif inverted and key == str(field): + key = '~' + key header = A(header, marker, _href=url(vars=dict( - keywords=keywords, - order=key)), cid=request.cid) + keywords=keywords, + order=key)), cid=request.cid) headcols.append(TH(header, _class=ui.get('default'))) toadd = [] From 563de284f7230ed3976d696fd917275e57d81b63 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 17 Apr 2019 21:58:57 -0700 Subject: [PATCH 034/111] better logic for passing parameters to scheduler, thanks Paolo --- gluon/packages/dal | 2 +- gluon/restricted.py | 2 +- gluon/widget.py | 73 ++++++++++++++++++--------------------------- 3 files changed, 31 insertions(+), 46 deletions(-) diff --git a/gluon/packages/dal b/gluon/packages/dal index 37784cb6..f3401dd8 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit 37784cb6aaa37340eb706eb550164a3f56be4186 +Subproject commit f3401dd8c05d089cb49cfd1215e2991388eace09 diff --git a/gluon/restricted.py b/gluon/restricted.py index 115470e3..a06312fd 100644 --- a/gluon/restricted.py +++ b/gluon/restricted.py @@ -167,7 +167,7 @@ class RestrictedError(Exception): ticket_storage = TicketStorage(db=request.tickets_db) ticket_storage.store(request, request.uuid.split('/', 1)[1], d) cmd_opts = global_settings.cmd_options - if cmd_opts and cmd_opts.print_errors: + if cmd_opts and cmd_opts.errors_to_console: logger.error(self.traceback) return request.uuid except: diff --git a/gluon/widget.py b/gluon/widget.py index 87558e18..68e3b87a 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -316,8 +316,8 @@ class web2pyDialog(object): # the widget takes care of starting the scheduler if self.options.scheduler and self.options.with_scheduler: apps = [app.strip() for app - in self.options.scheduler.split(',') - if app in available_apps] + in self.options.scheduler.split(',').split(':', 1)[0] + if app.strip() in available_apps] for app in apps: self.try_start_scheduler(app) @@ -687,7 +687,7 @@ web2py will attempt to run a GUI to ask for it when starting the web server action='store_true', help='disable all output') - parser.add_option('-e', '--errors_to_console', dest='print_errors', + parser.add_option('-e', '--errors_to_console', default=False, action='store_true', help='log all errors to console') @@ -826,11 +826,8 @@ web2py will attempt to run a GUI to ask for it when starting the web server default=False, action='store_true', help=\ - 'adds coverage reporting (should be used with --run_system_tests), ' \ - 'needs Python 2.7+ and the coverage module installed. ' \ - 'You can alter the default path setting the environment ' \ - 'variable "COVERAGE_PROCESS_START" ' \ - '(by default it takes gluon/tests/coverage.ini)') + 'collect coverage data when used with --run_system_tests; ' \ + 'require Python 2.7+ and the coverage module installed') if '-A' in sys.argv: k = sys.argv.index('-A') @@ -890,17 +887,6 @@ web2py will attempt to run a GUI to ask for it when starting the web server interface[1] = int(interface[1]) # numeric port options.interfaces.append(tuple(interface)) - # strip group infos from options.scheduler, in the form - # "app:group1:group2,app2:group1", and put into a list of lists - # in options.scheduler_groups - if options.scheduler and ':' in options.scheduler: - sg = options.scheduler_groups = [] - for awg in options.scheduler.split(','): - sg.append(awg.split(':')) - options.scheduler = ','.join([app[0] for app in sg]) - else: - options.scheduler_groups = None - if options.numthreads is not None and options.minthreads is None: options.minthreads = options.numthreads # legacy @@ -916,18 +902,17 @@ def check_existent_app(options, appname): return True -def get_code_for_scheduler(app, options): - if len(app) == 1 or app[1] is None: - code = "from gluon.globals import current;current._scheduler.loop()" - else: - code = "from gluon.globals import current;current._scheduler.group_names = ['%s'];" - code += "current._scheduler.loop()" - code = code % ("','".join(app[1:])) - app_ = app[0] - if not check_existent_app(options, app_): - print("Application '%s' doesn't exist, skipping" % app_) +def get_code_for_scheduler(app_groups, options): + app = app_groups[0] + if not check_existent_app(options, app): + print("Application '%s' doesn't exist, skipping" % app) return None, None - return app_, code + code = 'from gluon.globals import current;' + if len(app_groups) > 1: + code += "current._scheduler.group_names=['%s'];" % "','".join( + app_groups[1:]) + code += "current._scheduler.loop()" + return app, code def start_schedulers(options): @@ -936,17 +921,16 @@ def start_schedulers(options): except: sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') return - processes = [] - apps = options.scheduler_groups or \ - [(app.strip(), None) for app in options.scheduler.split(',')] - code = "from gluon.globals import current;current._scheduler.loop()" logging.getLogger().setLevel(options.debuglevel) + + apps = [[n.strip() for n in sched_app.split(':')] + for sched_app in options.scheduler.split(',')] if len(apps) == 1 and not options.with_scheduler: - app_, code = get_code_for_scheduler(apps[0], options) - if not app_: + app, code = get_code_for_scheduler(apps[0], options) + if not app: return - print('starting single-scheduler for "%s"...' % app_) - run(app_, True, True, None, False, code) + print('starting single-scheduler for "%s"...' % app) + run(app, True, True, None, False, code) return # Work around OS X problem: http://bugs.python.org/issue9405 @@ -956,12 +940,13 @@ def start_schedulers(options): import urllib.request as urllib urllib.getproxies() - for app in apps: - app_, code = get_code_for_scheduler(app, options) - if not app_: + processes = [] + for app_groups in apps: + app, code = get_code_for_scheduler(app_groups, options) + if not app: continue - print('starting scheduler for "%s"...' % app_) - args = (app_, True, True, None, False, code) + print('starting scheduler for "%s"...' % app) + args = (app, True, True, None, False, code) p = Process(target=run, args=args) processes.append(p) print("Currently running %s scheduler processes" % (len(processes))) @@ -1060,7 +1045,7 @@ def start(cron=True): if options.scheduler: # run cron for applications listed with --scheduler (-K) apps = [app.strip() for app in options.scheduler.split( - ',') if check_existent_app(options, app.strip())] + ',').split(':', 1)[0] if check_existent_app(options, app.strip())] else: apps = None extcron = newcron.extcron(options.folder, apps=apps) From 2f351172821472cc629b4f068b781ab13b1987a8 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 18 Apr 2019 21:32:46 -0700 Subject: [PATCH 035/111] better options and regexp, thanks Paolo --- gluon/globals.py | 20 ++++++++++++-------- gluon/newcron.py | 22 +++++++++++++--------- gluon/packages/dal | 2 +- gluon/widget.py | 41 ++++++++++++++++++----------------------- 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/gluon/globals.py b/gluon/globals.py index 44c9b794..fc14ceff 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -56,8 +56,6 @@ except ImportError: have_minify = False -regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$') - __all__ = ['Request', 'Response', 'Session'] current = threading.local() # thread-local storage for request-scope globals @@ -357,9 +355,11 @@ class Request(Storage): """ cmd_opts = global_settings.cmd_options # checking if this is called within the scheduler or within the shell - # in addition to checking if it's not a cronjob - if ((cmd_opts and (cmd_opts.shell or cmd_opts.scheduler)) - or global_settings.cronjob or self.is_https): + # in addition to checking if it's a cronjob + # FIXME: cmd_opts.scheduler does not imply that + # we are running in the scheduler + if (self.is_https or cmd_opts and ( + cmd_opts.shell or cmd_opts.scheduler or cmd_opts.cronjob)): current.session.secure() else: current.session.forget() @@ -602,6 +602,7 @@ class Response(Storage): # for attachment settings and backward compatibility keys = [item.lower() for item in headers] if attachment: + # FIXME: should be done like in next download method if filename is None: attname = "" else: @@ -654,6 +655,7 @@ class Response(Storage): Downloads from http://..../download/filename """ + from pydal.helpers.regex import REGEX_UPLOAD_PATTERN from pydal.exceptions import NotAuthorizedException, NotFoundException current.session.forget(current.response) @@ -661,10 +663,10 @@ class Response(Storage): if not request.args: raise HTTP(404) name = request.args[-1] - items = re.compile('(?P.*?)\.(?P.*?)\..*').match(name) + items = re.match(REGEX_UPLOAD_PATTERN, name) if not items: raise HTTP(404) - (t, f) = (items.group('table'), items.group('field')) + t = items.group('table'); f = items.group('field') try: field = db[t][f] except (AttributeError, KeyError): @@ -801,6 +803,8 @@ class Session(Storage): - session_filename """ + REGEX_SESSION_FILE = r'^(?:[\w-]+/)?[\w.-]+$' + def connect(self, request=None, response=None, @@ -891,7 +895,7 @@ class Session(Storage): response.session_file = None # check if the session_id points to a valid sesion filename if response.session_id: - if not regex_session_id.match(response.session_id): + if not re.match(self.REGEX_SESSION_FILE, response.session_id): response.session_id = None else: response.session_filename = \ diff --git a/gluon/newcron.py b/gluon/newcron.py index a415b530..5eba24e0 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -31,7 +31,6 @@ _cron_subprocs = [] def absolute_path_link(path): """ Returns an absolute path for the destination of a symlink - """ if os.path.islink(path): link = os.readlink(path) @@ -73,16 +72,17 @@ class extcron(threading.Thread): class hardcron(threading.Thread): - def __init__(self, applications_parent): + def __init__(self, applications_parent, apps=None): threading.Thread.__init__(self) self.setDaemon(True) self.path = applications_parent - crondance(self.path, 'hard', startup=True) + self.apps = apps + crondance(self.path, 'hard', startup=True, apps=self.apps) def launch(self): if not _cron_stopping: logger.debug('hard cron invocation') - crondance(self.path, 'hard', startup=False) + crondance(self.path, 'hard', startup=False, apps=self.apps) def run(self): s = sched.scheduler(time.time, time.sleep) @@ -95,15 +95,16 @@ class hardcron(threading.Thread): class softcron(threading.Thread): - def __init__(self, applications_parent): + def __init__(self, applications_parent, apps=None): threading.Thread.__init__(self) self.path = applications_parent - # crondance(self.path, 'soft', startup=True) + self.apps = apps + # crondance(self.path, 'soft', startup=True, apps=self.apps) def run(self): if not _cron_stopping: logger.debug('soft cron invocation') - crondance(self.path, 'soft', startup=False) + crondance(self.path, 'soft', startup=False, apps=self.apps) class Token(object): @@ -270,7 +271,10 @@ class cronlauncher(threading.Thread): def crondance(applications_parent, ctype='soft', startup=False, apps=None): - # TODO: docstring + """ + Does the periodic job of cron service: read the crontab(s) and launch + the various commands. + """ apppath = os.path.join(applications_parent, 'applications') token = Token(applications_parent) cronmaster = token.acquire(startup=startup) @@ -299,7 +303,7 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): base_commands.append(w2p_path) if applications_parent != global_settings.gluon_parent: base_commands.extend(('-f', applications_parent)) - base_commands.extend(('-J', + base_commands.extend(('--cronjob', '--no-banner', '--nogui', '--plain', # FIXME: this should not be needed since we are # not launching the web server '-a', '""')) diff --git a/gluon/packages/dal b/gluon/packages/dal index f3401dd8..f1cf5aab 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit f3401dd8c05d089cb49cfd1215e2991388eace09 +Subproject commit f1cf5aab12b839ec168cc194f10c33e026b3de6b diff --git a/gluon/widget.py b/gluon/widget.py index 68e3b87a..bed5a665 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -216,6 +216,14 @@ class web2pyDialog(object): self.bannerarea.after(1000, self.update_canvas) # IP + # retrieves the list of server IP addresses + try: + if_ips = list(set( # no duplicates + [addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn()) + if not is_loopback_ip_address(addrinfo=addrinfo)])) + except socket.gaierror: + if_ips = [] + tkinter.Label(self.root, text='Server IP:', bg=bg_color, justify=tkinter.RIGHT).grid(row=4, @@ -226,7 +234,7 @@ class web2pyDialog(object): row = 4 ips = [('127.0.0.1', 'Local (IPv4)')] + \ ([('::1', 'Local (IPv6)')] if socket.has_ipv6 else []) + \ - [(ip, 'Public') for ip in options.ips] + \ + [(ip, 'Public') for ip in if_ips] + \ [('0.0.0.0', 'Public')] for ip, legend in ips: self.ips[ip] = tkinter.Radiobutton( @@ -704,7 +712,7 @@ web2py will attempt to run a GUI to ask for it when starting the web server metavar='APPNAME', help=\ 'run web2py in interactive shell or IPython (if installed) with ' \ 'specified appname (if app does not exist it will be created). ' \ - 'APPNAME like a/c/f?x=y (c,f and vars x,y optional)') + 'APPNAME like a/c/f?x=y (c, f and vars optional)') parser.add_option('-B', '--bpython', default=False, @@ -752,7 +760,7 @@ web2py will attempt to run a GUI to ask for it when starting the web server default=None, metavar='TEST_PATH', help=\ 'run doctests in web2py environment; ' \ - 'TEST_PATH like a/c/f (c,f optional)') + 'TEST_PATH like a/c/f (c, f optional)') parser.add_option('-C', '--cron', dest='extcron', default=False, @@ -777,7 +785,9 @@ web2py will attempt to run a GUI to ask for it when starting the web server parser.add_option('-J', '--cronjob', default=False, action='store_true', - help='identify cron-initiated command') + # NOTE: help suppressed because this option is + # intended for internal use only + help=optparse.SUPPRESS_HELP) parser.add_option('-L', '--config', default='', @@ -840,6 +850,10 @@ web2py will attempt to run a GUI to ask for it when starting the web server # TODO: warn or error if args (should be no unparsed arguments) options.args = other_args + if options.taskbar and os.name != 'nt': + # TODO: warn and disable taskbar instead of exit + die('taskbar not supported on this platform') + if options.config.endswith('.py'): options.config = options.config[:-3] if options.config: @@ -853,20 +867,6 @@ web2py will attempt to run a GUI to ask for it when starting the web server if hasattr(options, key): setattr(options, key, getattr(options2, key)) - # store in options.ips the list of server IP addresses - try: - options.ips = list(set( # no duplicates - [addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn()) - if not is_loopback_ip_address(addrinfo=addrinfo)])) - except socket.gaierror: - options.ips = [] - - if options.cronjob: - global_settings.cronjob = True # tell the world - options.plain = True # cronjobs use a plain shell - options.nobanner = True - options.nogui = True - # transform options.interfaces, in the form # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" # (no spaces; optional key:cert:ca_cert indicate SSL), into @@ -1074,11 +1074,6 @@ def start(cron=True): # if no password provided and have Tk library start GUI (when not # explicitly disabled), we also need a GUI to put in taskbar (system tray) # when requested - - # FIXME: this check should be done first - if options.taskbar and os.name != 'nt': - die('taskbar not supported on this platform') - root = None if (not options.nogui and options.password == '') or options.taskbar: From 0d04b8a511de9f018343515a071dd01632754f33 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 19 Apr 2019 21:22:29 -0700 Subject: [PATCH 036/111] widget fix, thanks Paolo --- gluon/widget.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gluon/widget.py b/gluon/widget.py index bed5a665..15ee70d4 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -323,9 +323,9 @@ class web2pyDialog(object): if start: # the widget takes care of starting the scheduler if self.options.scheduler and self.options.with_scheduler: - apps = [app.strip() for app - in self.options.scheduler.split(',').split(':', 1)[0] - if app.strip() in available_apps] + apps = [app for app + in map(lambda ag : ag.split(':', 1)[0].strip(), self.options.scheduler.split(',')) + if app in available_apps] for app in apps: self.try_start_scheduler(app) @@ -1044,8 +1044,9 @@ def start(cron=True): global_settings.web2py_crontype = 'external' if options.scheduler: # run cron for applications listed with --scheduler (-K) - apps = [app.strip() for app in options.scheduler.split( - ',').split(':', 1)[0] if check_existent_app(options, app.strip())] + apps = [app for app + in map(lambda ag : ag.split(':', 1)[0].strip(), options.scheduler.split(',')) + if check_existent_app(options, app)] else: apps = None extcron = newcron.extcron(options.folder, apps=apps) From 8a9b2d687e2c0b850059f15234dba2ee1d1172c2 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 19 Apr 2019 21:23:59 -0700 Subject: [PATCH 037/111] better option handling for the shell, thanks Paolo --- gluon/shell.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/gluon/shell.py b/gluon/shell.py index bcac912e..36a5be44 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -135,11 +135,14 @@ def env( request.function = f or 'index' response.view = '%s/%s.html' % (request.controller, request.function) - if global_settings.cmd_options: - ip = global_settings.cmd_options.ip - port = global_settings.cmd_options.port - request.is_shell = global_settings.cmd_options.shell is not None - request.is_scheduler = global_settings.cmd_options.scheduler is not None + cmd_opts = global_settings.cmd_options + if cmd_opts: + ip = cmd_opts.ip + port = cmd_opts.port + request.is_shell = cmd_opts.shell is not None + # FIXME: cmd_opts.scheduler does not imply that + # we are running in the scheduler + request.is_scheduler = cmd_opts.scheduler is not None else: ip, port = '127.0.0.1', '8000' request.env.http_host = '%s:%s' % (ip, port) @@ -206,14 +209,18 @@ def run( import_models=False, startfile=None, bpython=False, - python_code=False, - cronjob=False): + python_code=None, + cronjob=False, + scheduler_job=False): """ Start interactive shell or run Python script (startfile) in web2py controller environment. appname is formatted like: - a : web2py application name - a/c : exec the controller c into the application environment + - a/c/f : exec the controller c, then the action f + into the application environment + - a/c/f?x=y : as above """ (a, c, f, args, vars) = parse_path_info(appname, av=True) @@ -223,7 +230,8 @@ def run( adir = os.path.join('applications', a) if not os.path.exists(adir): - if sys.stdin and not sys.stdin.name == '/dev/null': + if not scheduler_job and \ + sys.stdin and not sys.stdin.name == '/dev/null': confirm = raw_input( 'application %s does not exist, create (y/n)?' % a) else: @@ -242,6 +250,8 @@ def run( db = os.path.join(adir, 'models/db.py') if os.path.exists(db): data = fileutils.read_file(db) + # NOTE: is this for backward compatibility ? + # there is no need for import gluon.utils otherwise data = data.replace( '', 'sha512:' + web2py_uuid()) fileutils.write_file(db, data) From 7c1bb810fc0357c8eace7cdfb575da41f5e3806e Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 20 Apr 2019 13:53:43 -0700 Subject: [PATCH 038/111] cleaner fileutils and untarring/unzipping in fileutils, thanks Paolo --- gluon/fileutils.py | 138 ++++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 76 deletions(-) diff --git a/gluon/fileutils.py b/gluon/fileutils.py index c1644be7..9424d32e 100644 --- a/gluon/fileutils.py +++ b/gluon/fileutils.py @@ -18,12 +18,13 @@ import glob import time import datetime import logging +import shutil from gluon.http import HTTP from gzip import open as gzopen from gluon.recfile import generate from gluon._compat import PY2 -__all__ = [ +__all__ = ( 'parse_version', 'read_file', 'write_file', @@ -41,11 +42,11 @@ __all__ = [ 'check_credentials', 'w2p_pack', 'w2p_unpack', + 'create_app', 'w2p_pack_plugin', 'w2p_unpack_plugin', 'fix_newlines', - 'make_fake_file_like_object', -] +) def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"): @@ -58,7 +59,7 @@ def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"): tuple: Major, Minor, Patch, Release, Build Date """ - re_version = re.compile('(\d+)\.(\d+)\.(\d+)(\-(?P
[^\s+]*))?(\+(?P\S*))')
+    re_version = re.compile(r'(\d+)\.(\d+)\.(\d+)(-(?P
[^\s+]*))?(\+(?P\S*))')
     m = re_version.match(version.strip().split()[-1])
     if not m:
         return None
@@ -80,7 +81,7 @@ def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
         tuple: Major, Minor, Patch, Release, Build Date
 
     """
-    re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P.+?)\)\s*(?P[a-z]+)?')
+    re_version = re.compile(r'[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P.+?)\)\s*(?P[a-z]+)?')
     m = re_version.match(version)
     a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
     pre_release = m.group('type') or 'dev'
@@ -109,22 +110,16 @@ def read_file(filename, mode='r'):
     """Returns content from filename, making sure to close the file explicitly
     on exit.
     """
-    f = open_file(filename, mode)
-    try:
+    with open_file(filename, mode) as f:
         return f.read()
-    finally:
-        f.close()
 
 
 def write_file(filename, value, mode='w'):
     """Writes  to filename, making sure to close the file
     explicitly on exit.
     """
-    f = open_file(filename, mode)
-    try:
+    with open_file(filename, mode) as f:
         return f.write(value)
-    finally:
-        f.close()
 
 
 def readlines_file(filename, mode='r'):
@@ -200,10 +195,10 @@ def cleanpath(path):
 
     items = path.split('.')
     if len(items) > 1:
-        path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
+        path = re.sub(r'[^\w.]+', '_', '_'.join(items[:-1]) + '.'
                       + ''.join(items[-1:]))
     else:
-        path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
+        path = re.sub(r'[^\w.]+', '_', ''.join(items[-1:]))
     return path
 
 
@@ -250,59 +245,64 @@ def w2p_pack(filename, path, compiled=False, filenames=None):
     path = abspath(path)
     tarname = filename + '.tar'
     if compiled:
-        tar_compiled(tarname, path, '^[\w\.\-]+$',
+        tar_compiled(tarname, path, r'^[\w.-]+$',
                      exclude_content_from=['cache', 'sessions', 'errors'])
     else:
-        tar(tarname, path, '^[\w\.\-]+$', filenames=filenames,
+        tar(tarname, path, r'^[\w.-]+$', filenames=filenames,
             exclude_content_from=['cache', 'sessions', 'errors'])
-    w2pfp = gzopen(filename, 'wb')
-    tarfp = open(tarname, 'rb')
-    w2pfp.write(tarfp.read())
-    w2pfp.close()
-    tarfp.close()
+    with open(tarname, 'rb') as tarfp, gzopen(filename, 'wb') as gzfp:
+        shutil.copyfileobj(tarfp, gzfp, 4194304) # 4 MB buffer
     os.unlink(tarname)
 
 
 def create_welcome_w2p():
-    is_newinstall_file = os.path.exists('NEWINSTALL')
-    if not os.path.exists('welcome.w2p') or is_newinstall_file:
+    is_newinstall = os.path.exists('NEWINSTALL')
+    if not os.path.exists('welcome.w2p') or is_newinstall:
+        logger = logging.getLogger("web2py")
         try:
             w2p_pack('welcome.w2p', 'applications/welcome')
-            logging.info("New installation: created welcome.w2p file")
+            logger.info("New installation: created welcome.w2p file")
         except:
-            logging.error("New installation error: unable to create welcome.w2p file")
+            logger.exception("New installation error: unable to create welcome.w2p file")
             return
-        if is_newinstall_file:
+        if is_newinstall:
             try:
                 os.unlink('NEWINSTALL')
-                logging.info("New installation: removed NEWINSTALL file")
+                logger.info("New installation: removed NEWINSTALL file")
             except:
-                logging.error("New installation error: unable to remove NEWINSTALL file")
+                logger.exception("New installation error: unable to remove NEWINSTALL file")
 
 
 def w2p_unpack(filename, path, delete_tar=True):
-
     if filename == 'welcome.w2p':
         create_welcome_w2p()
     filename = abspath(filename)
-    path = abspath(path)
-    if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
-        if filename[-4:] == '.w2p':
-            tarname = filename[:-4] + '.tar'
-        else:
-            tarname = filename[:-3] + '.tar'
-        fgzipped = gzopen(filename, 'rb')
-        tarfile = open(tarname, 'wb')
-        tarfile.write(fgzipped.read())
-        tarfile.close()
-        fgzipped.close()
+    tarname = None
+    if filename.endswith('.w2p'):
+        tarname = filename[:-4] + '.tar'
+    elif filename.endswith('.gz'):
+        tarname = filename[:-3] + '.tar'
+    if tarname is not None:
+        with gzopen(filename, 'rb') as gzfp, open(tarname, 'wb') as tarfp:
+            shutil.copyfileobj(gzfp, tarfp, 4194304) # 4 MB buffer
     else:
         tarname = filename
+    path = abspath(path)
     untar(tarname, path)
     if delete_tar:
         os.unlink(tarname)
 
 
+def create_app(path):
+    w2p_unpack('welcome.w2p', path)
+    for subfolder in ('models', 'views', 'controllers', 'databases',
+                      'modules', 'cron', 'errors', 'sessions',
+                      'languages', 'static', 'private', 'uploads'):
+        subpath = os.path.join(path, subfolder)
+        if not os.path.exists(subpath):
+            os.mkdir(subpath)
+
+
 def w2p_pack_plugin(filename, path, plugin_name):
     """Packs the given plugin into a w2p file.
     Will match files at::
@@ -314,11 +314,10 @@ def w2p_pack_plugin(filename, path, plugin_name):
     filename = abspath(filename)
     path = abspath(path)
     if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
-        raise Exception("Not a web2py plugin name")
-    plugin_tarball = tarfile.open(filename, 'w:gz')
-    try:
+        raise ValueError('Not a web2py plugin')
+    with tarfile.open(filename, 'w:gz') as plugin_tarball:
         app_dir = path
-        while app_dir[-1] == '/':
+        while app_dir.endswith('/'):
             app_dir = app_dir[:-1]
         files1 = glob.glob(
             os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
@@ -326,15 +325,13 @@ def w2p_pack_plugin(filename, path, plugin_name):
             os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
         for file in files1 + files2:
             plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
-    finally:
-        plugin_tarball.close()
 
 
 def w2p_unpack_plugin(filename, path, delete_tar=True):
     filename = abspath(filename)
     path = abspath(path)
     if not os.path.basename(filename).startswith('web2py.plugin.'):
-        raise Exception("Not a web2py plugin")
+        raise ValueError('Not a web2py plugin')
     w2p_unpack(filename, path, delete_tar)
 
 
@@ -344,23 +341,22 @@ def tar_compiled(file, dir, expression='^.+$',
     The content of models, views, controllers is not stored in the tar file.
     """
 
-    tar = tarfile.TarFile(file, 'w')
-    for file in listdir(dir, expression, add_dirs=True,
-                        exclude_content_from=exclude_content_from):
-        filename = os.path.join(dir, file)
-        if os.path.islink(filename):
-            continue
-        if os.path.isfile(filename) and file[-4:] != '.pyc':
-            if file[:6] == 'models':
+    with tarfile.TarFile(file, 'w') as tar:
+        for file in listdir(dir, expression, add_dirs=True,
+                            exclude_content_from=exclude_content_from):
+            filename = os.path.join(dir, file)
+            if os.path.islink(filename):
                 continue
-            if file[:5] == 'views':
-                continue
-            if file[:11] == 'controllers':
-                continue
-            if file[:7] == 'modules':
-                continue
-        tar.add(filename, file, False)
-    tar.close()
+            if os.path.isfile(filename) and not file.endswith('.pyc'):
+                if file.startswith('models'):
+                    continue
+                if file.startswith('views'):
+                    continue
+                if file.startswith('controllers'):
+                    continue
+                if file.startswith('modules'):
+                    continue
+            tar.add(filename, file, False)
 
 
 def up(path):
@@ -378,7 +374,7 @@ def get_session(request, other_application='admin'):
         if not os.path.exists(session_filename):
             session_filename = generate(session_filename)
         osession = storage.load_storage(session_filename)
-    except Exception as e:
+    except:
         osession = storage.Storage()
     return osession
 
@@ -421,7 +417,7 @@ def fix_newlines(path):
     regex = re.compile(r'''(\r
 |\r|
 )''')
-    for filename in listdir(path, '.*\.(py|html)$', drop=False):
+    for filename in listdir(path, r'.*\.(py|html)$', drop=False):
         rdata = read_file(filename, 'r')
         wdata = regex.sub('\n', rdata)
         if wdata != rdata:
@@ -455,16 +451,6 @@ def copystream(
     return
 
 
-def make_fake_file_like_object():
-    class LogFile(object):
-        def write(self, value):
-            pass
-
-        def close(self):
-            pass
-    return LogFile()
-
-
 from gluon.settings import global_settings  # we need to import settings here because
                                       # settings imports fileutils too
 

From 9d5a16351bc695b94e7f46420d6058765df8231a Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Sun, 21 Apr 2019 09:24:56 -0700
Subject: [PATCH 039/111] simplify admin by using create_app, thanks Paolo

---
 gluon/admin.py | 27 ++++++++-------------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/gluon/admin.py b/gluon/admin.py
index 01e4eccb..0c09bb69 100644
--- a/gluon/admin.py
+++ b/gluon/admin.py
@@ -14,11 +14,11 @@ import os
 import sys
 import traceback
 import zipfile
-from shutil import rmtree
-from gluon.utils import web2py_uuid
-from gluon.fileutils import w2p_pack, w2p_unpack, w2p_pack_plugin, w2p_unpack_plugin
-from gluon.fileutils import up, fix_newlines, abspath, recursive_unlink
-from gluon.fileutils import read_file, write_file, parse_version
+from shutil import rmtree, copyfileobj
+from gluon.fileutils import (w2p_pack, create_app, w2p_unpack,
+    w2p_pack_plugin, w2p_unpack_plugin,
+    up, fix_newlines, abspath, recursive_unlink,
+    read_file, write_file, parse_version)
 from gluon.restricted import RestrictedError
 from gluon.settings import global_settings
 from gluon.cache import CacheOnDisk
@@ -178,19 +178,7 @@ def app_create(app, request, force=False, key=None, info=False):
         else:
             return False
     try:
-        w2p_unpack('welcome.w2p', path)
-        for subfolder in ['models', 'views', 'controllers', 'databases',
-                          'modules', 'cron', 'errors', 'sessions', 'cache',
-                          'languages', 'static', 'private', 'uploads']:
-            subpath = os.path.join(path, subfolder)
-            if not os.path.exists(subpath):
-                os.mkdir(subpath)
-        db = os.path.join(path, 'models', 'db.py')
-        if os.path.exists(db):
-            data = read_file(db)
-            data = data.replace('',
-                                'sha512:' + (key or web2py_uuid()))
-            write_file(db, data)
+        create_app(path)
         if info:
             return True, None
         else:
@@ -232,7 +220,8 @@ def app_install(app, fobj, request, filename, overwrite=None):
     upname = apath('../deposit/%s.%s' % (app, extension), request)
 
     try:
-        write_file(upname, fobj.read(), 'wb')
+        with open(upname, 'wb') as appfp:
+            copyfileobj(fobj, appfp, 4194304) # 4 MB buffer
         path = apath(app, request)
         if not overwrite:
             os.mkdir(path)

From 14dee6b466c00d5b795809c85ffc1f759ca4a533 Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Sun, 21 Apr 2019 09:27:02 -0700
Subject: [PATCH 040/111] better shell using create_app, thanks Paolo

---
 gluon/shell.py | 40 +++++++++++++++-------------------------
 1 file changed, 15 insertions(+), 25 deletions(-)

diff --git a/gluon/shell.py b/gluon/shell.py
index 36a5be44..89be48dc 100644
--- a/gluon/shell.py
+++ b/gluon/shell.py
@@ -23,7 +23,6 @@ import glob
 import traceback
 import gluon.fileutils as fileutils
 from gluon.settings import global_settings
-from gluon.utils import web2py_uuid
 from gluon.compileapp import build_environment, read_pyc, run_models_in
 from gluon.restricted import RestrictedError
 from gluon.globals import Request, Response, Session
@@ -137,14 +136,18 @@ def env(
                                     request.function)
     cmd_opts = global_settings.cmd_options
     if cmd_opts:
-        ip = cmd_opts.ip
-        port = cmd_opts.port
+        if not cmd_opts.interfaces:
+            ip = cmd_opts.ip
+            port = cmd_opts.port
+        else:
+            first_if = cmd_opts.interfaces[0]
+            ip = first_if[0]
+            port = first_if[1]
         request.is_shell = cmd_opts.shell is not None
-        # FIXME: cmd_opts.scheduler does not imply that
-        #        we are running in the scheduler
-        request.is_scheduler = cmd_opts.scheduler is not None
     else:
-        ip, port = '127.0.0.1', '8000'
+        ip = '127.0.0.1'; port = 8000
+        # FIXME: what about request.is_shell ?
+    request.is_scheduler = False
     request.env.http_host = '%s:%s' % (ip, port)
     request.env.remote_addr = '127.0.0.1'
     request.env.web2py_runtime_gae = global_settings.web2py_runtime_gae
@@ -230,37 +233,24 @@ def run(
     adir = os.path.join('applications', a)
 
     if not os.path.exists(adir):
-        if not scheduler_job and \
+        if not cronjob and not scheduler_job and \
             sys.stdin and not sys.stdin.name == '/dev/null':
             confirm = raw_input(
                 'application %s does not exist, create (y/n)?' % a)
         else:
             logging.warn('application does not exist and will not be created')
             return
-        if confirm.lower() in ['y', 'yes']:
-
+        if confirm.lower() in ('y', 'yes'):
             os.mkdir(adir)
-            w2p_unpack('welcome.w2p', adir)
-            for subfolder in ['models', 'views', 'controllers', 'databases',
-                              'modules', 'cron', 'errors', 'sessions',
-                              'languages', 'static', 'private', 'uploads']:
-                subpath = os.path.join(adir, subfolder)
-                if not os.path.exists(subpath):
-                    os.mkdir(subpath)
-            db = os.path.join(adir, 'models/db.py')
-            if os.path.exists(db):
-                data = fileutils.read_file(db)
-                # NOTE: is this for backward compatibility ?
-                #       there is no need for import gluon.utils otherwise
-                data = data.replace(
-                    '', 'sha512:' + web2py_uuid())
-                fileutils.write_file(db, data)
+            fileutils.create_app(adir)
 
     if c:
         import_models = True
     extra_request = {}
     if args:
         extra_request['args'] = args
+    if scheduler_job:
+        extra_request['is_scheduler'] = True
     if vars:
         # underscore necessary because request.vars is a property
         extra_request['_vars'] = vars

From 0a9975809cf2565b7b7c726b442a4f7b72dc848d Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Tue, 23 Apr 2019 22:10:53 -0700
Subject: [PATCH 041/111] cleanup in main, thanks Paolo

---
 gluon/main.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/gluon/main.py b/gluon/main.py
index 89ef709a..68e4307d 100644
--- a/gluon/main.py
+++ b/gluon/main.py
@@ -28,7 +28,7 @@ import string
 from gluon._compat import Cookie, urllib_quote
 # from thread import allocate_lock
 
-from gluon.fileutils import abspath, write_file
+from gluon.fileutils import abspath, read_file, write_file
 from gluon.settings import global_settings
 from gluon.utils import web2py_uuid, unlocalised_http_header_date
 from gluon.admin import add_path_first, create_missing_folders, create_missing_app_folders
@@ -98,13 +98,9 @@ requests = 0    # gc timer
 # Security Checks: validate URL and session_id here,
 # accept_language is validated in languages
 
-# pattern used to validate client address
-regex_client = re.compile('[\w\-:]+(\.[\w\-]+)*\.?')  # ## to account for IPV6
-
 try:
-    version_info = open(pjoin(global_settings.gluon_parent, 'VERSION'), 'r')
-    raw_version_string = version_info.read().split()[-1].strip()
-    version_info.close()
+    version_info = read_file(pjoin(global_settings.gluon_parent, 'VERSION'))
+    raw_version_string = version_info.split()[-1].strip()
     global_settings.web2py_version = raw_version_string
     web2py_version = global_settings.web2py_version
 except:
@@ -121,6 +117,9 @@ load_routes()
 HTTPS_SCHEMES = set(('https', 'HTTPS'))
 
 
+# pattern used to validate client address
+REGEX_CLIENT = re.compile(r'[\w:-]+(\.[\w-]+)*\.?')  # ## to account for IPV6
+
 def get_client(env):
     """
     Guesses the client address from the environment variables
@@ -129,12 +128,12 @@ def get_client(env):
     if all fails, assume '127.0.0.1' or '::1' (running locally)
     """
     eget = env.get
-    g = regex_client.search(eget('http_x_forwarded_for', ''))
-    client = (g.group() or '').split(',')[0] if g else None
+    m = REGEX_CLIENT.search(eget('http_x_forwarded_for', ''))
+    client = m and m.group()
     if client in (None, '', 'unknown'):
-        g = regex_client.search(eget('remote_addr', ''))
-        if g:
-            client = g.group()
+        m = REGEX_CLIENT.search(eget('remote_addr', ''))
+        if m:
+            client = m.group()
         elif env.http_host.startswith('['):  # IPv6
             client = '::1'
         else:
@@ -352,7 +351,6 @@ def wsgibase(environ, responder):
                     local_hosts = global_settings.local_hosts
                 client = get_client(env)
                 x_req_with = str(env.http_x_requested_with).lower()
-                cmd_opts = global_settings.cmd_options
 
                 request.update(
                     client=client,
@@ -558,6 +556,7 @@ def wsgibase(environ, responder):
     if not http_response:
         return wsgibase(new_environ, responder)
     if global_settings.web2py_crontype == 'soft':
+        cmd_opts = global_settings.cmd_options
         newcron.softcron(global_settings.applications_parent).start()
     return http_response.to(responder, env=env)
 
@@ -711,7 +710,6 @@ class HttpServer(object):
         if interfaces:
             # if interfaces is specified, it must be tested for rocket parameter correctness
             # not necessarily completely tested (e.g. content of tuples or ip-format)
-            import types
             if isinstance(interfaces, list):
                 for i in interfaces:
                     if not isinstance(i, tuple):

From 89c441cdc834c4eb028f9655788cd12e9199938f Mon Sep 17 00:00:00 2001
From: Nico Zanferrari 
Date: Wed, 24 Apr 2019 19:28:46 +0200
Subject: [PATCH 042/111] PY2 small print fix

---
 gluon/widget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gluon/widget.py b/gluon/widget.py
index 15ee70d4..3cadb87e 100644
--- a/gluon/widget.py
+++ b/gluon/widget.py
@@ -97,7 +97,7 @@ def get_url(host, path='/', proto='http', port=80):
 def start_browser(url, startup=False):
     if startup:
         print('please visit:')
-        print('\t', url)
+        print('\t' + url)
         print('starting browser...')
     try:
         import webbrowser

From 7b9d2c87c2b676b2713498b107eb03508dcc9e58 Mon Sep 17 00:00:00 2001
From: Nico Zanferrari 
Date: Wed, 24 Apr 2019 23:28:48 +0200
Subject: [PATCH 043/111] PY2 compatibility

---
 extras/build_web2py/build_web2py.py | 90 ++++++++++++++++++-----------
 1 file changed, 55 insertions(+), 35 deletions(-)

diff --git a/extras/build_web2py/build_web2py.py b/extras/build_web2py/build_web2py.py
index 1734ab2c..0259fc7f 100644
--- a/extras/build_web2py/build_web2py.py
+++ b/extras/build_web2py/build_web2py.py
@@ -23,11 +23,21 @@ USAGE = """
 build_web2py - make web2py Windows and MacOS binaries with pyinstaller 
 
 Usage:
-    Install pyinstaller program, copy this file to web2py root folder and run:
+    Install the pyinstaller program, copy this file (plus web2py.ngm.spec and web2py.ngm_no_console.spec) 
+    to web2py root folder and run:
     
     python build_py3.py
-        
+    
+    (tested with python 3.7 only)
 """
+BUILD_DEBUG = False
+"""
+If BUILD_DEBUG is set to False, no gluon modules will be embedded inside the binary web2py.exe. 
+    Thus, you can easily update the build version by changing the gluon folder inside the resulting ZIP file.
+In case of problem , set BUILD_DEBUG to True. Then all the gluon modules will be analyzed and embedded, too.
+    You can later analyze the .exe with 'pyi-archive_viewer web2py.exe' and then 'o PYZ-00.pyz'
+    in order to check for missing system modules to be manually inserted in the SPEC file
+ """
 
 if len(sys.argv) != 1 or not os.path.isfile('web2py.py'):
     print(USAGE)
@@ -66,64 +76,73 @@ python_version = sys.version_info[:3]
 
 if os_version == 'Windows':
     print("\nBuilding binary web2py for Windows\n")
-    # to make executable without GUI we need this trick
-    shutil.copy("web2py.py", "web2py_no_console.py")
+    if BUILD_DEBUG: # debug only
+        subprocess.call('pyinstaller --clean  --icon=extras/icons/web2py.ico \
+                        --hidden-import=site-packages --hidden-import=gluon.packages.dal.pydal \
+                        --hidden-import=gluon.packages.yatl.yatl web2py.py')
+        zip_filename = 'web2py_win_debug'
+    else: # normal run    
+        subprocess.call('pyinstaller --clean  web2py.win.spec')
+        subprocess.call('pyinstaller --clean  web2py.win_no_console.spec')
+        source_no_console = 'dist/web2py_no_console/'
+        files = 'web2py_no_console.exe'
+        shutil.move(os.path.join(source_no_console, files), 'dist')
+        shutil.rmtree(source_no_console)
+        shutil.rmtree('build')
+        zip_filename = 'web2py_win'
 
-    subprocess.run('pyinstaller --clean  --icon=extras/icons/web2py.ico --hidden-import=gluon.packages.dal.pydal  \
-                        --hidden-import=gluon.packages.yatl.yatl --hidden-import=site-packages web2py.py')
-    subprocess.run('pyinstaller -w --clean  --icon=extras/icons/web2py.ico --hidden-import=gluon.packages.dal.pydal  \
-                        --hidden-import=gluon.packages.yatl.yatl --hidden-import=site-packages web2py_no_console.py')
-
-    # cleanup + move binary files to dist folder
-    os.unlink('web2py_no_console.py')
-    os.unlink('web2py_no_console.spec')
     source = 'dist/web2py/'
     for files in os.listdir(source):
         shutil.move(os.path.join(source, files), 'dist')
-    source2 = 'dist/web2py_no_console/'
-    files = 'web2py_no_console.exe'
-    shutil.move(os.path.join(source2, files), 'dist')
     shutil.rmtree(source)
-    shutil.rmtree(source2)
     os.unlink('dist/web2py.exe.manifest')
 
-    zip_filename = 'web2py_win'
-    bin_folder = 'dist'
+
+
+    bin_folders = ['dist',]
 
 
 elif os_version == 'Darwin':
     print("\nBuilding binary web2py for MacOS\n")
 
-    import subprocess
-    subprocess.call("pyinstaller --clean  --windowed --icon=extras/icons/web2py.icns --hidden-import=gluon.packages.dal.pydal  --hidden-import=gluon.packages.yatl.yatl \
-                    --hidden-import=site-packages --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' \
-                    --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl'  web2py.py", shell=True)
+    if BUILD_DEBUG: #debug only    
+        subprocess.call("pyinstaller --clean --icon=extras/icons/web2py.icns --hidden-import=gluon.packages.dal.pydal  --hidden-import=gluon.packages.yatl.yatl \
+                        --hidden-import=site-packages --windowed web2py.py", shell=True)
+        zip_filename = 'web2py_osx_debug'
+    else: # normal run
+        subprocess.call("pyinstaller --clean web2py.mac.spec", shell=True)
+        # cleanup + move binary files to dist folder
+        #shutil.rmtree(os.path.join('dist', 'web2py'))
+        shutil.rmtree('build')
+        zip_filename = 'web2py_osx'
 
-    # cleanup + move binary files to dist folder
-    shutil.rmtree(os.path.join('dist', 'web2py'))
-    shutil.rmtree('build')
-
-    zip_filename = 'web2py_osx'
-    bin_folder = (os.path.join('dist', 'web2py.app/Contents/MacOS'))
-
-print("\nWeb2py binary successfully built!\n")
+    shutil.move((os.path.join('dist', 'web2py')),(os.path.join('dist', 'web2py_cmd')))
+    bin_folders = [(os.path.join('dist', 'web2py.app/Contents/MacOS')), (os.path.join('dist', 'web2py_cmd'))]
+    print("\nWeb2py binary successfully built!\n")
 
 
 # add data_files
 for req in ['CHANGELOG', 'LICENSE', 'VERSION']:
-    shutil.copy(req, os.path.join(bin_folder, req))
+    for bin_folder in bin_folders:
+        shutil.copy(req, os.path.join(bin_folder, req))
 # cleanup unuseful binary cache
 for dirpath, dirnames, files in os.walk('.'):
     if dirpath.endswith('__pycache__'):
         print('Deleting cached binary directory : %s' % dirpath)
         shutil.rmtree(dirpath)
-
+for dirpath, dirnames, files in os.walk('.'):
+    for file in files:
+        if file.endswith('.pyc'):
+            print('Deleting cached binary file : %s' % file)
+            os.unlink(os.path.join(dirpath, file))
+        
 print("\nPreparing package ...")
 # misc
 for folders in ['gluon', 'extras', 'site-packages', 'scripts', 'applications', 'examples', 'handlers']:
-    shutil.copytree(folders, os.path.join(bin_folder, folders))
-os.mkdir(os.path.join(bin_folder, 'logs'))
-os.unlink('web2py.spec')
+    for bin_folder in bin_folders:
+        shutil.copytree(folders, os.path.join(bin_folder, folders))
+        if not os.path.exists(os.path.join(bin_folder, 'logs')):
+             os.mkdir(os.path.join(bin_folder, 'logs'))
 
 
 # create a web2py folder & copy dist's files into it
@@ -139,6 +158,7 @@ zipf.close()
 shutil.rmtree('zip_temp')
 shutil.rmtree('dist')
 
+
 print("Your binary version of web2py can be found in " + \
     zip_filename + ".zip")
 print("You may extract the archive anywhere and then run web2py without worrying about dependency")

From b79b5951f83eab37031db6274074a16d5ed3d678 Mon Sep 17 00:00:00 2001
From: Nico Zanferrari 
Date: Wed, 24 Apr 2019 23:35:44 +0200
Subject: [PATCH 044/111] Update docs

---
 extras/build_web2py/build_web2py.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/extras/build_web2py/build_web2py.py b/extras/build_web2py/build_web2py.py
index 0259fc7f..59aeb76d 100644
--- a/extras/build_web2py/build_web2py.py
+++ b/extras/build_web2py/build_web2py.py
@@ -21,14 +21,13 @@ import platform
 
 USAGE = """
 build_web2py - make web2py Windows and MacOS binaries with pyinstaller 
-
 Usage:
-    Install the pyinstaller program, copy this file (plus web2py.ngm.spec and web2py.ngm_no_console.spec) 
+    Install the pyinstaller program, copy this file (plus web2py.*.spec files) 
     to web2py root folder and run:
     
     python build_py3.py
     
-    (tested with python 3.7 only)
+    (tested with python 3.7.3 and 2.7.16 with PyInstaller 3.4)
 """
 BUILD_DEBUG = False
 """

From 9769314f01d2c2edcc3e65233e1ed382c0be3b75 Mon Sep 17 00:00:00 2001
From: Nico Zanferrari 
Date: Wed, 24 Apr 2019 23:38:06 +0200
Subject: [PATCH 045/111] PY2 compatibility

---
 extras/build_web2py/web2py.mac.spec           | 52 +++++++++++++++++++
 extras/build_web2py/web2py.win.spec           | 44 ++++++++++++++++
 .../build_web2py/web2py.win_no_console.spec   | 44 ++++++++++++++++
 3 files changed, 140 insertions(+)
 create mode 100644 extras/build_web2py/web2py.mac.spec
 create mode 100644 extras/build_web2py/web2py.win.spec
 create mode 100644 extras/build_web2py/web2py.win_no_console.spec

diff --git a/extras/build_web2py/web2py.mac.spec b/extras/build_web2py/web2py.mac.spec
new file mode 100644
index 00000000..1a1894b0
--- /dev/null
+++ b/extras/build_web2py/web2py.mac.spec
@@ -0,0 +1,52 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['web2py.py'],
+             pathex=['.'],
+             binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')],
+             datas=[],
+             hiddenimports=['site-packages', 'cgi', 'cgitb', 'code', 'concurrent', 'concurrent.futures', 
+              'concurrent.futures._base', 'concurrent.futures.process', 'concurrent.futures.thread', 'configparser', 'cProfile', 'csv', 'ctypes.wintypes',
+              'email.mime', 'email.mime.base', 'email.mime.multipart', 'email.mime.nonmultipart', 'email.mime.text', 'html.parser', 'http.cookies',
+              'ipaddress', 'imaplib', 'imp', 'json', 'json.decoder', 'json.encoder', 'json.scanner', 'logging.config', 'logging.handlers', 'profile', 'pstats',
+              'psycopg2', 'psycopg2._ipaddress', 'psycopg2._json', 'psycopg2._range', 'psycopg2.extensions', 'psycopg2.extras', 'psycopg2.sql',
+              'psycopg2.tz', 'pyodbc', 'python-ldap', 'rlcompleter', 'sched', 'site', 'smtplib', 'sqlite3', 'sqlite3.dbapi2', 'sqlite3.dump', 'timeit', 'tkinter',
+              'tkinter.commondialog', 'tkinter.constants', 'tkinter.messagebox', 'uuid', 'win32evtlogutil', 'wsgiref',
+              'wsgiref.handlers', 'wsgiref.headers', 'wsgiref.simple_server', 'wsgiref.util', 'xml.dom', 'xml.dom.NodeFilter', 'xml.dom.domreg',
+              'xml.dom.expatbuilder', 'xml.dom.minicompat', 'xml.dom.minidom', 'xml.dom.pulldom', 'xml.dom.xmlbuilder', 'xmlrpc.server'],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=['gluon'],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='web2py',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=False,
+          icon='extras/icons/web2py.icns')
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               name='web2py')
+app = BUNDLE(coll,
+             name='web2py.app',
+             icon='extras/icons/web2py.icns',
+             bundle_identifier=None,
+             info_plist={
+            'NSPrincipleClass': 'NSApplication',
+            'NSAppleScriptEnabled': False})
diff --git a/extras/build_web2py/web2py.win.spec b/extras/build_web2py/web2py.win.spec
new file mode 100644
index 00000000..6dce1ebf
--- /dev/null
+++ b/extras/build_web2py/web2py.win.spec
@@ -0,0 +1,44 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['web2py.py'],
+             pathex=['.'],
+             binaries=[],
+             datas=[],
+             hiddenimports=['site-packages', 'cgi', 'cgitb', 'code', 'concurrent', 'concurrent.futures', 
+              'concurrent.futures._base', 'concurrent.futures.process', 'concurrent.futures.thread', 'configparser', 'csv', 'ctypes.wintypes',
+              'email.mime', 'email.mime.base', 'email.mime.multipart', 'email.mime.nonmultipart', 'email.mime.text', 'html.parser', 'http.cookies',
+              'ipaddress', 'imp', 'json', 'json.decoder', 'json.encoder', 'json.scanner', 'logging.config', 'logging.handlers', 'profile', 'pstats',
+              'psycopg2', 'psycopg2._ipaddress', 'psycopg2._json', 'psycopg2._range', 'psycopg2.extensions', 'psycopg2.extras', 'psycopg2.sql',
+              'psycopg2.tz', 'pyodbc', 'python-ldap', 'rlcompleter', 'sched', 'site', 'smtplib', 'sqlite3', 'sqlite3.dbapi2', 'sqlite3.dump', 'timeit', 'tkinter',
+              'tkinter.commondialog', 'tkinter.constants', 'tkinter.messagebox', 'uuid', 'win32con', 'win32evtlogutil', 'winerror', 'wsgiref',
+              'wsgiref.handlers', 'wsgiref.headers', 'wsgiref.simple_server', 'wsgiref.util', 'xml.dom', 'xml.dom.NodeFilter', 'xml.dom.domreg',
+              'xml.dom.expatbuilder', 'xml.dom.minicompat', 'xml.dom.minidom', 'xml.dom.pulldom', 'xml.dom.xmlbuilder', 'xmlrpc.server'],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=['gluon'],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='web2py',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=True , icon='extras\\icons\\web2py.ico')
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               name='web2py')
diff --git a/extras/build_web2py/web2py.win_no_console.spec b/extras/build_web2py/web2py.win_no_console.spec
new file mode 100644
index 00000000..75932924
--- /dev/null
+++ b/extras/build_web2py/web2py.win_no_console.spec
@@ -0,0 +1,44 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['web2py.py'],
+             pathex=['.'],
+             binaries=[],
+             datas=[],
+             hiddenimports=['site-packages', 'cgi', 'cgitb', 'code', 'concurrent', 'concurrent.futures', 
+              'concurrent.futures._base', 'concurrent.futures.process', 'concurrent.futures.thread', 'configparser', 'csv', 'ctypes.wintypes',
+              'email.mime', 'email.mime.base', 'email.mime.multipart', 'email.mime.nonmultipart', 'email.mime.text', 'html.parser', 'http.cookies',
+              'ipaddress', 'imp', 'json', 'json.decoder', 'json.encoder', 'json.scanner', 'logging.config', 'logging.handlers', 'profile', 'pstats',
+              'psycopg2', 'psycopg2._ipaddress', 'psycopg2._json', 'psycopg2._range', 'psycopg2.extensions', 'psycopg2.extras', 'psycopg2.sql',
+              'psycopg2.tz', 'pyodbc', 'python-ldap', 'rlcompleter', 'sched', 'site', 'smtplib', 'sqlite3', 'sqlite3.dbapi2', 'sqlite3.dump', 'timeit', 'tkinter',
+              'tkinter.commondialog', 'tkinter.constants', 'tkinter.messagebox', 'uuid', 'win32con', 'win32evtlogutil', 'winerror', 'wsgiref',
+              'wsgiref.handlers', 'wsgiref.headers', 'wsgiref.simple_server', 'wsgiref.util', 'xml.dom', 'xml.dom.NodeFilter', 'xml.dom.domreg',
+              'xml.dom.expatbuilder', 'xml.dom.minicompat', 'xml.dom.minidom', 'xml.dom.pulldom', 'xml.dom.xmlbuilder', 'xmlrpc.server'],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=['gluon'],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='web2py_no_console',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=False , icon='extras\\icons\\web2py.ico')
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               name='web2py_no_console')

From 6f12be7e20b1801fb55eadd1f44b2c01f049eb0d Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Fri, 26 Apr 2019 22:36:02 -0700
Subject: [PATCH 046/111] even better option processing in widget.py, thanks
 Paolo

---
 gluon/packages/yatl |  2 +-
 gluon/widget.py     | 46 +++++++++++++++++++++++----------------------
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/gluon/packages/yatl b/gluon/packages/yatl
index 694f630f..8c6f1e1f 160000
--- a/gluon/packages/yatl
+++ b/gluon/packages/yatl
@@ -1 +1 @@
-Subproject commit 694f630f793945e70249ba54ff4341454200ff72
+Subproject commit 8c6f1e1f17f08d5638490fb3dfe736953f585f84
diff --git a/gluon/widget.py b/gluon/widget.py
index 15ee70d4..37d1e730 100644
--- a/gluon/widget.py
+++ b/gluon/widget.py
@@ -9,6 +9,8 @@ The widget is called from web2py
 ----------------------------------
 """
 
+from __future__ import print_function
+
 import sys
 from gluon._compat import thread, xrange, PY2
 import time
@@ -38,8 +40,6 @@ if sys.version_info < (2, 7) or (3, 0) < sys.version_info < (3, 5):
     sys.stderr.write("Warning: web2py requires at least Python 2.7/3.5"
         " but you are running %s\n" % python_version())
 
-logger = logging.getLogger("web2py")
-
 
 def run_system_tests(options):
     """
@@ -318,7 +318,7 @@ class web2pyDialog(object):
         apps = []
         available_apps = [
             arq for arq in os.listdir(applications_folder)
-            if os.path.exists(os.path.join(applications_folder, arq, 'models', 'scheduler.py'))
+            if os.path.isdir(os.path.join(applications_folder, arq))
         ]
         if start:
             # the widget takes care of starting the scheduler
@@ -350,7 +350,7 @@ class web2pyDialog(object):
             return
         code = "from gluon.globals import current;current._scheduler.loop()"
         print('starting scheduler from widget for "%s"...' % app)
-        args = (app, True, True, None, False, code)
+        args = (app, True, True, None, False, code, False, True)
         logging.getLogger().setLevel(self.options.debuglevel)
         p = Process(target=run, args=args)
         self.scheduler_processes[app] = p
@@ -494,7 +494,7 @@ class web2pyDialog(object):
                 profiler_dir=options.profiler_dir,
                 ssl_certificate=options.ssl_certificate,
                 ssl_private_key=options.ssl_private_key,
-                ssl_ca_certificate=options.ssl_ca_certificate,
+                ssl_ca_certificate=options.ca_cert,
                 min_threads=options.minthreads,
                 max_threads=options.maxthreads,
                 server_name=options.server_name,
@@ -627,7 +627,7 @@ web2py will attempt to run a GUI to ask for it when starting the web server
                       default=None,
                       metavar='FILE', help='server private key file')
 
-    parser.add_option('--ca-cert', dest='ssl_ca_certificate',
+    parser.add_option('--ca-cert', dest='ca_cert', # not needed
                       default=None,
                       metavar='FILE', help='CA certificate file')
 
@@ -814,7 +814,7 @@ web2py will attempt to run a GUI to ask for it when starting the web server
         'to be used with -S; NOTE: must be the last option because eat all ' \
         'remaining arguments')
 
-    parser.add_option('--no-banner', dest='nobanner',
+    parser.add_option('--no-banner', dest='no_banner', # not needed
                       default=False,
                       action='store_true',
                       help='do not print header banner')
@@ -930,7 +930,7 @@ def start_schedulers(options):
         if not app:
             return
         print('starting single-scheduler for "%s"...' % app)
-        run(app, True, True, None, False, code)
+        run(app, True, True, None, False, code, False, True)
         return
 
     # Work around OS X problem: http://bugs.python.org/issue9405
@@ -946,7 +946,7 @@ def start_schedulers(options):
         if not app:
             continue
         print('starting scheduler for "%s"...' % app)
-        args = (app, True, True, None, False, code)
+        args = (app, True, True, None, False, code, False, True)
         p = Process(target=run, args=args)
         processes.append(p)
         print("Currently running %s scheduler processes" % (len(processes)))
@@ -964,7 +964,7 @@ def start_schedulers(options):
             p.join()
 
 
-def start(cron=True):
+def start():
     """ Starts server and other services """
 
     # get command line arguments
@@ -989,6 +989,10 @@ def start(cron=True):
             print("gaehandler.py alreday exists in the web2py folder")
         return
 
+    logger = logging.getLogger("web2py")
+    logger.setLevel(options.debuglevel)
+
+    # on new installation build the scaffolding app
     create_welcome_w2p()
 
     if options.run_system_tests:
@@ -1015,9 +1019,7 @@ def start(cron=True):
                     l.removeHandler(h)
         # NOTE: stderr.write() is still working
 
-    logger.setLevel(options.debuglevel)
-
-    if not options.nobanner:
+    if not options.no_banner:
         # banner
         print(ProgramName)
         print(ProgramAuthor)
@@ -1062,7 +1064,7 @@ def start(cron=True):
             pass
         return
 
-    if cron and options.runcron:
+    if options.runcron:
         if options.softcron:
             print('Using softcron (but this is not very efficient)')
             global_settings.web2py_crontype = 'soft'
@@ -1117,12 +1119,6 @@ end tell
 
         sys.exit()
 
-    if options.password == '':
-        options.password = getpass.getpass('choose a password:')
-
-    if not options.password and not options.nobanner:
-        print('no password, disable admin interface')
-
     spt = None
 
     if options.scheduler and options.with_scheduler:
@@ -1132,6 +1128,12 @@ end tell
 
     # start server
 
+    if options.password == '':
+        options.password = getpass.getpass('choose a password:')
+
+    if not options.password and not options.no_banner:
+        print('no password, disable admin interface')
+
     # Use first interface IP and port if interfaces specified, since the
     # interfaces option overrides the IP (and related) options.
     if not options.interfaces:
@@ -1149,7 +1151,7 @@ end tell
 
     url = get_url(ip, proto=proto, port=port)
 
-    if not options.nobanner:
+    if not options.no_banner:
         message = '\nplease visit:\n\t%s\n'
         if sys.platform.startswith('win'):
             message += 'use "taskkill /f /pid %i" to shutdown the web2py server\n\n'
@@ -1186,7 +1188,7 @@ end tell
                              profiler_dir=options.profiler_dir,
                              ssl_certificate=options.ssl_certificate,
                              ssl_private_key=options.ssl_private_key,
-                             ssl_ca_certificate=options.ssl_ca_certificate,
+                             ssl_ca_certificate=options.ca_cert,
                              min_threads=options.minthreads,
                              max_threads=options.maxthreads,
                              server_name=options.server_name,

From 1c08c07a0f255e80d11c7eb6ec8ab47799f7531a Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Wed, 1 May 2019 21:07:52 -0700
Subject: [PATCH 047/111] new command line options

---
 CHANGELOG                                     |  21 +
 docker/alpine/web2py-rocket-ssl/Dockerfile    |   2 +-
 docker/alpine/web2py-rocket/Dockerfile        |   2 +-
 docker/centos/web2py-rocket/Dockerfile        |   2 +-
 docker/debian/web2py-rocket/Dockerfile        |   2 +-
 docker/fedora/web2py-rocket/Dockerfile        |   2 +-
 docker/opensuse/web2py-rocket/Dockerfile      |   2 +-
 docker/python/web2py-rocket-ssl/Dockerfile    |   2 +-
 docker/python/web2py-rocket/Dockerfile        |   2 +-
 .../stack/web2py-rocket-nginx/web2py-rocket   |   2 +-
 .../web2py-rocket-ssl                         |   2 +-
 .../web2py-rocket-ssl                         |   2 +-
 .../web2py-rocket-ssl                         |   2 +-
 .../web2py-rocket-ssl-nginx/web2py-rocket-ssl |   2 +-
 docker/ubuntu/web2py-rocket/Dockerfile        |   2 +-
 gluon/console.py                              | 712 ++++++++++++++++++
 gluon/fileutils.py                            |   3 +-
 gluon/globals.py                              |   9 +-
 gluon/main.py                                 |   4 +-
 gluon/newcron.py                              |   7 +-
 gluon/packages/yatl                           |   2 +-
 gluon/shell.py                                |  10 +-
 gluon/tests/test_scheduler.py                 |   3 +-
 gluon/widget.py                               | 417 ++--------
 scripts/web2py.fedora.sh                      |   2 +-
 25 files changed, 809 insertions(+), 409 deletions(-)
 create mode 100644 gluon/console.py

diff --git a/CHANGELOG b/CHANGELOG
index b5909b91..47b185a4 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,24 @@
+## 2.19.0
+- new command line options (Thanks Paolo Pastori)
+
+OLD NAME                   NEW NAME
+==================         ==================
+--debug                     --log_level
+--nogui                     --no_gui
+--ssl_private_key           --server_key
+--ssl_certificate           --server_cert
+--minthreads                --min_threads
+--maxthreads                --max_threads
+--profiler                  --profiler_dir
+--run-cron                  --with_cron
+--softcron                  --soft_cron
+--cron                      --cron_run
+--cronjob *                 --cron_job *
+--test                      --run_doctests
+                            --add_options
+                            --interface
+                            --crontab
+
 ## 2.18.1-2.18.5
 - pydal 19.04
 - made template its own module (Yet Another Template Language)
diff --git a/docker/alpine/web2py-rocket-ssl/Dockerfile b/docker/alpine/web2py-rocket-ssl/Dockerfile
index 05a8c47d..f0d34fd1 100755
--- a/docker/alpine/web2py-rocket-ssl/Dockerfile
+++ b/docker/alpine/web2py-rocket-ssl/Dockerfile
@@ -19,4 +19,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443
diff --git a/docker/alpine/web2py-rocket/Dockerfile b/docker/alpine/web2py-rocket/Dockerfile
index 404e0f45..dd08901a 100755
--- a/docker/alpine/web2py-rocket/Dockerfile
+++ b/docker/alpine/web2py-rocket/Dockerfile
@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/centos/web2py-rocket/Dockerfile b/docker/centos/web2py-rocket/Dockerfile
index 212f84bb..bd86aad4 100755
--- a/docker/centos/web2py-rocket/Dockerfile
+++ b/docker/centos/web2py-rocket/Dockerfile
@@ -25,4 +25,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/debian/web2py-rocket/Dockerfile b/docker/debian/web2py-rocket/Dockerfile
index 802491cc..43c3164d 100755
--- a/docker/debian/web2py-rocket/Dockerfile
+++ b/docker/debian/web2py-rocket/Dockerfile
@@ -25,4 +25,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/fedora/web2py-rocket/Dockerfile b/docker/fedora/web2py-rocket/Dockerfile
index 3fa20078..bdd1db99 100755
--- a/docker/fedora/web2py-rocket/Dockerfile
+++ b/docker/fedora/web2py-rocket/Dockerfile
@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/opensuse/web2py-rocket/Dockerfile b/docker/opensuse/web2py-rocket/Dockerfile
index a8fca1e2..24797491 100755
--- a/docker/opensuse/web2py-rocket/Dockerfile
+++ b/docker/opensuse/web2py-rocket/Dockerfile
@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/python/web2py-rocket-ssl/Dockerfile b/docker/python/web2py-rocket-ssl/Dockerfile
index 9308ca4b..8f238582 100755
--- a/docker/python/web2py-rocket-ssl/Dockerfile
+++ b/docker/python/web2py-rocket-ssl/Dockerfile
@@ -18,4 +18,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443
diff --git a/docker/python/web2py-rocket/Dockerfile b/docker/python/web2py-rocket/Dockerfile
index 28058453..94fdf95e 100755
--- a/docker/python/web2py-rocket/Dockerfile
+++ b/docker/python/web2py-rocket/Dockerfile
@@ -20,4 +20,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/stack/web2py-rocket-nginx/web2py-rocket b/docker/stack/web2py-rocket-nginx/web2py-rocket
index ed228fcf..5b64b076 100644
--- a/docker/stack/web2py-rocket-nginx/web2py-rocket
+++ b/docker/stack/web2py-rocket-nginx/web2py-rocket
@@ -22,4 +22,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl
index 5a7ffe54..1d6629b7 100644
--- a/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl
+++ b/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl
@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443
diff --git a/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl
index 6720178a..6f271006 100644
--- a/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl
+++ b/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl
@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443
diff --git a/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl
index 0b8c82f9..c6bb03fc 100644
--- a/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl
+++ b/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl
@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443
diff --git a/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl
index 6720178a..6f271006 100644
--- a/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl
+++ b/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl
@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443
diff --git a/docker/ubuntu/web2py-rocket/Dockerfile b/docker/ubuntu/web2py-rocket/Dockerfile
index ea48283d..ab96dca2 100755
--- a/docker/ubuntu/web2py-rocket/Dockerfile
+++ b/docker/ubuntu/web2py-rocket/Dockerfile
@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000
diff --git a/gluon/console.py b/gluon/console.py
new file mode 100644
index 00000000..5a17c52f
--- /dev/null
+++ b/gluon/console.py
@@ -0,0 +1,712 @@
+# -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
+"""
+| This file is part of the web2py Web Framework
+| Copyrighted by Massimo Di Pierro 
+| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
+
+Command line interface
+----------------------
+
+The processing of all command line arguments is done using
+the argparse library in the console function.
+
+The basic principle is to process and check for all options
+in a single place, this place is the parse_args function.
+Notice that when I say all options I mean really all,
+options sourced from a configuration file are included.
+
+A brief summary of options style follows,
+for the benefit of code maintainers/developers:
+
+- use the underscore to split words in long names (as in
+  '--run_system_tests')
+- remember to allow the '-' too as word separator (e.g.
+  '--run-system-tests') but do not use this form on help
+- prefer short names on help messages, instead use
+  all options names in warning/error messages (e.g.
+  '-R/--run requires -S/--shell')
+"""
+
+from __future__ import print_function
+
+__author__ = 'Paolo Pastori'
+
+import os.path
+import argparse
+import logging
+import socket
+import sys
+import re
+import ast
+from collections import OrderedDict
+import copy
+
+from gluon._compat import PY2
+from gluon.shell import die
+from gluon.utils import is_valid_ip_address
+from gluon.settings import global_settings
+
+
+def warn(msg):
+    print("%s: warning: %s" % (sys.argv[0], msg), file=sys.stderr)
+
+def is_appdir(applications_parent, app):
+    return os.path.isdir(os.path.join(applications_parent, 'applications', app))
+
+
+def console(version):
+    """
+    Load command line options.
+    Trivial -h/--help and --version options are also processed.
+
+    Returns a namespace object (in the sense of argparse)
+    with all options loaded.
+    """
+
+    # replacement hints for deprecated options
+    deprecated_opts = {
+        '--debug': '--log_level',
+        '--nogui': '--no_gui',
+        '--ssl_private_key': '--server_key',
+        '--ssl_certificate': '--server_cert',
+        '--interfaces': None, # dest is 'interfaces', hint is '--interface'
+        '-n': '--min_threads', '--numthreads': '--min_threads',
+        '--minthreads': '--min_threads',
+        '--maxthreads': '--max_threads',
+        '-z': None, '--shutdown_timeout': None,
+        '--profiler': '--profiler_dir',
+        '--run-cron': '--with_cron',
+        '--softcron': '--soft_cron',
+        '--cron': '--cron_run',
+        '--test': '--run_doctests'
+    }
+
+    class HelpFormatter2(argparse.HelpFormatter):
+        """Hides the options listed in _hidden_options in usage help."""
+
+        # NOTE: preferred style for long options name is to use '_'
+        #       between words (as in 'no_gui'), also accept the '-' in
+        #       most of the options but do not show both versions on help
+        _omitted_opts = ('--add-options', '--errors-to-console',
+            '--no-banner', '--log-level', '--no-gui', '--import-models',
+            '--server-name', '--server-key', '--server-cert', '--ca-cert',
+            '--pid-filename', '--log-filename', '--min-threads',
+            '--max-threads', '--request-queue-size', '--socket-timeout',
+            '--profiler-dir', '--with-scheduler', '--with-cron',
+            '--soft-cron', '--cron-run',
+            '--run-doctests', '--run-system-tests', '--with-coverage')
+
+        _hidden_options = _omitted_opts + tuple(deprecated_opts.keys())
+
+        def _format_action_invocation(self, action):
+            if not action.option_strings:
+                return super(HelpFormatter2, self)._format_action_invocation(action)
+            parts = []
+            if action.nargs == 0:
+                parts.extend(filter(lambda o : o not in self._hidden_options,
+                                    action.option_strings))
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    if option_string in self._hidden_options:
+                        continue
+                    parts.append('%s %s' % (option_string, args_string))
+            return ', '.join(parts)
+
+    class ExtendAction(argparse._AppendAction):
+        """Action to accumulate values in a flat list."""
+
+        def __call__(self, parser, namespace, values, option_string=None):
+            if isinstance(values, list):
+                # must copy to avoid altering the option default value
+                items = argparse._ensure_value(namespace, self.dest, [])[:]
+                # for options that allows multiple args (i.e. those declared
+                # with add_argument(..., nargs='+', ...)) the values are
+                # always placed into a list
+                while len(values) == 1 and isinstance(values[0], list):
+                    values = values[0]
+                items.extend(values)
+                setattr(namespace, self.dest, items)
+            else:
+                super(ExtendAction, self).__call__(parser, namespace, values, option_string)
+
+    parser = argparse.ArgumentParser(
+        usage='python %(prog)s [options]',
+        description='web2py Web Framework startup script.',
+        epilog='''NOTE: unless a password is specified (-a 'passwd')
+web2py will attempt to run a GUI to ask for it when starting the web server
+(if not disabled with --no_gui).''',
+        formatter_class=HelpFormatter2,
+        add_help=False) # do not add -h/--help option
+
+    # global options
+    g = parser.add_argument_group('global options')
+    g.add_argument('-h', '--help', action='help',
+                   help='show this help message and exit')
+    g.add_argument('--version', action='version',
+                   version=version,
+                   help="show program's version and exit")
+    folder = os.getcwd()
+    g.add_argument('-f', '--folder',
+                   default=folder, metavar='WEB2PY_DIR',
+                   help='web2py installation directory (%(default)s)')
+    def existing_file(v):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if not os.path.exists(v):
+            raise argparse.ArgumentTypeError("file %r not found" % v)
+        return v
+    g.add_argument('-L', '--config',
+                   type=existing_file,
+                   metavar='PYTHON_FILE',
+                   help='read all options from PYTHON_FILE')
+    g.add_argument('--add_options', '--add-options',
+                   default=False,
+                   action='store_true', help=
+        'add options to existing ones, useful with -L only')
+    g.add_argument('-a', '--password',
+                   default='', help=
+        'password to be used for administration (use "" '
+        'to reuse the last password), when no password is available '
+        'the administrative web interface will be disabled')
+    g.add_argument('-e', '--errors_to_console', '--errors-to-console',
+                   default=False,
+                   action='store_true',
+                   help='log application errors to console')
+    g.add_argument('--no_banner', '--no-banner',
+                   default=False,
+                   action='store_true',
+                   help='do not print header banner')
+    g.add_argument('-Q', '--quiet',
+                   default=False,
+                   action='store_true',
+                   help='disable all output')
+    integer_log_level = []
+    def log_level(v):
+        # try to convert a lgging level name to its numeric value,
+        # could use logging.getLevelName but not with
+        # 3.4 <= Python < 3.4.2, see
+        # https://docs.python.org/3/library/logging.html#logging.getLevelName)
+        try:
+            name2level = logging._levelNames
+        except AttributeError:
+            # logging._levelNames has gone with Python 3.4, see
+            # https://github.com/python/cpython/commit/3b84eae03ebd8122fdbdced3d85999dd9aedfc7e
+            name2level = logging._nameToLevel
+        try:
+            return name2level[v.upper()]
+        except KeyError:
+            pass
+        try:
+            ill = int(v)
+            # value deprecated: integer in range(101)
+            if 0 <= ill <= 100:
+                integer_log_level.append(ill)
+                return ill
+        except ValueError:
+            pass
+        raise argparse.ArgumentTypeError("bad level %r" % v)
+    g.add_argument('-D', '--log_level', '--log-level',
+                   '--debug', # deprecated
+                   default='WARNING',
+                   type=log_level,
+                   metavar='LOG_LEVEL', help=
+        'set log level, allowed values are: NOTSET, DEBUG, INFO, WARN, '
+        'WARNING, ERROR, and CRITICAL, also lowercase (default is '
+        '%(default)s)')
+
+    # GUI options
+    g = parser.add_argument_group('GUI options')
+    g.add_argument('--no_gui', '--no-gui',
+                   '--nogui', # deprecated
+                   default=False,
+                   action='store_true',
+                   help='do not run GUI')
+    g.add_argument('-t', '--taskbar',
+                   default=False,
+                   action='store_true',
+                   help='run in taskbar (system tray)')
+
+    # console options
+    g = parser.add_argument_group('console options')
+    g.add_argument('-S', '--shell',
+                   metavar='APP_ENV', help=
+        'run web2py in Python interactive shell or IPython (if installed) '
+        'with specified application environment (if application does not '
+        'exist it will be created). APP_ENV like a/c/f?x=y (c, f and vars '
+        'optional), if APP_ENV include the action f then after the '
+        'action execution the interpreter is exited')
+    g.add_argument('-B', '--bpython',
+                   default=False,
+                   action='store_true', help=
+        'use bpython (if installed) when running in interactive shell, '
+        'see -S above')
+    g.add_argument('-P', '--plain',
+                   default=False,
+                   action='store_true', help=
+        'use plain Python shell when running in interactive shell, '
+        'see -S above')
+    g.add_argument('-M', '--import_models', '--import-models',
+                   default=False,
+                   action='store_true', help=
+        'auto import model files when running in interactive shell '
+        '(default is %(default)s), see -S above. NOTE: when the APP_ENV '
+        'argument of -S include a controller c automatic import of '
+        'models is always enabled')
+    g.add_argument('-R', '--run',
+                   type=existing_file,
+                   metavar='PYTHON_FILE', help=
+        'run PYTHON_FILE in web2py environment; require -S')
+    g.add_argument('-A', '--args',
+                   default=[],
+                   nargs=argparse.REMAINDER, help=
+        'use this to pass arguments to the PYTHON_FILE above; require '
+        '-R. NOTE: must be the last option because eat all remaining '
+        'arguments')
+
+    # web server options
+    g = parser.add_argument_group('web server options')
+    g.add_argument('-s', '--server_name', '--server-name',
+                   default=socket.gethostname(),
+                   help='web server name (%(default)s)')
+    def ip_addr(v):
+        if not is_valid_ip_address(v):
+            raise argparse.ArgumentTypeError("bad IP address %s" % v)
+        return v
+    g.add_argument('-i', '--ip',
+                   default='127.0.0.1',
+                   type=ip_addr, metavar='IP_ADDR', help=
+        'IP address of the server (%(default)s), accept either IPv4 or '
+        'IPv6 (e.g. ::1) addresses. NOTE: this option is ignored if '
+        '--interface is specified')
+    def not_negative_int(v, err_label='value'):
+        try:
+            iv = int(v)
+            if iv < 0: raise ValueError()
+            return iv
+        except ValueError:
+            pass
+        raise argparse.ArgumentTypeError("bad %s %s" % (err_label, v))
+    def port(v):
+        return not_negative_int(v, err_label='port')
+    g.add_argument('-p', '--port',
+                   default=8000,
+                   type=port, metavar='NUM', help=
+        'port of server (%(default)d). '
+        'NOTE: this option is ignored if --interface is specified')
+    g.add_argument('-k', '--server_key', '--server-key',
+                   '--ssl_private_key', # deprecated
+                   type=existing_file,
+                   metavar='FILE', help='server private key')
+    g.add_argument('-c', '--server_cert', '--server-cert',
+                   '--ssl_certificate', # deprecated
+                   type=existing_file,
+                   metavar='FILE', help='server certificate')
+    g.add_argument('--ca_cert', '--ca-cert',
+                   type=existing_file,
+                   metavar='FILE', help='CA certificate')
+    def iface(v, sep=','):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if sep == ':':
+            # deprecated --interfaces ip:port:key:cert:ca_cert
+            # IPv6 addresses in square brackets
+            if v.startswith('['):
+                # IPv6
+                ip, v_remainder = v.split(']', 1)
+                ip = ip[1:]
+                ifp = v_remainder[1:].split(':')
+                ifp.insert(0, ip)
+            else:
+                # IPv4
+                ifp = v.split(':')
+        else:
+            # --interface
+            ifp = v.split(sep, 5)
+        if not len(ifp) in (2, 4, 5):
+            raise argparse.ArgumentTypeError("bad interface %r" % v)
+        try:
+            ip_addr(ifp[0])
+            ifp[1] = port(ifp[1])
+            for fv in ifp[2:]:
+                existing_file(fv)
+        except argparse.ArgumentTypeError as ex:
+            raise argparse.ArgumentTypeError("bad interface %r (%s)" % (v, ex))
+        return tuple(ifp)
+    g.add_argument('--interface', dest='interfaces',
+                   default=[], action=ExtendAction,
+                   type=iface, nargs='+',
+                   metavar='IF_INFO', help=
+        'listen on specified interface, IF_INFO = '
+        'IP_ADDR,PORT[,KEY_FILE,CERT_FILE[,CA_CERT_FILE]].'
+        ' NOTE: this option can be used multiple times to provide additional '
+        'interfaces to choose from but you can choose which one to listen to '
+        'only using the GUI otherwise the first interface specified is used')
+    def ifaces(v):
+        # deprecated --interfaces 'if1;if2;...'
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        return [iface(i, ':') for i in v.split(';')]
+    g.add_argument('--interfaces', # deprecated
+                   default=argparse.SUPPRESS, # do not set if absent
+                   action=ExtendAction,
+                   type=ifaces,
+                   help=argparse.SUPPRESS) # do not show on help
+    g.add_argument('-d', '--pid_filename', '--pid-filename',
+                   default='httpserver.pid',
+                   metavar='FILE', help='server pid file (%(default)s)')
+    g.add_argument('-l', '--log_filename', '--log-filename',
+                   default='httpserver.log',
+                   metavar='FILE', help='server log file (%(default)s)')
+    g.add_argument('--min_threads', '--min-threads',
+                   '--minthreads', '-n', '--numthreads', # deprecated
+                   type=not_negative_int, metavar='NUM',
+                   help='minimum number of server threads')
+    g.add_argument('--max_threads', '--max-threads',
+                   '--maxthreads', # deprecated
+                   type=not_negative_int, metavar='NUM',
+                   help='maximum number of server threads')
+    g.add_argument('-q', '--request_queue_size', '--request-queue-size',
+                   default=5,
+                   type=not_negative_int, metavar='NUM', help=
+        'max number of queued requests when server busy (%(default)d)')
+    g.add_argument('-o', '--timeout',
+                   default=10,
+                   type=not_negative_int, metavar='SECONDS',
+                   help='timeout for individual request (%(default)d seconds)')
+    g.add_argument('--socket_timeout', '--socket-timeout',
+                   default=5,
+                   type=not_negative_int, metavar='SECONDS',
+                   help='timeout for socket (%(default)d seconds)')
+    g.add_argument('-z', '--shutdown_timeout', # deprecated
+                   type=not_negative_int,
+                   help=argparse.SUPPRESS) # do not show on help
+    g.add_argument('-F', '--profiler_dir', '--profiler-dir',
+                   '--profiler', # deprecated
+                   help='profiler directory')
+
+    # scheduler options
+    g = parser.add_argument_group('scheduler options')
+    g.add_argument('-X', '--with_scheduler', '--with-scheduler',
+                   default=False,
+                   action='store_true', help=
+        'run schedulers alongside web server; require --K')
+    def is_app(app):
+        return is_appdir(folder, app)
+    def scheduler(v):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if ',' in v:
+            # legacy "app1,..."
+            vl = [n.strip() for n in v.split(',')]
+            return [scheduler(iv) for iv in vl]
+        vp = [n.strip() for n in v.split(':')]
+        app = vp[0]
+        if not app:
+            raise argparse.ArgumentTypeError('empty application')
+        if not is_app(app):
+            warn("argument -K/--scheduler: bad application %r, skipped" % app)
+            return None
+        return ':'.join(filter(None, vp))
+    g.add_argument('-K', '--scheduler', dest='schedulers',
+                   default=[], action=ExtendAction,
+                   type=scheduler, nargs='+',
+                   metavar='APP_INFO', help=
+        'run scheduler for the specified application(s), APP_INFO = '
+        'APP_NAME[:GROUPS], that is an optional list of groups can follow '
+        'the application name (e.g. app:group1:group2); require a scheduler '
+        "to be defined in the application's models. NOTE: this option can "
+        'be used multiple times to add schedulers')
+
+    # cron options
+    g = parser.add_argument_group('cron options')
+    g.add_argument('-Y', '--with_cron', '--with-cron',
+                   '--run-cron', # deprecated
+                   default=False,
+                   action='store_true', help=
+        'run cron service alongside web server')
+    def crontab(v):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if not is_app(v):
+            warn("argument --crontab: bad application %r, skipped" % v)
+            return None
+        return v
+    g.add_argument('--crontab', dest='crontabs',
+                   default=[], action=ExtendAction,
+                   type=crontab, nargs='+',
+                   metavar='APP_NAME', help=
+        'tell cron to read the crontab for the specified application(s) '
+        'only, the default behaviour is to read the crontab for all of the '
+        'installed applications. NOTE: this option can be used multiple '
+        'times to build the list of crontabs to be processed by cron')
+    g.add_argument('--soft_cron', '--soft-cron',
+                   '--softcron', # deprecated
+                   default=False,
+                   action='store_true', help=
+        'use cron software emulation instead of separate cron process; '
+        'require -Y. NOTE: use of cron software emulation is strongly '
+        'discouraged')
+    g.add_argument('-C', '--cron_run', '--cron-run',
+                   '--cron', # deprecated
+                   default=False,
+                   action='store_true', help=
+        'trigger a cron run and exit; usually used when invoked '
+        'from a system (external) crontab')
+    g.add_argument('--cron_job', # NOTE: this is intended for internal use only
+                   default=False,
+                   action='store_true',
+                   help=argparse.SUPPRESS) # do not show on help
+
+    # test options
+    g = parser.add_argument_group('test options')
+    g.add_argument('-v', '--verbose',
+                   default=False,
+                   action='store_true', help='increase verbosity')
+    g.add_argument('-T', '--run_doctests', '--run-doctests',
+                   '--test', # deprecated
+                   metavar='APP_ENV', help=
+        'run doctests in application environment. APP_ENV like a/c/f (c, f '
+        'optional)')
+    g.add_argument('--run_system_tests', '--run-system-tests',
+                   default=False,
+                   action='store_true', help='run web2py test suite')
+    g.add_argument('--with_coverage', '--with-coverage',
+                   default=False,
+                   action='store_true', help=
+        'collect coverage data when used with --run_system_tests; '
+        'require Python 2.7+ and the coverage module installed')
+
+    # other options
+    g = parser.add_argument_group('other options')
+    g.add_argument('-G', '--GAE', dest='gae',
+                   metavar='APP_NAME', help=
+        'will create app.yaml and gaehandler.py and exit')
+
+    options = parse_args(parser, sys.argv[1:],
+                         deprecated_opts, integer_log_level)
+
+    # make a copy of all options for global_settings
+    copy_options = copy.deepcopy(options)
+    copy_options.password = '******'
+    global_settings.cmd_options = copy_options
+
+    return options
+
+
+REGEX_PEP263 = r'^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)'
+
+def get_pep263_encoding(source):
+    """
+    Read python source file encoding, according to PEP 263, see
+    https://www.python.org/dev/peps/pep-0263/
+    """
+    with open(source, 'r') as sf:
+        l12 = (sf.readline(), sf.readline())
+    m12 = re.match(REGEX_PEP263, l12[0]) or re.match(REGEX_PEP263, l12[1])
+    return m12 and m12.group(1)
+
+
+IGNORE = lambda: None
+
+def load_config(config_file, opt_map):
+    """
+    Load options from config file (a Python script).
+
+    config_file(str): file name
+    opt_map(dict): mapping fom option name (key) to callable (val),
+        used to post-process parsed value for the option
+
+    Notice that the configuring Python script is never executed/imported,
+    instead the ast library is used to evaluate each option assignment,
+    provided that it is writen on a single line.
+
+    Returns an OrderedDict with sourced options.
+    """
+    REGEX_ASSIGN_EXP = re.compile(r'\s*=\s*(.+)')
+    map_items = opt_map.items()
+    # preserve the order of loaded options even if this is not needed
+    pl = OrderedDict()
+    config_encoding = get_pep263_encoding(config_file)
+    # NOTE: assume 'ascii' encoding when not explicitly stated (Python 2),
+    #       this is not correct for Python 3 where the default is 'utf-8'
+    open_kwargs = dict() if PY2 else dict(encoding=config_encoding or 'ascii')
+    with open(config_file, 'r', **open_kwargs) as cfil:
+        for linenum, clin in enumerate(cfil, start=1):
+            if PY2 and config_encoding:
+                clin = unicode(clin, config_encoding)
+            clin = clin.strip()
+            for opt, mapr in map_items:
+                if clin.startswith(opt):
+                    m = REGEX_ASSIGN_EXP.match(clin[len(opt):])
+                    if m is None: continue
+                    try:
+                        val = opt_map[opt](ast.literal_eval(m.group(1)))
+                    except:
+                        die("cannot parse config file %r at line %d" % (config_file, linenum))
+                    if val is not IGNORE:
+                        pl[opt] = val
+    return pl
+
+
+def parse_args(parser, cli_args, deprecated_opts, integer_log_level,
+               namespace=None):
+
+    #print('PARSING ARGS:', cli_args)
+    del integer_log_level[:]
+    options = parser.parse_args(cli_args, namespace)
+    #print('PARSED OPTIONS:', options)
+
+    # warn for deprecated options
+    deprecated_args = [a for a in cli_args if a in deprecated_opts]
+    for da in deprecated_args:
+        # verify if it was a real option by looking into
+        # parsed values for the actual destination
+        hint = deprecated_opts[da]
+        dest = (hint or da).lstrip('-')
+        default = parser.get_default(dest)
+        if da == '--interfaces':
+            hint = '--interface'
+        if getattr(options, dest) is not default:
+            # the option has been specified
+            msg = "%s is deprecated" % da
+            if hint:
+                msg += ", use %s instead" % hint
+            warn(msg)
+    # warn for deprecated values
+    if integer_log_level and '--debug' not in deprecated_args:
+        warn('integer argument for -D/--log_level is deprecated, '
+             'use label instead')
+    # fix schedulers and die if all were skipped
+    if None in options.schedulers:
+        options.schedulers = [i for i in options.schedulers if i is not None]
+        if not options.schedulers:
+            die('no scheduler left')
+    # fix crontabs and die if all were skipped
+    if None in options.crontabs:
+        options.crontabs = [i for i in options.crontabs if i is not None]
+        if not options.crontabs:
+            die('no crontab left')
+    # taskbar
+    if options.taskbar and os.name != 'nt':
+        warn('--taskbar not supported on this platform, skipped')
+        options.taskbar = False
+    # options consistency checkings
+    if options.run and not options.shell:
+        die('-R/--run requires -S/--shell', exit_status=2)
+    if options.args and not options.run:
+        die('-A/--args requires -R/--run', exit_status=2)
+    if options.with_scheduler and not options.schedulers:
+        die('-X/--with_scheduler requires -K/--scheduler', exit_status=2)
+    if options.soft_cron and not options.with_cron:
+        die('--soft_cron requires -Y/--with_cron', exit_status=2)
+    if options.shell:
+        for o, os in dict(with_scheduler='-X/--with_scheduler',
+                          schedulers='-K/--scheduler',
+                          with_cron='-Y/--with_cron',
+                          cron_run='-C/--cron_run',
+                          run_doctests='-T/--run_doctests',
+                          run_system_tests='--run_system_tests').items():
+            if getattr(options, o):
+                die("-S/--shell and %s are conflicting options" % os,
+                    exit_status=2)
+    if options.bpython and options.plain:
+        die('-B/--bpython and -P/--plain are conflicting options',
+            exit_status=2)
+    if options.cron_run:
+        for o, os in dict(with_cron='-Y/--with_cron',
+                          run_doctests='-T/--run_doctests',
+                          run_system_tests='--run_system_tests').items():
+            if getattr(options, o):
+                die("-C/--cron_run and %s are conflicting options" % os,
+                    exit_status=2)
+    if options.run_doctests and options.run_system_tests:
+        die('-T/--run_doctests and --run_system_tests are conflicting options',
+            exit_status=2)
+
+    if options.config:
+        # load options from file,
+        # all options sourced from file that evaluates to False
+        # are skipped, the special IGNORE value is used for this
+        store_true = lambda v: True if v else IGNORE
+        str_or_default = lambda v : str(v) if v else IGNORE
+        list_or_default = lambda v : (
+            [str(i) for i in v] if isinstance(v, list) else [str(v)]) if v \
+            else IGNORE
+        # NOTE: 'help', 'version', 'folder', 'cron_job' and 'GAE' are not
+        #       sourced from file, the same applies to deprecated options
+        opt_map = {
+            # global options
+            'config': str_or_default,
+            'add_options': store_true,
+            'password': str_or_default,
+            'errors_to_console': store_true,
+            'no_banner': store_true,
+            'quiet': store_true,
+            'log_level': str_or_default,
+            # GUI options
+            'no_gui': store_true,
+            'taskbar': store_true,
+            # console options
+            'shell': str_or_default,
+            'bpython': store_true,
+            'plain': store_true,
+            'import_models': store_true,
+            'run': str_or_default,
+            'args': list_or_default,
+            # web server options
+            'server_name': str_or_default,
+            'ip': str_or_default,
+            'port': str_or_default,
+            'server_key': str_or_default,
+            'server_cert': str_or_default,
+            'ca_cert': str_or_default,
+            'interface': list_or_default,
+            'pid_filename': str_or_default,
+            'log_filename': str_or_default,
+            'min_threads': str_or_default,
+            'max_threads': str_or_default,
+            'request_queue_size': str_or_default,
+            'timeout': str_or_default,
+            'socket_timeout': str_or_default,
+            'profiler_dir': str_or_default,
+            # scheduler options
+            'with_scheduler': store_true,
+            'scheduler': list_or_default,
+            # cron options
+            'with_cron': store_true,
+            'crontab': list_or_default,
+            'soft_cron': store_true,
+            'cron_run': store_true,
+            # test options
+            'verbose': store_true,
+            'run_doctests': str_or_default,
+            'run_system_tests': store_true,
+            'with_coverage': store_true,
+        }
+        od = load_config(options.config, opt_map)
+        #print("LOADED FROM %s:" % options.config, od)
+        # convert loaded options dict as retuned by load_config
+        # into a list of arguments for further parsing by parse_args
+        file_args = []; args_args = [] # '--args' must be the last
+        for key, val in od.items():
+            if key != 'args':
+                file_args.append('--' + key)
+                if isinstance(val, list): file_args.extend(val)
+                elif not isinstance(val, bool): file_args.append(val)
+            else:
+                args_args = ['--args'] + val
+        file_args += args_args
+
+        if options.add_options:
+            # add options to existing ones,
+            # must clear config to avoid infinite recursion
+            options.config = options.add_options = None
+            return parse_args(parser, file_args,
+                deprecated_opts, integer_log_level, options)
+        return parse_args(parser, file_args,
+            deprecated_opts, integer_log_level)
+
+    return options
diff --git a/gluon/fileutils.py b/gluon/fileutils.py
index 9424d32e..2d02bcff 100644
--- a/gluon/fileutils.py
+++ b/gluon/fileutils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -374,7 +373,7 @@ def get_session(request, other_application='admin'):
         if not os.path.exists(session_filename):
             session_filename = generate(session_filename)
         osession = storage.load_storage(session_filename)
-    except:
+    except Exception:
         osession = storage.Storage()
     return osession
 
diff --git a/gluon/globals.py b/gluon/globals.py
index fc14ceff..9dcefe61 100644
--- a/gluon/globals.py
+++ b/gluon/globals.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -355,11 +354,9 @@ class Request(Storage):
         """
         cmd_opts = global_settings.cmd_options
         # checking if this is called within the scheduler or within the shell
-        # in addition to checking if it's a cronjob
-        # FIXME: cmd_opts.scheduler does not imply that
-        #        we are running in the scheduler
-        if (self.is_https or cmd_opts and (
-                cmd_opts.shell or cmd_opts.scheduler or cmd_opts.cronjob)):
+        # in addition to checking if it's a cron job
+        if (self.is_https or self.is_scheduler or cmd_opts and (
+                cmd_opts.shell or cmd_opts.cron_job)):
             current.session.secure()
         else:
             current.session.forget()
diff --git a/gluon/main.py b/gluon/main.py
index 68e4307d..c94825dc 100644
--- a/gluon/main.py
+++ b/gluon/main.py
@@ -1,4 +1,3 @@
-#!/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -557,7 +556,8 @@ def wsgibase(environ, responder):
         return wsgibase(new_environ, responder)
     if global_settings.web2py_crontype == 'soft':
         cmd_opts = global_settings.cmd_options
-        newcron.softcron(global_settings.applications_parent).start()
+        newcron.softcron(global_settings.applications_parent,
+                         apps=cmd_opts and cmd_opts.crontabs).start()
     return http_response.to(responder, env=env)
 
 
diff --git a/gluon/newcron.py b/gluon/newcron.py
index 5eba24e0..b94f48fc 100644
--- a/gluon/newcron.py
+++ b/gluon/newcron.py
@@ -287,7 +287,7 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
               ('dom', now_s.tm_mday),
               ('dow', (now_s.tm_wday + 1) % 7))
 
-    if apps is None:
+    if not apps:
         apps = [x for x in os.listdir(apppath)
                 if os.path.isdir(os.path.join(apppath, x))]
 
@@ -303,10 +303,7 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
         base_commands.append(w2p_path)
     if applications_parent != global_settings.gluon_parent:
         base_commands.extend(('-f', applications_parent))
-    base_commands.extend(('--cronjob', '--no-banner', '--nogui', '--plain',
-                          # FIXME: this should not be needed since we are
-                          #        not launching the web server
-                          '-a', '""'))
+    base_commands.extend(('--cron_job', '--no_banner', '--no_gui', '--plain'))
 
     for app in apps:
         if _cron_stopping:
diff --git a/gluon/packages/yatl b/gluon/packages/yatl
index 8c6f1e1f..3fb9abba 160000
--- a/gluon/packages/yatl
+++ b/gluon/packages/yatl
@@ -1 +1 @@
-Subproject commit 8c6f1e1f17f08d5638490fb3dfe736953f585f84
+Subproject commit 3fb9abbac896a8c00ce102b3d0e0503638bb7fd9
diff --git a/gluon/shell.py b/gluon/shell.py
index 89be48dc..889035ff 100644
--- a/gluon/shell.py
+++ b/gluon/shell.py
@@ -146,7 +146,7 @@ def env(
         request.is_shell = cmd_opts.shell is not None
     else:
         ip = '127.0.0.1'; port = 8000
-        # FIXME: what about request.is_shell ?
+        request.is_shell = False
     request.is_scheduler = False
     request.env.http_host = '%s:%s' % (ip, port)
     request.env.remote_addr = '127.0.0.1'
@@ -213,7 +213,7 @@ def run(
     startfile=None,
     bpython=False,
     python_code=None,
-    cronjob=False,
+    cron_job=False,
     scheduler_job=False):
     """
     Start interactive shell or run Python script (startfile) in web2py
@@ -233,7 +233,7 @@ def run(
     adir = os.path.join('applications', a)
 
     if not os.path.exists(adir):
-        if not cronjob and not scheduler_job and \
+        if not cron_job and not scheduler_job and \
             sys.stdin and not sys.stdin.name == '/dev/null':
             confirm = raw_input(
                 'application %s does not exist, create (y/n)?' % a)
@@ -259,7 +259,7 @@ def run(
         pyfile = os.path.join('applications', a, 'controllers', c + '.py')
         pycfile = os.path.join('applications', a, 'compiled',
                                  "controllers_%s_%s.pyc" % (c, f))
-        if ((cronjob and os.path.isfile(pycfile))
+        if ((cron_job and os.path.isfile(pycfile))
             or not os.path.isfile(pyfile)):
             exec(read_pyc(pycfile), _env)
         elif os.path.isfile(pyfile):
@@ -380,7 +380,7 @@ def test(testpath, import_models=True, verbose=False):
 
     import doctest
     if os.path.isfile(testpath):
-        mo = re.match(r'(|.*/)applications/(?P[^/]+)', testpath)
+        mo = re.search('/?applications/(?P[^/]+)', testpath)
         if not mo:
             die('test file is not in application directory: %s'
                 % testpath)
diff --git a/gluon/tests/test_scheduler.py b/gluon/tests/test_scheduler.py
index 06fc50fb..aecbcb5a 100644
--- a/gluon/tests/test_scheduler.py
+++ b/gluon/tests/test_scheduler.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -636,7 +635,7 @@ def termination():
 
     def exec_sched(self):
         import subprocess
-        call_args = [sys.executable, 'web2py.py', '--no-banner', '-D', '20','-K', 'welcome']
+        call_args = [sys.executable, 'web2py.py', '--no_banner', '-D', 'INFO','-K', 'welcome']
         ret = subprocess.call(call_args, env=dict(os.environ))
         return ret
 
diff --git a/gluon/widget.py b/gluon/widget.py
index 37d1e730..5b6d909a 100644
--- a/gluon/widget.py
+++ b/gluon/widget.py
@@ -5,8 +5,8 @@
 | Copyrighted by Massimo Di Pierro 
 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
 
-The widget is called from web2py
-----------------------------------
+GUI widget and services start function
+--------------------------------------
 """
 
 from __future__ import print_function
@@ -16,15 +16,15 @@ from gluon._compat import thread, xrange, PY2
 import time
 import threading
 import os
-import copy
 import socket
 import signal
 import math
 import logging
 import getpass
-from gluon import main, newcron
 
-from gluon.fileutils import read_file, write_file, create_welcome_w2p
+from gluon import main, newcron
+from gluon.fileutils import read_file, create_welcome_w2p
+from gluon.console import console
 from gluon.settings import global_settings
 from gluon.shell import die, run, test
 from gluon.utils import is_valid_ip_address, is_loopback_ip_address, getipaddrinfo
@@ -314,22 +314,20 @@ class web2pyDialog(object):
             self.tb = None
 
     def update_schedulers(self, start=False):
-        applications_folder = os.path.join(self.options.folder, 'applications')
-        apps = []
-        available_apps = [
-            arq for arq in os.listdir(applications_folder)
-            if os.path.isdir(os.path.join(applications_folder, arq))
-        ]
-        if start:
-            # the widget takes care of starting the scheduler
-            if self.options.scheduler and self.options.with_scheduler:
-                apps = [app for app
-                        in map(lambda ag : ag.split(':', 1)[0].strip(), self.options.scheduler.split(','))
-                        if app in available_apps]
+        if start and self.options.with_scheduler and self.options.with_schedulers:
+            # the widget takes care of starting the schedulers
+            apps = [ag.split(':', 1)[0] for ag in self.options.with_schedulers]
+        else:
+            apps = []
         for app in apps:
             self.try_start_scheduler(app)
 
         # reset the menu
+        applications_folder = os.path.join(self.options.folder, 'applications')
+        available_apps = [
+            arq for arq in os.listdir(applications_folder)
+            if os.path.isdir(os.path.join(applications_folder, arq))
+        ]
         self.schedmenu.delete(0, len(available_apps))
 
         for arq in available_apps:
@@ -351,7 +349,7 @@ class web2pyDialog(object):
         code = "from gluon.globals import current;current._scheduler.loop()"
         print('starting scheduler from widget for "%s"...' % app)
         args = (app, True, True, None, False, code, False, True)
-        logging.getLogger().setLevel(self.options.debuglevel)
+        logging.getLogger().setLevel(self.options.log_level)
         p = Process(target=run, args=args)
         self.scheduler_processes[app] = p
         self.update_schedulers()
@@ -473,7 +471,7 @@ class web2pyDialog(object):
         except:
             return self.error('invalid port number')
 
-        if self.options.ssl_certificate and self.options.ssl_private_key:
+        if self.options.server_key and self.options.server_cert:
             proto = 'https'
         else:
             proto = 'http'
@@ -492,11 +490,11 @@ class web2pyDialog(object):
                 pid_filename=options.pid_filename,
                 log_filename=options.log_filename,
                 profiler_dir=options.profiler_dir,
-                ssl_certificate=options.ssl_certificate,
-                ssl_private_key=options.ssl_private_key,
+                ssl_certificate=options.server_cert,
+                ssl_private_key=options.server_key,
                 ssl_ca_certificate=options.ca_cert,
-                min_threads=options.minthreads,
-                max_threads=options.maxthreads,
+                min_threads=options.min_threads,
+                max_threads=options.max_threads,
                 server_name=options.server_name,
                 request_queue_size=req_queue_size,
                 timeout=options.timeout,
@@ -582,321 +580,6 @@ class web2pyDialog(object):
         self.canvas.after(1000, self.update_canvas)
 
 
-def console():
-    """ Defines the behavior of the console web2py execution """
-    import optparse
-
-    parser = optparse.OptionParser(
-        usage='python %prog [options]',
-        version=ProgramVersion,
-        description='web2py Web Framework startup script.',
-        epilog='''NOTE: unless a password is specified (-a 'passwd')
-web2py will attempt to run a GUI to ask for it when starting the web server
-(if not disabled with --nogui).''')
-
-    parser.add_option('-i', '--ip',
-                      default='127.0.0.1',
-                      metavar='IP_ADDR', help=\
-        'IP address of the server (e.g., 127.0.0.1 or ::1); ' \
-        'Note: This value is ignored when using the --interfaces option')
-
-    parser.add_option('-p', '--port',
-                      default=8000,
-                      type='int', metavar='NUM', help=\
-        'port of server (%default); ' \
-        'Note: This value is ignored when using the --interfaces option')
-
-    parser.add_option('-G', '--GAE', dest='gae',
-                      default=None,
-                      metavar='APP_NAME', help=\
-        'will create app.yaml and gaehandler.py and exit')
-
-    parser.add_option('-a', '--password',
-                      default='',
-                      help=\
-        'password to be used for administration ' \
-        '(use "" to reuse the last password), ' \
-        'when no password is available the administrative ' \
-        'interface will be disabled')
-
-    parser.add_option('-c', '--ssl_certificate',
-                      default=None,
-                      metavar='FILE', help='server certificate file')
-
-    parser.add_option('-k', '--ssl_private_key',
-                      default=None,
-                      metavar='FILE', help='server private key file')
-
-    parser.add_option('--ca-cert', dest='ca_cert', # not needed
-                      default=None,
-                      metavar='FILE', help='CA certificate file')
-
-    parser.add_option('-d', '--pid_filename',
-                      default='httpserver.pid',
-                      metavar='FILE', help='server pid file (%default)')
-
-    parser.add_option('-l', '--log_filename',
-                      default='httpserver.log',
-                      metavar='FILE', help='server log file (%default)')
-
-    parser.add_option('-n', '--numthreads',
-                      default=None,
-                      type='int', metavar='NUM',
-                      help='number of threads (deprecated)')
-
-    parser.add_option('--minthreads',
-                      default=None,
-                      type='int', metavar='NUM',
-                      help='minimum number of server threads')
-
-    parser.add_option('--maxthreads',
-                      default=None,
-                      type='int', metavar='NUM',
-                      help='maximum number of server threads')
-
-    parser.add_option('-s', '--server_name',
-                      default=socket.gethostname(),
-                      help='web server name (%default)')
-
-    parser.add_option('-q', '--request_queue_size',
-                      default=5,
-                      type='int', metavar='NUM',
-                      help=\
-        'max number of queued requests when server unavailable (%default)')
-
-    parser.add_option('-o', '--timeout',
-                      default=10,
-                      type='int', metavar='SECONDS',
-                      help='timeout for individual request (%default seconds)')
-
-    parser.add_option('-z', '--shutdown_timeout',
-                      default=None,
-                      type='int', metavar='SECONDS',
-                      help=\
-        'timeout on server shutdown; this value is not used by ' \
-        'Rocket web server')
-
-    parser.add_option('--socket-timeout', dest='socket_timeout', # not needed
-                      default=5,
-                      type='int', metavar='SECONDS',
-                      help='timeout for socket (%default seconds)')
-
-    parser.add_option('-f', '--folder',
-                      default=os.getcwd(), metavar='WEB2PY_DIR',
-                      help='folder from which to run web2py')
-
-    parser.add_option('-v', '--verbose',
-                      default=False,
-                      action='store_true',
-                      help='increase --test and --run_system_tests verbosity')
-
-    parser.add_option('-Q', '--quiet',
-                      default=False,
-                      action='store_true',
-                      help='disable all output')
-
-    parser.add_option('-e', '--errors_to_console',
-                      default=False,
-                      action='store_true',
-                      help='log all errors to console')
-
-    parser.add_option('-D', '--debug', dest='debuglevel',
-                      default=30,
-                      type='int',
-                      metavar='LOG_LEVEL', help=\
-        'set log level (0-100, 0 means all, 100 means none; ' \
-        'default is %default)')
-
-    parser.add_option('-S', '--shell',
-                      default=None,
-                      metavar='APPNAME', help=\
-        'run web2py in interactive shell or IPython (if installed) with ' \
-        'specified appname (if app does not exist it will be created). ' \
-        'APPNAME like a/c/f?x=y (c, f and vars optional)')
-
-    parser.add_option('-B', '--bpython',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'run web2py in interactive shell or bpython (if installed) with ' \
-        'specified appname (if app does not exist it will be created). ' \
-        'Use combined with --shell')
-
-    parser.add_option('-P', '--plain',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'only use plain python shell; should be used with --shell option')
-
-    parser.add_option('-M', '--import_models',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'auto import model files (default is %default); should be used ' \
-        'with --shell option')
-
-    parser.add_option('-R', '--run',
-                      default='', # NOTE: used for sys.argv[0] if --shell
-                      metavar='PYTHON_FILE', help=\
-        'run PYTHON_FILE in web2py environment; ' \
-        'should be used with --shell option')
-
-    parser.add_option('-K', '--scheduler',
-                      default=None,
-                      metavar='APP_LIST', help=\
-        'run scheduled tasks for the specified apps: expects a list of ' \
-        'app names as app1,app2,app3 ' \
-        'or a list of app:groups as app1:group1:group2,app2:group1 ' \
-        '(only strings, no spaces allowed). NOTE: ' \
-        'Requires a scheduler defined in the models')
-
-    parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed
-                      default=False,
-                      action='store_true',
-                      help=\
-        'run schedulers alongside webserver, needs -K')
-
-    parser.add_option('-T', '--test',
-                      default=None,
-                      metavar='TEST_PATH', help=\
-        'run doctests in web2py environment; ' \
-        'TEST_PATH like a/c/f (c, f optional)')
-
-    parser.add_option('-C', '--cron', dest='extcron',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'trigger a cron run and exit; usually used when invoked ' \
-        'from a system crontab')
-
-    parser.add_option('--softcron',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'use software cron emulation instead of separate cron process, '\
-        'needs -Y; NOTE: use of software cron emulation is strongly '
-        'discouraged')
-
-    parser.add_option('-Y', '--run-cron', dest='runcron',
-                      default=False,
-                      action='store_true',
-                      help='start the background cron process')
-
-    parser.add_option('-J', '--cronjob',
-                      default=False,
-                      action='store_true',
-                      # NOTE: help suppressed because this option is
-                      #       intended for internal use only
-                      help=optparse.SUPPRESS_HELP)
-
-    parser.add_option('-L', '--config',
-                      default='',
-                      help='config file')
-
-    parser.add_option('-F', '--profiler', dest='profiler_dir',
-                      default=None,
-                      help='profiler dir')
-
-    parser.add_option('-t', '--taskbar',
-                      default=False,
-                      action='store_true',
-                      help='use web2py GUI and run in taskbar (system tray)')
-
-    parser.add_option('--nogui',
-                      default=False,
-                      action='store_true',
-                      help='do not run GUI')
-
-    parser.add_option('-A', '--args',
-                      default=None,
-                      help=\
-        'should be followed by a list of arguments to be passed to script, ' \
-        'to be used with -S; NOTE: must be the last option because eat all ' \
-        'remaining arguments')
-
-    parser.add_option('--no-banner', dest='no_banner', # not needed
-                      default=False,
-                      action='store_true',
-                      help='do not print header banner')
-
-    parser.add_option('--interfaces',
-                      default=None,
-                      help=\
-        'listen on multiple addresses: ' \
-        '"ip1:port1:key1:cert1:ca_cert1;ip2:port2:key2:cert2:ca_cert2;..." ' \
-        '(:key:cert:ca_cert optional; no spaces; IPv6 addresses must be in ' \
-        'square [] brackets)')
-
-    parser.add_option('--run_system_tests',
-                      default=False,
-                      action='store_true',
-                      help='run web2py tests')
-
-    parser.add_option('--with_coverage',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'collect coverage data when used with --run_system_tests; ' \
-        'require Python 2.7+ and the coverage module installed')
-
-    if '-A' in sys.argv:
-        k = sys.argv.index('-A')
-    elif '--args' in sys.argv:
-        k = sys.argv.index('--args')
-    else:
-        k = len(sys.argv)
-    sys.argv, other_args = sys.argv[:k], sys.argv[k + 1:]
-    (options, args) = parser.parse_args()
-    # TODO: warn or error if args (should be no unparsed arguments)
-    options.args = other_args
-
-    if options.taskbar and os.name != 'nt':
-        # TODO: warn and disable taskbar instead of exit
-        die('taskbar not supported on this platform')
-
-    if options.config.endswith('.py'):
-        options.config = options.config[:-3]
-    if options.config:
-        # import options from options.config file
-        try:
-            # FIXME: avoid __import__
-            options2 = __import__(options.config)
-        except:
-            die("cannot import config file %s" % options.config)
-        for key in dir(options2):
-            if hasattr(options, key):
-                setattr(options, key, getattr(options2, key))
-
-    # transform options.interfaces, in the form
-    # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3"
-    # (no spaces; optional key:cert:ca_cert indicate SSL), into
-    # a list of tuples
-    if options.interfaces:
-        interfaces = options.interfaces.split(';')
-        options.interfaces = []
-        for interface in interfaces:
-            if interface.startswith('['):
-                # IPv6
-                ip, if_remainder = interface.split(']', 1)
-                ip = ip[1:]
-                interface = if_remainder[1:].split(':')
-                interface.insert(0, ip)
-            else:
-                # IPv4
-                interface = interface.split(':')
-            interface[1] = int(interface[1])  # numeric port
-            options.interfaces.append(tuple(interface))
-
-    if options.numthreads is not None and options.minthreads is None:
-        options.minthreads = options.numthreads  # legacy
-
-    copy_options = copy.deepcopy(options)
-    copy_options.password = '******'
-    global_settings.cmd_options = copy_options
-
-    return options, args
-
-
 def check_existent_app(options, appname):
     if os.path.isdir(os.path.join(options.folder, 'applications', appname)):
         return True
@@ -921,11 +604,10 @@ def start_schedulers(options):
     except:
         sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
         return
-    logging.getLogger().setLevel(options.debuglevel)
+    logging.getLogger().setLevel(options.log_level)
 
-    apps = [[n.strip() for n in sched_app.split(':')]
-            for sched_app in options.scheduler.split(',')]
-    if len(apps) == 1 and not options.with_scheduler:
+    apps = [ag.split(':') for ag in options.schedulers]
+    if not options.with_scheduler and len(apps) == 1:
         app, code = get_code_for_scheduler(apps[0], options)
         if not app:
             return
@@ -968,7 +650,7 @@ def start():
     """ Starts server and other services """
 
     # get command line arguments
-    (options, args) = console()
+    options = console(version=ProgramVersion)
 
     if options.gae:
         # write app.yaml, gaehandler.py, and exit
@@ -990,7 +672,7 @@ def start():
         return
 
     logger = logging.getLogger("web2py")
-    logger.setLevel(options.debuglevel)
+    logger.setLevel(options.log_level)
 
     # on new installation build the scaffolding app
     create_welcome_w2p()
@@ -1027,36 +709,29 @@ def start():
         from pydal.drivers import DRIVERS
         print('Database drivers available: %s' % ', '.join(DRIVERS))
 
-    if options.test:
+    if options.run_doctests:
         # run doctests and exit
-        test(options.test, verbose=options.verbose)
+        test(options.run_doctests, verbose=options.verbose)
         return
 
     if options.shell:
         # run interactive shell and exit
-        sys.argv = [options.run] + options.args
+        sys.argv = [options.run or ''] + options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
             import_models=options.import_models, startfile=options.run,
-            cronjob=options.cronjob)
+            cron_job=options.cron_job)
         return
 
-    if options.extcron:
+    if options.cron_run:
         # run cron (extcron) and exit
         logger.debug('Starting extcron...')
         global_settings.web2py_crontype = 'external'
-        if options.scheduler:
-            # run cron for applications listed with --scheduler (-K)
-            apps = [app for app
-                    in map(lambda ag : ag.split(':', 1)[0].strip(), options.scheduler.split(','))
-                    if check_existent_app(options, app)]
-        else:
-            apps = None
-        extcron = newcron.extcron(options.folder, apps=apps)
+        extcron = newcron.extcron(options.folder, apps=options.crontabs)
         extcron.start()
         extcron.join()
         return
 
-    if options.scheduler and not options.with_scheduler:
+    if not options.with_scheduler and options.schedulers:
         # run schedulers and exit
         try:
             start_schedulers(options)
@@ -1064,22 +739,22 @@ def start():
             pass
         return
 
-    if options.runcron:
-        if options.softcron:
-            print('Using softcron (but this is not very efficient)')
+    if options.with_cron:
+        if options.soft_cron:
+            print('Using cron software emulation (but this is not very efficient)')
             global_settings.web2py_crontype = 'soft'
         else:
             # start hardcron thread
             logger.debug('Starting hardcron...')
             global_settings.web2py_crontype = 'hard'
-            newcron.hardcron(options.folder).start()
+            newcron.hardcron(options.folder, apps=options.crontabs).start()
 
     # if no password provided and have Tk library start GUI (when not
     # explicitly disabled), we also need a GUI to put in taskbar (system tray)
     # when requested
     root = None
 
-    if (not options.nogui and options.password == '') or options.taskbar:
+    if (not options.no_gui and options.password == '') or options.taskbar:
         try:
             if PY2:
                 import Tkinter as tkinter
@@ -1089,10 +764,10 @@ def start():
         except (ImportError, OSError):
             logger.warn(
                 'GUI not available because Tk library is not installed')
-            options.nogui = True
+            options.no_gui = True
         except:
             logger.exception('cannot get Tk root window, GUI disabled')
-            options.nogui = True
+            options.no_gui = True
 
     if root:
         # run GUI and exit
@@ -1121,7 +796,7 @@ end tell
 
     spt = None
 
-    if options.scheduler and options.with_scheduler:
+    if options.with_scheduler and options.schedulers:
         # start schedulers in a separate thread
         spt = threading.Thread(target=start_schedulers, args=(options,))
         spt.start()
@@ -1144,7 +819,7 @@ end tell
         ip = first_if[0]
         port = first_if[1]
 
-    if options.ssl_certificate and options.ssl_private_key:
+    if options.server_key and options.server_cert:
         proto = 'https'
     else:
         proto = 'http'
@@ -1186,11 +861,11 @@ end tell
                              pid_filename=options.pid_filename,
                              log_filename=options.log_filename,
                              profiler_dir=options.profiler_dir,
-                             ssl_certificate=options.ssl_certificate,
-                             ssl_private_key=options.ssl_private_key,
+                             ssl_certificate=options.server_cert,
+                             ssl_private_key=options.server_key,
                              ssl_ca_certificate=options.ca_cert,
-                             min_threads=options.minthreads,
-                             max_threads=options.maxthreads,
+                             min_threads=options.min_threads,
+                             max_threads=options.max_threads,
                              server_name=options.server_name,
                              request_queue_size=options.request_queue_size,
                              timeout=options.timeout,
diff --git a/scripts/web2py.fedora.sh b/scripts/web2py.fedora.sh
index bb3a2534..6a334565 100644
--- a/scripts/web2py.fedora.sh
+++ b/scripts/web2py.fedora.sh
@@ -29,7 +29,7 @@ cd $DAEMON_DIR
 
 start() {
         echo -n $"Starting $DESC ($NAME): "
-        daemon --check $NAME $PYTHON $DAEMON_DIR/web2py.py -Q --nogui -a $ADMINPASS -d $PIDFILE -p $PORT &
+        daemon --check $NAME $PYTHON $DAEMON_DIR/web2py.py -Q --no_gui -a $ADMINPASS -d $PIDFILE -p $PORT &
         RETVAL=$?
         if [ $RETVAL -eq 0 ]; then
                 touch /var/lock/subsys/$NAME

From 15daf70298bf6f8a0039d43bd8f813a44157bf73 Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Wed, 1 May 2019 21:09:36 -0700
Subject: [PATCH 048/111] pydal sync

---
 gluon/packages/dal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gluon/packages/dal b/gluon/packages/dal
index f1cf5aab..8c524fad 160000
--- a/gluon/packages/dal
+++ b/gluon/packages/dal
@@ -1 +1 @@
-Subproject commit f1cf5aab12b839ec168cc194f10c33e026b3de6b
+Subproject commit 8c524fad7307967bae695c6579642e458ab18f7f

From 20416b4d1cc9096e3d303bc9c54aac9c13b48576 Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Wed, 1 May 2019 21:23:34 -0700
Subject: [PATCH 049/111] syncing

---
 gluon/packages/dal  | 2 +-
 gluon/packages/yatl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/gluon/packages/dal b/gluon/packages/dal
index 8c524fad..cd9e8fd8 160000
--- a/gluon/packages/dal
+++ b/gluon/packages/dal
@@ -1 +1 @@
-Subproject commit 8c524fad7307967bae695c6579642e458ab18f7f
+Subproject commit cd9e8fd84b777a2e35e48d4eab1fc700eceee1a9
diff --git a/gluon/packages/yatl b/gluon/packages/yatl
index 3fb9abba..97ce196d 160000
--- a/gluon/packages/yatl
+++ b/gluon/packages/yatl
@@ -1 +1 @@
-Subproject commit 3fb9abbac896a8c00ce102b3d0e0503638bb7fd9
+Subproject commit 97ce196d2c248163254be15e4b22b1b855cc17e2

From 1a169b340ebe4867f434cab0e4c5fe947a6e5010 Mon Sep 17 00:00:00 2001
From: mdipierro 
Date: Wed, 1 May 2019 21:45:12 -0700
Subject: [PATCH 050/111] fixed python3 compatibility

---
 gluon/console.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/gluon/console.py b/gluon/console.py
index 5a17c52f..36036b4d 100644
--- a/gluon/console.py
+++ b/gluon/console.py
@@ -121,7 +121,11 @@ def console(version):
         def __call__(self, parser, namespace, values, option_string=None):
             if isinstance(values, list):
                 # must copy to avoid altering the option default value
-                items = argparse._ensure_value(namespace, self.dest, [])[:]
+                value = getattr(namespace, self.dest, None)
+                if value is None:
+                    value = []
+                    setattr(namespace, self.dest, value)
+                items = value[:]
                 # for options that allows multiple args (i.e. those declared
                 # with add_argument(..., nargs='+', ...)) the values are
                 # always placed into a list

From e3a981fc2c58e9603445822bf73d7fe0b22a0f3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Leonel=20C=C3=A2mara?= 
Date: Thu, 2 May 2019 16:09:10 +0100
Subject: [PATCH 051/111] Fixes #2182 possibly Fixes #2190

---
 gluon/serializers.py | 44 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 7 deletions(-)

diff --git a/gluon/serializers.py b/gluon/serializers.py
index 440b8657..a72bc216 100644
--- a/gluon/serializers.py
+++ b/gluon/serializers.py
@@ -119,13 +119,43 @@ def xml(value, encoding='UTF-8', key='document', quote=True):
     return ('' % encoding) + str(xml_rec(value, key, quote))
 
 
-def json(value, default=custom_json, indent=None, sort_keys=False):
-    value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent)
-    # replace JavaScript incompatible spacing
-    # http://timelessrepo.com/json-isnt-a-javascript-subset
-    # PY3 FIXME
-    # return value.replace(ur'\u2028', '\\u2028').replace(ur'\2029', '\\u2029')
-    return value
+class JSONEncoderForHTML(json_parser.JSONEncoder):
+    """An encoder that produces JSON safe to embed in HTML.
+    To embed JSON content in, say, a script tag on a web page, the
+    characters &, < and > should be escaped. They cannot be escaped
+    with the usual entities (e.g. &) because they are not expanded
+    within