Merge branch 'master' into migrator-pr

CHANGELOG

@@ -1,3 +1,24 @@
+## 2.19.0
+- new command line options (Thanks Paolo Pastori)
+
+OLD NAME                   NEW NAME
+==================         ==================
+--debug                     --log_level
+--nogui                     --no_gui
+--ssl_private_key           --server_key
+--ssl_certificate           --server_cert
+--minthreads                --min_threads
+--maxthreads                --max_threads
+--profiler                  --profiler_dir
+--run-cron                  --with_cron
+--softcron                  --soft_cron
+--cron                      --cron_run
+--cronjob *                 --cron_job *
+--test                      --run_doctests
+                            --add_options
+                            --interface
+                            --crontab
+
 ## 2.18.1-2.18.5
 - pydal 19.04
 - made template its own module (Yet Another Template Language)

Makefile

@@ -19,11 +19,11 @@ clean:
 	find applications/welcome/ -name '.*' -exec rm -f {} \; 
 	find . -name '*.pyc' -exec rm -f {} \;
 tests:
-	python web2py.py --run_system_tests
+	python web2py.py --verbose --run_system_tests
 coverage:
 	coverage erase --rcfile=gluon/tests/coverage.ini
 	export COVERAGE_PROCESS_START=gluon/tests/coverage.ini
-	python web2py.py --run_system_tests --with_coverage
+	python web2py.py --verbose --run_system_tests --with_coverage
 	coverage combine --rcfile=gluon/tests/coverage.ini
 	sleep 1
 	coverage html --rcfile=gluon/tests/coverage.ini

applications/admin/controllers/default.py

@@ -166,9 +166,9 @@ def check_version():
     new_version, version = check_new_version(request.env.web2py_version,
                                              WEB2PY_VERSION_URL)
 
-    if new_version == -1:
+    if new_version in (-1, -2):
         return A(T('Unable to check for upgrades'), _href=WEB2PY_URL)
-    elif new_version != True:
+    elif not new_version:
         return A(T('web2py is up to date'), _href=WEB2PY_URL)
     elif platform.system().lower() in ('windows', 'win32', 'win64') and os.path.exists("web2py.exe"):
         return SPAN('You should upgrade to %s' % version.split('(')[0])
@@ -1120,7 +1120,7 @@ def design():
     defines = {}
     for m in models:
         data = safe_read(apath('%s/models/%s' % (app, m), r=request))
-        defines[m] = regex_tables.findall(data)
+        defines[m] = re.findall(REGEX_DEFINE_TABLE, data, re.MULTILINE)
         defines[m].sort()
 
     # Get all controllers
@@ -1144,12 +1144,12 @@ def design():
     include = {}
     for c in views:
         data = safe_read(apath('%s/views/%s' % (app, c), r=request))
-        items = regex_extend.findall(data)
+        items = re.findall(REGEX_EXTEND, data, re.MULTILINE)
 
         if items:
             extend[c] = items[0][1]
 
-        items = regex_include.findall(data)
+        items = re.findall(REGEX_INCLUDE, data)
         include[c] = [i[1] for i in items]
 
     # Get all modules
@@ -1285,11 +1285,11 @@ def plugin():
     include = {}
     for c in views:
         data = safe_read(apath('%s/views/%s' % (app, c), r=request))
-        items = regex_extend.findall(data)
+        items = re.findall(REGEX_EXTEND, data, re.MULTILINE)
         if items:
             extend[c] = items[0][1]
 
-        items = regex_include.findall(data)
+        items = re.findall(REGEX_INCLUDE, data)
         include[c] = [i[1] for i in items]
 
     # Get all modules

applications/admin/models/0_imports.py

@@ -25,6 +25,5 @@ from gluon.utils import md5_hash
 from gluon.fileutils import listdir, cleanpath, up
 from gluon.fileutils import tar, tar_compiled, untar, fix_newlines
 from gluon.languages import findT, update_all_languages
-from gluon.myregex import *
 from gluon.restricted import *
 from gluon.compileapp import compile_application, remove_compiled_application

docker/alpine/web2py-rocket-ssl/Dockerfile

@@ -19,4 +19,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443

docker/alpine/web2py-rocket/Dockerfile

@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/centos/web2py-rocket/Dockerfile

@@ -25,4 +25,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/debian/web2py-rocket/Dockerfile

@@ -25,4 +25,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/fedora/web2py-rocket/Dockerfile

@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/opensuse/web2py-rocket/Dockerfile

@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/python/web2py-rocket-ssl/Dockerfile

@@ -18,4 +18,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443

docker/python/web2py-rocket/Dockerfile

@@ -20,4 +20,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/stack/web2py-rocket-nginx/web2py-rocket

@@ -22,4 +22,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl

@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443

docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl

@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443

docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl

@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443

docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl

@@ -17,4 +17,4 @@ WORKDIR /web2py
 
 EXPOSE 443
 
-CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443
+CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443

docker/ubuntu/web2py-rocket/Dockerfile

@@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py
 
 EXPOSE 8000
 
-CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000
+CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000

examples/options_std.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# when web2py is run as a windows service (web2py.py -W)
-# it does not load the command line options but it
-# expects to find configuration settings in a file called
-#
-#   web2py/options.py
-#
-# this file is an example for options.py
-
-import socket
-import os
-
-ip = '0.0.0.0'
-port = 80
-interfaces = [('0.0.0.0', 80)]
-               #,('0.0.0.0',443,'ssl_private_key.pem','ssl_certificate.pem')]
-password = '<recycle>'  # ## <recycle> means use the previous password
-pid_filename = 'httpserver.pid'
-log_filename = 'httpserver.log'
-profiler_filename = None
-ssl_certificate = ''  # 'ssl_certificate.pem'  # ## path to certificate file
-ssl_private_key = ''  # 'ssl_private_key.pem'  # ## path to private key file
-#numthreads = 50 # ## deprecated; remove
-minthreads = None
-maxthreads = None
-server_name = socket.gethostname()
-request_queue_size = 5
-timeout = 30
-shutdown_timeout = 5
-folder = os.getcwd()
-extcron = None
-nocron = None

gluon/admin.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
 
 """
 | This file is part of the web2py Web Framework
@@ -10,24 +10,34 @@ Utility functions for the Admin application
 -------------------------------------------
 """
 
+from __future__ import print_function
+
 import os
-import sys
 import traceback
-import zipfile
 from shutil import rmtree, copyfileobj
+import zipfile
+import sys
+
 from gluon.fileutils import (w2p_pack, create_app, w2p_unpack,
     w2p_pack_plugin, w2p_unpack_plugin,
     up, fix_newlines, abspath, recursive_unlink,
-    read_file, write_file, parse_version)
+    read_file, write_file, parse_version, missing_app_folders)
 from gluon.restricted import RestrictedError
 from gluon.settings import global_settings
 from gluon.cache import CacheOnDisk
 from gluon._compat import urlopen, to_native
 
+# TODO: move into add_path_first
 if not global_settings.web2py_runtime_gae:
     import site
 
 
+REGEX_DEFINE_TABLE = r"""^\w+\.define_table\(\s*['"](?P<name>\w+)['"]"""
+REGEX_EXTEND = r"""^\s*(?P<all>\{\{\s*extend\s+['"](?P<name>[^'"]+)['"]\s*\}\})"""
+REGEX_INCLUDE = r"""(?P<all>\{\{\s*include\s+['"](?P<name>[^'"]+)['"]\s*\}\})"""
+
+
+# TODO: swap arguments, let first ('r' or whatever) be mandatory 
 def apath(path='', r=None):
     """Builds a path inside an application folder
 
@@ -38,8 +48,9 @@ def apath(path='', r=None):
     """
 
     opath = up(r.folder)
-    while path[:3] == '../':
-        (opath, path) = (up(opath), path[3:])
+    while path.startswith('../'):
+        opath = up(opath)
+        path = path[3:]
     return os.path.join(opath, path).replace('\\', '/')
 
 
@@ -81,7 +92,7 @@ def app_pack_compiled(app, request, raise_ex=False):
         filename = apath('../deposit/%s.w2p' % app, request)
         w2p_pack(filename, apath(app, request), compiled=True)
         return filename
-    except Exception as e:
+    except Exception:
         if raise_ex:
             raise
         return None
@@ -105,7 +116,7 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.':
+                if not f.startswith('.'):
                     os.unlink(os.path.join(path, f))
             except IOError:
                 r = False
@@ -115,7 +126,7 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.':
+                if not f.startswith('.'):
                     recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
@@ -126,7 +137,7 @@ def app_cleanup(app, request):
         CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
-                if f[:1] != '.':
+                if not f.startswith('.'):
                     recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
@@ -211,9 +222,9 @@ def app_install(app, fobj, request, filename, overwrite=None):
 
     """
     did_mkdir = False
-    if filename[-4:] == '.w2p':
+    if filename.endswith('.w2p'):
         extension = 'w2p'
-    elif filename[-7:] == '.tar.gz':
+    elif filename.endswith('.tar.gz'):
         extension = 'tar.gz'
     else:
         extension = 'tar'
@@ -300,7 +311,8 @@ def plugin_install(app, fobj, request, filename):
     upname = apath('../deposit/%s' % filename, request)
 
     try:
-        write_file(upname, fobj.read(), 'wb')
+        with open(upname, 'wb') as appfp:
+            copyfileobj(fobj, appfp, 4194304) # 4 MB buffer
         path = apath(app, request)
         w2p_unpack_plugin(upname, path)
         fix_newlines(path)
@@ -322,7 +334,9 @@ def check_new_version(myversion, version_url):
         tuple: state, version
 
         - state : `True` if upgrade available, `False` if current
-          version is up-to-date, -1 on error
+                  version is up-to-date, -1 on error,
+                  -2 when the system is likely to be offline (no
+                  internet link available)
         - version : the most up-to-version available
 
     """
@@ -330,10 +344,19 @@ def check_new_version(myversion, version_url):
         version = to_native(urlopen(version_url).read())
         pversion = parse_version(version)
         pmyversion = parse_version(myversion)
-    except IOError:
-        import traceback
-        print(traceback.format_exc())
-        return -1, myversion
+    except IOError as e:
+        from socket import gaierror
+        if isinstance(getattr(e, 'reason', None), gaierror) and \
+            e.reason.errno == -2:
+            # assuming the version_url is ok the socket.gaierror
+            # (gaierror stands for getaddrinfo() error) that
+            # originates the exception is probably due to a
+            # missing internet link (i.e. the system is offline)
+            print('system is offline, cannot retrieve latest web2py version')
+            return -2, myversion
+        else:
+            print(traceback.format_exc())
+            return -1, myversion
 
     if pversion[:3]+pversion[-6:] > pmyversion[:3]+pmyversion[-6:]:
         return True, version
@@ -360,7 +383,7 @@ def unzip(filename, dir, subfolder=''):
     for name in sorted(zf.namelist()):
         if not name.startswith(subfolder):
             continue
-        # print name[n:]
+        # print(name[n:])
         if name.endswith('/'):
             folder = os.path.join(dir, name[n:])
             if not os.path.exists(folder):
@@ -421,6 +444,7 @@ def upgrade(request, url='http://web2py.com'):
         return False, e
 
 
+# TODO: move to fileutils
 def add_path_first(path):
     sys.path = [path] + [p for p in sys.path if (
         not p == path and not p == (path + '/'))]
@@ -429,21 +453,24 @@ def add_path_first(path):
             site.addsitedir(path)
 
 
+# TODO: move to fileutils
 def try_mkdir(path):
     if not os.path.exists(path):
         try:
             if os.path.islink(path):
-                # path is a broken link, try to mkdir the target of the link instead of the link itself.
+                # path is a broken link, try to mkdir the target of the link
+                # instead of the link itself.
                 os.mkdir(os.path.realpath(path))
             else:
                 os.mkdir(path)
         except OSError as e:
-            if e.strerror == 'File exists':  # In case of race condition.
+            if e.errno == 17:  # "File exists" (race condition).
                 pass
             else:
-                raise e
+                raise
 
 
+# TODO: move to fileutils
 def create_missing_folders():
     if not global_settings.web2py_runtime_gae:
         for path in ('applications', 'deposit', 'site-packages', 'logs'):
@@ -453,16 +480,16 @@ def create_missing_folders():
     paths = (global_settings.gluon_parent, abspath(
         'site-packages', gluon=True), abspath('gluon', gluon=True), '')
     """
-    paths = (global_settings.gluon_parent, abspath(
-        'site-packages', gluon=True), '')
-    [add_path_first(p) for p in paths]
+    for p in (global_settings.gluon_parent,
+              abspath('site-packages', gluon=True),
+              ''):
+        add_path_first(p)
 
 
+# TODO: move to fileutils
 def create_missing_app_folders(request):
     if not global_settings.web2py_runtime_gae:
         if request.folder not in global_settings.app_folders:
-            for subfolder in ('models', 'views', 'controllers', 'databases',
-                              'modules', 'cron', 'errors', 'sessions',
-                              'languages', 'static', 'private', 'uploads'):
-                try_mkdir(os.path.join(request.folder, subfolder))
+            for amf in missing_app_folders(request.folder):
+                try_mkdir(amf)
             global_settings.app_folders.add(request.folder)

gluon/compileapp.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -14,93 +13,43 @@ Note:
 """
 
 import re
-import fnmatch
-import os, sys
-import copy
 import random
-from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, integer_types, basestring, reduce, xrange, long, reload
-from gluon.storage import Storage, List
-from gluon.template import parse_template
-from gluon.restricted import restricted, compile2
-from gluon.fileutils import mktree, listdir, read_file, write_file
-from gluon.myregex import regex_expose, regex_longcomments
-from gluon.languages import TranslatorFactory
-from gluon.dal import DAL, Field
-from gluon.validators import Validator
-from pydal.base import BaseAdapter
-from gluon.sqlhtml import SQLFORM, SQLTABLE
-from gluon.cache import Cache
-from gluon.globals import current, Response
-from gluon import settings
-from gluon.cfs import getcfs
-from gluon import html
-from gluon import validators
-from gluon.http import HTTP, redirect
-import marshal
-import shutil
-import imp
-import logging
 import types
+import copy
 from functools import reduce
-from gluon import rewrite
-from gluon.custom_import import custom_import_install
 import py_compile
+import os
+from os.path import join as pjoin, exists
+import sys
+import imp
+import marshal
+import fnmatch
+import shutil
 
-logger = logging.getLogger("web2py")
+from gluon._compat import (builtin, PY2, unicodeT, to_native, to_bytes,
+    iteritems, integer_types, basestring, xrange, reload)
+from gluon.storage import Storage, List
+from gluon.globals import current, Response
+from gluon import html, validators, rewrite
+from gluon.http import HTTP, redirect
+from gluon.dal import DAL, Field
+from gluon.sqlhtml import SQLFORM, SQLTABLE
+from gluon.languages import TranslatorFactory
+from gluon.cache import Cache
+from gluon.validators import Validator
+from gluon.settings import global_settings
+from pydal.base import BaseAdapter
+from gluon.custom_import import custom_import_install
+from gluon.fileutils import mktree, listdir, read_file, write_file, abspath
+from gluon.template import parse_template
+from gluon.cfs import getcfs
+from gluon.restricted import restricted, compile2
+from gluon.admin import add_path_first
 
-is_pypy = settings.global_settings.is_pypy
-is_gae = settings.global_settings.web2py_runtime_gae
-is_jython = settings.global_settings.is_jython
-pjoin = os.path.join
-
-if PY2:
-    marshal_header_size = 8
-else:
-    marshal_header_size = 16 if sys.version_info[1] >= 7 else 12
-
-TEST_CODE = \
-    r"""
-def _TEST():
-    import doctest, sys, cStringIO, types, cgi, gluon.fileutils
-    if not gluon.fileutils.check_credentials(request):
-        raise HTTP(401, web2py_error='invalid credentials')
-    stdout = sys.stdout
-    html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
-        % request.controller
-    for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
-        eval_key = eval(key)
-        if type(eval_key) == types.FunctionType:
-            number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
-            if number_doctests>0:
-                sys.stdout = cStringIO.StringIO()
-                name = '%s/controllers/%s.py in %s.__doc__' \
-                    % (request.folder, request.controller, key)
-                doctest.run_docstring_examples(eval_key,
-                    globals(), False, name=name)
-                report = sys.stdout.getvalue().strip()
-                if report:
-                    pf = 'failed'
-                else:
-                    pf = 'passed'
-                html += '<h3 class="%s">Function %s [%s]</h3>\n' \
-                    % (pf, key, pf)
-                if report:
-                    html += CODE(report, language='web2py', \
-                        link='/examples/global/vars/').xml()
-                html += '<br/>\n'
-            else:
-                html += \
-                    '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
-                    % (key)
-    response._vars = html
-    sys.stdout = stdout
-_TEST()
-"""
 
 CACHED_REGEXES = {}
 CACHED_REGEXES_MAX_SIZE = 1000
 
-
 def re_compile(regex):
     try:
         return CACHED_REGEXES[regex]
@@ -111,22 +60,6 @@ def re_compile(regex):
         return compiled_regex
 
 
-class mybuiltin(object):
-    """
-    NOTE could simple use a dict and populate it,
-    NOTE not sure if this changes things though if monkey patching import.....
-    """
-    # __builtins__
-    def __getitem__(self, key):
-        try:
-            return getattr(builtin, key)
-        except AttributeError:
-            raise KeyError(key)
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-
 def LOAD(c=None, f='index', args=None, vars=None,
          extension=None, target=None, ajax=False, ajax_trap=False,
          url=None, user_signature=False, timeout=None, times=1,
@@ -140,10 +73,10 @@ def LOAD(c=None, f='index', args=None, vars=None,
         vars(dict): vars
         extension(str): extension
         target(str): id of the target
-        ajax(bool): True to enable AJAX bahaviour
+        ajax(bool): True to enable AJAX behaviour
         ajax_trap(bool): True if `ajax` is set to `True`, traps
             both links and forms "inside" the target
-        url(str): overrides `c`,`f`,`args` and `vars`
+        url(str): overrides `c`, `f`, `args` and `vars`
         user_signature(bool): adds hmac signature to all links
             with a key that is different for every user
         timeout(int): in milliseconds, specifies the time to wait before
@@ -153,7 +86,6 @@ def LOAD(c=None, f='index', args=None, vars=None,
             "infinity" or "continuous" are accepted to reload indefinitely the
             component
     """
-    from gluon.html import TAG, DIV, URL, SCRIPT, XML
     if args is None:
         args = []
     vars = Storage(vars or {})
@@ -163,19 +95,21 @@ def LOAD(c=None, f='index', args=None, vars=None,
     if '.' in f:
         f, extension = f.rsplit('.', 1)
     if url or ajax:
-        url = url or URL(request.application, c, f, r=request,
-                         args=args, vars=vars, extension=extension,
-                         user_signature=user_signature)
+        url = url or html.URL(request.application, c, f, r=request,
+                              args=args, vars=vars, extension=extension,
+                              user_signature=user_signature)
         # timing options
         if isinstance(times, basestring):
             if times.upper() in ("INFINITY", "CONTINUOUS"):
                 times = "Infinity"
             else:
+                # FIXME: should be a ValueError
                 raise TypeError("Unsupported times argument %s" % times)
         elif isinstance(times, int):
             if times <= 0:
                 raise ValueError("Times argument must be greater than zero, 'Infinity' or None")
         else:
+            # NOTE: why do not use ValueError only?
             raise TypeError("Unsupported times argument type %s" % type(times))
         if timeout is not None:
             if not isinstance(timeout, integer_types):
@@ -191,7 +125,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
             statement = "$.web2py.component('%s','%s');" % (url, target)
         attr['_data-w2p_remote'] = url
         if target is not None:
-            return DIV(content, **attr)
+            return html.DIV(content, **attr)
 
     else:
         if not isinstance(args, (list, tuple)):
@@ -211,7 +145,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
             '/'.join([request.application, c, f] +
                      [str(a) for a in other_request.args])
         other_request.env.query_string = \
-            vars and URL(vars=vars).split('?')[1] or ''
+            vars and html.URL(vars=vars).split('?')[1] or ''
         other_request.env.http_web2py_component_location = \
             request.env.path_info
         other_request.cid = target
@@ -220,7 +154,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
         # A bit nasty but needed to use LOAD on action decorates with @request.restful()
         other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
 
-        other_environment = copy.copy(current.globalenv)  # NASTY
+        other_environment = copy.copy(current.globalenv)  # FIXME: NASTY
 
         other_response._view_environment = other_environment
         other_response.generic_patterns = \
@@ -241,12 +175,12 @@ def LOAD(c=None, f='index', args=None, vars=None,
         current.request, current.response = original_request, original_response
         js = None
         if ajax_trap:
-            link = URL(request.application, c, f, r=request,
-                       args=args, vars=vars, extension=extension,
-                       user_signature=user_signature)
+            link = html.URL(request.application, c, f, r=request,
+                            args=args, vars=vars, extension=extension,
+                            user_signature=user_signature)
             js = "$.web2py.trap_form('%s','%s');" % (link, target)
-        script = js and SCRIPT(js, _type="text/javascript") or ''
-        return TAG[''](DIV(XML(page), **attr), script)
+        script = js and html.SCRIPT(js, _type="text/javascript") or ''
+        return html.TAG[''](html.DIV(html.XML(page), **attr), script)
 
 
 class LoadFactory(object):
@@ -262,7 +196,6 @@ class LoadFactory(object):
         if args is None:
             args = []
         vars = Storage(vars or {})
-        from . import globals
         target = target or 'c' + str(random.random())[2:]
         attr['_id'] = target
         request = current.request
@@ -289,7 +222,7 @@ class LoadFactory(object):
             other_request.vars = vars
             other_request.get_vars = vars
             other_request.post_vars = Storage()
-            other_response = globals.Response()
+            other_response = Response()
             other_request.env.path_info = '/' + \
                 '/'.join([request.application, c, f] +
                          [str(a) for a in other_request.args])
@@ -354,37 +287,36 @@ def local_import_aux(name, reload_force=False, app='welcome'):
     if reload_force:
         reload(module)
     return module
+#
+# OLD IMPLEMENTATION:
+#
+#   items = name.replace('/','.').split('.')
+#   filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1])
+#   imp.acquire_lock()
+#   try:
+#       file=None
+#       (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
+#       if not path in sys.modules or reload:
+#           if is_gae:
+#               module={}
+#               execfile(path,{},module)
+#               module=Storage(module)
+#           else:
+#               module = imp.load_module(path,file,path,desc)
+#           sys.modules[path] = module
+#       else:
+#           module = sys.modules[path]
+#   except Exception, e:
+#       module = None
+#   if file:
+#       file.close()
+#   imp.release_lock()
+#   if not module:
+#       raise ImportError, "cannot find module %s in %s" % (
+#           filename, modulepath)
+#   return module
 
 
-"""
-OLD IMPLEMENTATION:
-    items = name.replace('/','.').split('.')
-    filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1])
-    imp.acquire_lock()
-    try:
-        file=None
-        (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
-        if not path in sys.modules or reload:
-            if is_gae:
-                module={}
-                execfile(path,{},module)
-                module=Storage(module)
-            else:
-                module = imp.load_module(path,file,path,desc)
-            sys.modules[path] = module
-        else:
-            module = sys.modules[path]
-    except Exception, e:
-        module = None
-    if file:
-        file.close()
-    imp.release_lock()
-    if not module:
-        raise ImportError, "cannot find module %s in %s" % (
-            filename, modulepath)
-    return module
-"""
-
 _base_environment_ = dict((k, getattr(html, k)) for k in html.__all__)
 _base_environment_.update(
     (k, getattr(validators, k)) for k in validators.__all__)
@@ -424,7 +356,7 @@ def build_environment(request, response, session, store_current=True):
         r'^%s/%s/\w+\.py$' % (request.controller, request.function)
         ]
 
-    T = environment['T'] = TranslatorFactory(os.path.join(request.folder, 'languages'),
+    T = environment['T'] = TranslatorFactory(pjoin(request.folder, 'languages'),
                                              request.env.http_accept_language)
     c = environment['cache'] = Cache(request)
 
@@ -439,7 +371,23 @@ def build_environment(request, response, session, store_current=True):
         current.T = T
         current.cache = c
 
-    if is_jython:  # jython hack
+    if global_settings.is_jython:
+        # jython hack
+        class mybuiltin(object):
+            """
+            NOTE could simple use a dict and populate it,
+            NOTE not sure if this changes things though if monkey patching import.....
+            """
+            # __builtins__
+            def __getitem__(self, key):
+                try:
+                    return getattr(builtin, key)
+                except AttributeError:
+                    raise KeyError(key)
+
+            def __setitem__(self, key, value):
+                setattr(self, key, value)
+
         global __builtins__
         __builtins__ = mybuiltin()
 
@@ -462,6 +410,11 @@ def save_pyc(filename):
     py_compile.compile(filename, cfile=cfile)
 
 
+if PY2:
+    MARSHAL_HEADER_SIZE = 8
+else:
+    MARSHAL_HEADER_SIZE = 16 if sys.version_info[1] >= 7 else 12
+
 def read_pyc(filename):
     """
     Read the code inside a bytecode compiled file if the MAGIC number is
@@ -471,44 +424,49 @@ def read_pyc(filename):
         a code object
     """
     data = read_file(filename, 'rb')
-    if not is_gae and data[:4] != imp.get_magic():
+    if not global_settings.web2py_runtime_gae and \
+        not data.startswith(imp.get_magic()):
         raise SystemError('compiled code is incompatible')
-    return marshal.loads(data[marshal_header_size:])
+    return marshal.loads(data[MARSHAL_HEADER_SIZE:])
 
 
+REGEX_PATH_CLASS = r"[\w%s-]" % os.sep if os.sep != '\\' else r'[\w\\-]'
+REGEX_VIEW_PATH = r"%s+(?:\.\w+)*$" % REGEX_PATH_CLASS
+
 def compile_views(folder, skip_failed_views=False):
     """
     Compiles all the views in the application specified by `folder`
     """
-
     path = pjoin(folder, 'views')
     failed_views = []
-    for fname in listdir(path, '^[\w/\-]+(\.\w+)*$'):
+    for fname in listdir(path, REGEX_VIEW_PATH):
         try:
             data = parse_template(fname, path)
         except Exception as e:
             if skip_failed_views:
                 failed_views.append(fname)
             else:
+                # FIXME: should raise something more specific than Exception
                 raise Exception("%s in %s" % (e, fname))
         else:
-            filename = 'views.%s.py' % fname.replace(os.path.sep, '.')
+            filename = 'views.%s.py' % fname.replace(os.sep, '.')
             filename = pjoin(folder, 'compiled', filename)
             write_file(filename, data)
             save_pyc(filename)
             os.unlink(filename)
-    return failed_views if failed_views else None
+    return failed_views or None
 
 
+REGEX_MODEL_PATH = r"%s+\.py$" % REGEX_PATH_CLASS
+
 def compile_models(folder):
     """
     Compiles all the models in the application specified by `folder`
     """
-
     path = pjoin(folder, 'models')
-    for fname in listdir(path, '.+\.py$'):
+    for fname in listdir(path, REGEX_MODEL_PATH):
         data = read_file(pjoin(path, fname))
-        modelfile = 'models.'+fname.replace(os.path.sep, '.')
+        modelfile = 'models.'+fname.replace(os.sep, '.')
         filename = pjoin(folder, 'compiled', modelfile)
         mktree(filename)
         write_file(filename, data)
@@ -516,19 +474,22 @@ def compile_models(folder):
         os.unlink(filename)
 
 
-def find_exposed_functions(data):
-    data = regex_longcomments.sub('', data)
-    return regex_expose.findall(data)
+REGEX_LONG_STRING = re.compile('(""".*?"""|' "'''.*?''')", re.DOTALL)
+REGEX_EXPOSED = re.compile(r'^def\s+(_?[a-zA-Z0-9]\w*)\( *\)\s*:', re.MULTILINE)
 
+def find_exposed_functions(data):
+    data = REGEX_LONG_STRING.sub('', data)
+    return REGEX_EXPOSED.findall(data)
+
+
+REGEX_CONTROLLER = r'[\w-]+\.py$'
 
 def compile_controllers(folder):
     """
     Compiles all the controllers in the application specified by `folder`
     """
-
     path = pjoin(folder, 'controllers')
-    for fname in listdir(path, '.+\.py$'):
-        ### why is this here? save_pyc(pjoin(path, file))
+    for fname in listdir(path, REGEX_CONTROLLER):
         data = read_file(pjoin(path, fname))
         exposed = find_exposed_functions(data)
         for function in exposed:
@@ -541,20 +502,22 @@ def compile_controllers(folder):
             os.unlink(filename)
 
 
-def model_cmp(a, b, sep='.'):
-    return cmp(a.count(sep), b.count(sep)) or cmp(a, b)
+if PY2:
+    def model_cmp(a, b, sep='.'):
+        return cmp(a.count(sep), b.count(sep)) or cmp(a, b)
+
+    def model_cmp_sep(a, b, sep=os.sep):
+        return model_cmp(a, b, sep)
 
 
-def model_cmp_sep(a, b, sep=os.path.sep):
-    return model_cmp(a, b, sep)
-
+REGEX_COMPILED_MODEL = r'models[_.][\w.-]+\.pyc$'
+REGEX_MODEL = r'[\w-]+\.py$'
 
 def run_models_in(environment):
     """
     Runs all models (in the app specified by the current folder)
     It tries pre-compiled models first before compiling them.
     """
-
     request = current.request
     folder = request.folder
     c = request.controller
@@ -563,19 +526,21 @@ def run_models_in(environment):
 
     path = pjoin(folder, 'models')
     cpath = pjoin(folder, 'compiled')
-    compiled = os.path.exists(cpath)
+    compiled = exists(cpath)
     if PY2:
         if compiled:
-            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), model_cmp)
+            models = sorted(listdir(cpath, REGEX_COMPILED_MODEL, 0),
+                            model_cmp)
         else:
-            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep)
+            models = sorted(listdir(path, REGEX_MODEL, 0, sort=False),
+                            model_cmp_sep)
     else:
         if compiled:
-            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0),
+            models = sorted(listdir(cpath, REGEX_COMPILED_MODEL, 0),
                             key=lambda f: '{0:03d}'.format(f.count('.')) + f)
         else:
-            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False),
-                            key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
+            models = sorted(listdir(path, REGEX_MODEL, 0, sort=False),
+                            key=lambda f: '{0:03d}'.format(f.count(os.sep)) + f)
 
     models_to_run = None
     for model in models:
@@ -589,7 +554,7 @@ def run_models_in(environment):
                 fname = model[n:-4].replace('.', '/')+'.py'
             else:
                 n = len(path)+1
-                fname = model[n:].replace(os.path.sep, '/')
+                fname = model[n:].replace(os.sep, '/')
             if not regex.search(fname) and c != 'appadmin':
                 continue
             elif compiled:
@@ -600,19 +565,56 @@ def run_models_in(environment):
             restricted(ccode, environment, layer=model)
 
 
+TEST_CODE = r"""
+def _TEST():
+    import doctest, sys, cStringIO, types, cgi, gluon.fileutils
+    if not gluon.fileutils.check_credentials(request):
+        raise HTTP(401, web2py_error='invalid credentials')
+    stdout = sys.stdout
+    html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
+        % request.controller
+    for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
+        eval_key = eval(key)
+        if type(eval_key) == types.FunctionType:
+            number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
+            if number_doctests>0:
+                sys.stdout = cStringIO.StringIO()
+                name = '%s/controllers/%s.py in %s.__doc__' \
+                    % (request.folder, request.controller, key)
+                doctest.run_docstring_examples(eval_key,
+                    globals(), False, name=name)
+                report = sys.stdout.getvalue().strip()
+                if report:
+                    pf = 'failed'
+                else:
+                    pf = 'passed'
+                html += '<h3 class="%s">Function %s [%s]</h3>\n' \
+                    % (pf, key, pf)
+                if report:
+                    html += CODE(report, language='web2py', \
+                        link='/examples/global/vars/').xml()
+                html += '<br/>\n'
+            else:
+                html += \
+                    '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
+                    % (key)
+    response._vars = html
+    sys.stdout = stdout
+_TEST()
+"""
+
 def run_controller_in(controller, function, environment):
     """
     Runs the controller.function() (for the app specified by
     the current folder).
     It tries pre-compiled controller.function.pyc first before compiling it.
     """
-
     # if compiled should run compiled!
     folder = current.request.folder
     cpath = pjoin(folder, 'compiled')
     badc = 'invalid controller (%s/%s)' % (controller, function)
     badf = 'invalid function (%s/%s)' % (controller, function)
-    if os.path.exists(cpath):
+    if exists(cpath):
         filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function))
         try:
             ccode = getcfs(filename, filename, lambda: read_pyc(filename))
@@ -622,8 +624,6 @@ def run_controller_in(controller, function, environment):
                        web2py_error=badf)
     elif function == '_TEST':
         # TESTING: adjust the path to include site packages
-        from gluon.settings import global_settings
-        from gluon.admin import abspath, add_path_first
         paths = (global_settings.gluon_parent, abspath(
             'site-packages', gluon=True), abspath('gluon', gluon=True), '')
         [add_path_first(path) for path in paths]
@@ -631,7 +631,7 @@ def run_controller_in(controller, function, environment):
 
         filename = pjoin(folder, 'controllers/%s.py'
                                  % controller)
-        if not os.path.exists(filename):
+        if not exists(filename):
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badc,
                        web2py_error=badc)
@@ -696,12 +696,12 @@ def run_view_in(environment):
         layer = 'file stream'
     else:
         filename = pjoin(folder, 'views', view)
-        if os.path.exists(cpath):  # compiled views
+        if exists(cpath):  # compiled views
             x = view.replace('/', '.')
             files = ['views.%s.pyc' % x]
-            is_compiled = os.path.exists(pjoin(cpath, files[0]))
+            is_compiled = exists(pjoin(cpath, files[0]))
             # Don't use a generic view if the non-compiled view exists.
-            if is_compiled or (not is_compiled and not os.path.exists(filename)):
+            if is_compiled or (not is_compiled and not exists(filename)):
                 if allow_generic:
                     files.append('views.generic.%s.pyc' % request.extension)
                 # for backward compatibility
@@ -712,16 +712,16 @@ def run_view_in(environment):
                 # end backward compatibility code
                 for f in files:
                     compiled = pjoin(cpath, f)
-                    if os.path.exists(compiled):
+                    if exists(compiled):
                         ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled))
                         layer = compiled
                         break
         # if the view is not compiled
         if not layer:
-            if not os.path.exists(filename) and allow_generic:
+            if not exists(filename) and allow_generic:
                 view = 'generic.' + request.extension
                 filename = pjoin(folder, 'views', view)
-            if not os.path.exists(filename):
+            if not exists(filename):
                 raise HTTP(404,
                            rewrite.THREAD_LOCAL.routes.error_message % badv,
                            web2py_error=badv)
@@ -737,6 +737,8 @@ def run_view_in(environment):
     return environment['response'].body.getvalue()
 
 
+REGEX_COMPILED_CONTROLLER = r'[\w-]+\.pyc$'
+
 def remove_compiled_application(folder):
     """
     Deletes the folder `compiled` containing the compiled application.
@@ -744,7 +746,7 @@ def remove_compiled_application(folder):
     try:
         shutil.rmtree(pjoin(folder, 'compiled'))
         path = pjoin(folder, 'controllers')
-        for file in listdir(path, '.*\.pyc$', drop=False):
+        for file in listdir(path, REGEX_COMPILED_CONTROLLER, drop=False):
             os.unlink(file)
     except OSError:
         pass

gluon/console.py

@@ -0,0 +1,722 @@
+# -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
+"""
+| This file is part of the web2py Web Framework
+| Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
+| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
+
+Command line interface
+----------------------
+
+The processing of all command line arguments is done using
+the argparse library in the console function.
+
+The basic principle is to process and check for all options
+in a single place, this place is the parse_args function.
+Notice that when I say all options I mean really all,
+options sourced from a configuration file are included.
+
+A brief summary of options style follows,
+for the benefit of code maintainers/developers:
+
+- use the underscore to split words in long names (as in
+  '--run_system_tests')
+- remember to allow the '-' too as word separator (e.g.
+  '--run-system-tests') but do not use this form on help
+- prefer short names on help messages, instead use
+  all options names in warning/error messages (e.g.
+  '-R/--run requires -S/--shell')
+"""
+
+from __future__ import print_function
+
+__author__ = 'Paolo Pastori'
+
+import os.path
+import argparse
+import logging
+import socket
+import sys
+import re
+import ast
+from collections import OrderedDict
+import copy
+
+from gluon._compat import PY2
+from gluon.shell import die
+from gluon.utils import is_valid_ip_address
+from gluon.settings import global_settings
+
+
+def warn(msg):
+    print("%s: warning: %s" % (sys.argv[0], msg), file=sys.stderr)
+
+def is_appdir(applications_parent, app):
+    return os.path.isdir(os.path.join(applications_parent, 'applications', app))
+
+
+def console(version):
+    """
+    Load command line options.
+    Trivial -h/--help and --version options are also processed.
+
+    Returns a namespace object (in the sense of argparse)
+    with all options loaded.
+    """
+
+    # replacement hints for deprecated options
+    deprecated_opts = {
+        '--debug': '--log_level',
+        '--nogui': '--no_gui',
+        '--ssl_private_key': '--server_key',
+        '--ssl_certificate': '--server_cert',
+        '--interfaces': None, # dest is 'interfaces', hint is '--interface'
+        '-n': '--min_threads', '--numthreads': '--min_threads',
+        '--minthreads': '--min_threads',
+        '--maxthreads': '--max_threads',
+        '-z': None, '--shutdown_timeout': None,
+        '--profiler': '--profiler_dir',
+        '--run-cron': '--with_cron',
+        '--softcron': '--soft_cron',
+        '--cron': '--cron_run',
+        '--test': '--run_doctests'
+    }
+
+    class HelpFormatter2(argparse.HelpFormatter):
+        """Hides the options listed in _hidden_options in usage help."""
+
+        # NOTE: preferred style for long options name is to use '_'
+        #       between words (as in 'no_gui'), also accept the '-' in
+        #       most of the options but do not show both versions on help
+        _omitted_opts = ('--add-options', '--errors-to-console',
+            '--no-banner', '--log-level', '--no-gui', '--import-models',
+            '--server-name', '--server-key', '--server-cert', '--ca-cert',
+            '--pid-filename', '--log-filename', '--min-threads',
+            '--max-threads', '--request-queue-size', '--socket-timeout',
+            '--profiler-dir', '--with-scheduler', '--with-cron',
+            '--soft-cron', '--cron-run',
+            '--run-doctests', '--run-system-tests', '--with-coverage')
+
+        _hidden_options = _omitted_opts + tuple(deprecated_opts.keys())
+
+        def _format_action_invocation(self, action):
+            if not action.option_strings:
+                return super(HelpFormatter2, self)._format_action_invocation(action)
+            parts = []
+            if action.nargs == 0:
+                parts.extend(filter(lambda o : o not in self._hidden_options,
+                                    action.option_strings))
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    if option_string in self._hidden_options:
+                        continue
+                    parts.append('%s %s' % (option_string, args_string))
+            return ', '.join(parts)
+
+    class ExtendAction(argparse._AppendAction):
+        """Action to accumulate values in a flat list."""
+
+        def __call__(self, parser, namespace, values, option_string=None):
+            if isinstance(values, list):
+                # must copy to avoid altering the option default value
+                value = getattr(namespace, self.dest, None)
+                if value is None:
+                    value = []
+                    setattr(namespace, self.dest, value)
+                items = value[:]
+                # for options that allows multiple args (i.e. those declared
+                # with add_argument(..., nargs='+', ...)) the values are
+                # always placed into a list
+                while len(values) == 1 and isinstance(values[0], list):
+                    values = values[0]
+                items.extend(values)
+                setattr(namespace, self.dest, items)
+            else:
+                super(ExtendAction, self).__call__(parser, namespace, values, option_string)
+
+    parser = argparse.ArgumentParser(
+        usage='python %(prog)s [options]',
+        description='web2py Web Framework startup script.',
+        epilog='''NOTE: unless a password is specified (-a 'passwd')
+web2py will attempt to run a GUI to ask for it when starting the web server
+(if not disabled with --no_gui).''',
+        formatter_class=HelpFormatter2,
+        add_help=False) # do not add -h/--help option
+
+    # global options
+    g = parser.add_argument_group('global options')
+    g.add_argument('-h', '--help', action='help',
+                   help='show this help message and exit')
+    g.add_argument('--version', action='version',
+                   version=version,
+                   help="show program's version and exit")
+    folder = os.getcwd()
+    g.add_argument('-f', '--folder',
+                   default=folder, metavar='WEB2PY_DIR',
+                   help='web2py installation directory (%(default)s)')
+    def existing_file(v):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if not os.path.exists(v):
+            raise argparse.ArgumentTypeError("file %r not found" % v)
+        return v
+    g.add_argument('-L', '--config',
+                   type=existing_file,
+                   metavar='PYTHON_FILE',
+                   help='read all options from PYTHON_FILE')
+    g.add_argument('--add_options', '--add-options',
+                   default=False,
+                   action='store_true', help=
+        'add options to existing ones, useful with -L only')
+    g.add_argument('-a', '--password',
+                   default='<ask>', help=
+        'password to be used for administration (use "<recycle>" '
+        'to reuse the last password), when no password is available '
+        'the administrative web interface will be disabled')
+    g.add_argument('-e', '--errors_to_console', '--errors-to-console',
+                   default=False,
+                   action='store_true',
+                   help='log application errors to console')
+    g.add_argument('--no_banner', '--no-banner',
+                   default=False,
+                   action='store_true',
+                   help='do not print header banner')
+    g.add_argument('-Q', '--quiet',
+                   default=False,
+                   action='store_true',
+                   help='disable all output')
+    integer_log_level = []
+    def log_level(v):
+        # try to convert a lgging level name to its numeric value,
+        # could use logging.getLevelName but not with
+        # 3.4 <= Python < 3.4.2, see
+        # https://docs.python.org/3/library/logging.html#logging.getLevelName)
+        try:
+            name2level = logging._levelNames
+        except AttributeError:
+            # logging._levelNames has gone with Python 3.4, see
+            # https://github.com/python/cpython/commit/3b84eae03ebd8122fdbdced3d85999dd9aedfc7e
+            name2level = logging._nameToLevel
+        try:
+            return name2level[v.upper()]
+        except KeyError:
+            pass
+        try:
+            ill = int(v)
+            # value deprecated: integer in range(101)
+            if 0 <= ill <= 100:
+                integer_log_level.append(ill)
+                return ill
+        except ValueError:
+            pass
+        raise argparse.ArgumentTypeError("bad level %r" % v)
+    g.add_argument('-D', '--log_level', '--log-level',
+                   '--debug', # deprecated
+                   default='WARNING',
+                   type=log_level,
+                   metavar='LOG_LEVEL', help=
+        'set log level, allowed values are: NOTSET, DEBUG, INFO, WARN, '
+        'WARNING, ERROR, and CRITICAL, also lowercase (default is '
+        '%(default)s)')
+
+    # GUI options
+    g = parser.add_argument_group('GUI options')
+    g.add_argument('--no_gui', '--no-gui',
+                   '--nogui', # deprecated
+                   default=False,
+                   action='store_true',
+                   help='do not run GUI')
+    g.add_argument('-t', '--taskbar',
+                   default=False,
+                   action='store_true',
+                   help='run in taskbar (system tray)')
+
+    # console options
+    g = parser.add_argument_group('console options')
+    g.add_argument('-S', '--shell',
+                   metavar='APP_ENV', help=
+        'run web2py in Python interactive shell or IPython (if installed) '
+        'with specified application environment (if application does not '
+        'exist it will be created). APP_ENV like a/c/f?x=y (c, f and vars '
+        'optional), if APP_ENV include the action f then after the '
+        'action execution the interpreter is exited')
+    g.add_argument('-B', '--bpython',
+                   default=False,
+                   action='store_true', help=
+        'use bpython (if installed) when running in interactive shell, '
+        'see -S above')
+    g.add_argument('-P', '--plain',
+                   default=False,
+                   action='store_true', help=
+        'use plain Python shell when running in interactive shell, '
+        'see -S above')
+    g.add_argument('-M', '--import_models', '--import-models',
+                   default=False,
+                   action='store_true', help=
+        'auto import model files when running in interactive shell '
+        '(default is %(default)s), see -S above. NOTE: when the APP_ENV '
+        'argument of -S include a controller c automatic import of '
+        'models is always enabled')
+    g.add_argument('--force_migrate',
+                   default=False,
+                   action='store_true',
+                   help=
+                   'force DAL to migrate all tables that should be migrated when enabled; '
+                   'monkeypatch in the DAL class to force _migrate_enabled=True')
+    g.add_argument('-R', '--run',
+                   type=existing_file,
+                   metavar='PYTHON_FILE', help=
+        'run PYTHON_FILE in web2py environment; require -S')
+    g.add_argument('-A', '--args',
+                   default=[],
+                   nargs=argparse.REMAINDER, help=
+        'use this to pass arguments to the PYTHON_FILE above; require '
+        '-R. NOTE: must be the last option because eat all remaining '
+        'arguments')
+
+    # web server options
+    g = parser.add_argument_group('web server options')
+    g.add_argument('-s', '--server_name', '--server-name',
+                   default=socket.gethostname(),
+                   help='web server name (%(default)s)')
+    def ip_addr(v):
+        if not is_valid_ip_address(v):
+            raise argparse.ArgumentTypeError("bad IP address %s" % v)
+        return v
+    g.add_argument('-i', '--ip',
+                   default='127.0.0.1',
+                   type=ip_addr, metavar='IP_ADDR', help=
+        'IP address of the server (%(default)s), accept either IPv4 or '
+        'IPv6 (e.g. ::1) addresses. NOTE: this option is ignored if '
+        '--interface is specified')
+    def not_negative_int(v, err_label='value'):
+        try:
+            iv = int(v)
+            if iv < 0: raise ValueError()
+            return iv
+        except ValueError:
+            pass
+        raise argparse.ArgumentTypeError("bad %s %s" % (err_label, v))
+    def port(v):
+        return not_negative_int(v, err_label='port')
+    g.add_argument('-p', '--port',
+                   default=8000,
+                   type=port, metavar='NUM', help=
+        'port of server (%(default)d). '
+        'NOTE: this option is ignored if --interface is specified')
+    g.add_argument('-k', '--server_key', '--server-key',
+                   '--ssl_private_key', # deprecated
+                   type=existing_file,
+                   metavar='FILE', help='server private key')
+    g.add_argument('-c', '--server_cert', '--server-cert',
+                   '--ssl_certificate', # deprecated
+                   type=existing_file,
+                   metavar='FILE', help='server certificate')
+    g.add_argument('--ca_cert', '--ca-cert',
+                   type=existing_file,
+                   metavar='FILE', help='CA certificate')
+    def iface(v, sep=','):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if sep == ':':
+            # deprecated --interfaces ip:port:key:cert:ca_cert
+            # IPv6 addresses in square brackets
+            if v.startswith('['):
+                # IPv6
+                ip, v_remainder = v.split(']', 1)
+                ip = ip[1:]
+                ifp = v_remainder[1:].split(':')
+                ifp.insert(0, ip)
+            else:
+                # IPv4
+                ifp = v.split(':')
+        else:
+            # --interface
+            ifp = v.split(sep, 5)
+        if not len(ifp) in (2, 4, 5):
+            raise argparse.ArgumentTypeError("bad interface %r" % v)
+        try:
+            ip_addr(ifp[0])
+            ifp[1] = port(ifp[1])
+            for fv in ifp[2:]:
+                existing_file(fv)
+        except argparse.ArgumentTypeError as ex:
+            raise argparse.ArgumentTypeError("bad interface %r (%s)" % (v, ex))
+        return tuple(ifp)
+    g.add_argument('--interface', dest='interfaces',
+                   default=[], action=ExtendAction,
+                   type=iface, nargs='+',
+                   metavar='IF_INFO', help=
+        'listen on specified interface, IF_INFO = '
+        'IP_ADDR,PORT[,KEY_FILE,CERT_FILE[,CA_CERT_FILE]].'
+        ' NOTE: this option can be used multiple times to provide additional '
+        'interfaces to choose from but you can choose which one to listen to '
+        'only using the GUI otherwise the first interface specified is used')
+    def ifaces(v):
+        # deprecated --interfaces 'if1;if2;...'
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        return [iface(i, ':') for i in v.split(';')]
+    g.add_argument('--interfaces', # deprecated
+                   default=argparse.SUPPRESS, # do not set if absent
+                   action=ExtendAction,
+                   type=ifaces,
+                   help=argparse.SUPPRESS) # do not show on help
+    g.add_argument('-d', '--pid_filename', '--pid-filename',
+                   default='httpserver.pid',
+                   metavar='FILE', help='server pid file (%(default)s)')
+    g.add_argument('-l', '--log_filename', '--log-filename',
+                   default='httpserver.log',
+                   metavar='FILE', help='server log file (%(default)s)')
+    g.add_argument('--min_threads', '--min-threads',
+                   '--minthreads', '-n', '--numthreads', # deprecated
+                   type=not_negative_int, metavar='NUM',
+                   help='minimum number of server threads')
+    g.add_argument('--max_threads', '--max-threads',
+                   '--maxthreads', # deprecated
+                   type=not_negative_int, metavar='NUM',
+                   help='maximum number of server threads')
+    g.add_argument('-q', '--request_queue_size', '--request-queue-size',
+                   default=5,
+                   type=not_negative_int, metavar='NUM', help=
+        'max number of queued requests when server busy (%(default)d)')
+    g.add_argument('-o', '--timeout',
+                   default=10,
+                   type=not_negative_int, metavar='SECONDS',
+                   help='timeout for individual request (%(default)d seconds)')
+    g.add_argument('--socket_timeout', '--socket-timeout',
+                   default=5,
+                   type=not_negative_int, metavar='SECONDS',
+                   help='timeout for socket (%(default)d seconds)')
+    g.add_argument('-z', '--shutdown_timeout', # deprecated
+                   type=not_negative_int,
+                   help=argparse.SUPPRESS) # do not show on help
+    g.add_argument('-F', '--profiler_dir', '--profiler-dir',
+                   '--profiler', # deprecated
+                   help='profiler directory')
+
+    # scheduler options
+    g = parser.add_argument_group('scheduler options')
+    g.add_argument('-X', '--with_scheduler', '--with-scheduler',
+                   default=False,
+                   action='store_true', help=
+        'run schedulers alongside web server; require --K')
+    def is_app(app):
+        return is_appdir(folder, app)
+    def scheduler(v):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if ',' in v:
+            # legacy "app1,..."
+            vl = [n.strip() for n in v.split(',')]
+            return [scheduler(iv) for iv in vl]
+        vp = [n.strip() for n in v.split(':')]
+        app = vp[0]
+        if not app:
+            raise argparse.ArgumentTypeError('empty application')
+        if not is_app(app):
+            warn("argument -K/--scheduler: bad application %r, skipped" % app)
+            return None
+        return ':'.join(filter(None, vp))
+    g.add_argument('-K', '--scheduler', dest='schedulers',
+                   default=[], action=ExtendAction,
+                   type=scheduler, nargs='+',
+                   metavar='APP_INFO', help=
+        'run scheduler for the specified application(s), APP_INFO = '
+        'APP_NAME[:GROUPS], that is an optional list of groups can follow '
+        'the application name (e.g. app:group1:group2); require a scheduler '
+        "to be defined in the application's models. NOTE: this option can "
+        'be used multiple times to add schedulers')
+
+    # cron options
+    g = parser.add_argument_group('cron options')
+    g.add_argument('-Y', '--with_cron', '--with-cron',
+                   '--run-cron', # deprecated
+                   default=False,
+                   action='store_true', help=
+        'run cron service alongside web server')
+    def crontab(v):
+        if not v:
+            raise argparse.ArgumentTypeError('empty argument')
+        if not is_app(v):
+            warn("argument --crontab: bad application %r, skipped" % v)
+            return None
+        return v
+    g.add_argument('--crontab', dest='crontabs',
+                   default=[], action=ExtendAction,
+                   type=crontab, nargs='+',
+                   metavar='APP_NAME', help=
+        'tell cron to read the crontab for the specified application(s) '
+        'only, the default behaviour is to read the crontab for all of the '
+        'installed applications. NOTE: this option can be used multiple '
+        'times to build the list of crontabs to be processed by cron')
+    g.add_argument('--soft_cron', '--soft-cron',
+                   '--softcron', # deprecated
+                   default=False,
+                   action='store_true', help=
+        'use cron software emulation instead of separate cron process; '
+        'require -Y. NOTE: use of cron software emulation is strongly '
+        'discouraged')
+    g.add_argument('-C', '--cron_run', '--cron-run',
+                   '--cron', # deprecated
+                   default=False,
+                   action='store_true', help=
+        'trigger a cron run and exit; usually used when invoked '
+        'from a system (external) crontab')
+    g.add_argument('--cron_job', # NOTE: this is intended for internal use only
+                   default=False,
+                   action='store_true',
+                   help=argparse.SUPPRESS) # do not show on help
+
+    # test options
+    g = parser.add_argument_group('test options')
+    g.add_argument('-v', '--verbose',
+                   default=False,
+                   action='store_true', help='increase verbosity')
+    g.add_argument('-T', '--run_doctests', '--run-doctests',
+                   '--test', # deprecated
+                   metavar='APP_ENV', help=
+        'run doctests in application environment. APP_ENV like a/c/f (c, f '
+        'optional)')
+    g.add_argument('--run_system_tests', '--run-system-tests',
+                   default=False,
+                   action='store_true', help='run web2py test suite')
+    g.add_argument('--with_coverage', '--with-coverage',
+                   default=False,
+                   action='store_true', help=
+        'collect coverage data when used with --run_system_tests; '
+        'require Python 2.7+ and the coverage module installed')
+
+    # other options
+    g = parser.add_argument_group('other options')
+    g.add_argument('-G', '--GAE', dest='gae',
+                   metavar='APP_NAME', help=
+        'will create app.yaml and gaehandler.py and exit')
+
+    options = parse_args(parser, sys.argv[1:],
+                         deprecated_opts, integer_log_level)
+
+    # make a copy of all options for global_settings
+    copy_options = copy.deepcopy(options)
+    copy_options.password = '******'
+    global_settings.cmd_options = copy_options
+
+    return options
+
+
+REGEX_PEP263 = r'^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)'
+
+def get_pep263_encoding(source):
+    """
+    Read python source file encoding, according to PEP 263, see
+    https://www.python.org/dev/peps/pep-0263/
+    """
+    with open(source, 'r') as sf:
+        l12 = (sf.readline(), sf.readline())
+    m12 = re.match(REGEX_PEP263, l12[0]) or re.match(REGEX_PEP263, l12[1])
+    return m12 and m12.group(1)
+
+
+IGNORE = lambda: None
+
+def load_config(config_file, opt_map):
+    """
+    Load options from config file (a Python script).
+
+    config_file(str): file name
+    opt_map(dict): mapping fom option name (key) to callable (val),
+        used to post-process parsed value for the option
+
+    Notice that the configuring Python script is never executed/imported,
+    instead the ast library is used to evaluate each option assignment,
+    provided that it is written on a single line.
+
+    Returns an OrderedDict with sourced options.
+    """
+    REGEX_ASSIGN_EXP = re.compile(r'\s*=\s*(.+)')
+    map_items = opt_map.items()
+    # preserve the order of loaded options even though this is not needed
+    pl = OrderedDict()
+    config_encoding = get_pep263_encoding(config_file)
+    # NOTE: assume 'ascii' encoding when not explicitly stated (Python 2),
+    #       this is not correct for Python 3 where the default is 'utf-8'
+    open_kwargs = dict() if PY2 else dict(encoding=config_encoding or 'ascii')
+    with open(config_file, 'r', **open_kwargs) as cfil:
+        for linenum, clin in enumerate(cfil, start=1):
+            if PY2 and config_encoding:
+                clin = unicode(clin, config_encoding)
+            clin = clin.strip()
+            for opt, mapr in map_items:
+                if clin.startswith(opt):
+                    m = REGEX_ASSIGN_EXP.match(clin[len(opt):])
+                    if m is None: continue
+                    try:
+                        val = opt_map[opt](ast.literal_eval(m.group(1)))
+                    except:
+                        die("cannot parse config file %r at line %d" % (config_file, linenum))
+                    if val is not IGNORE:
+                        pl[opt] = val
+    return pl
+
+
+def parse_args(parser, cli_args, deprecated_opts, integer_log_level,
+               namespace=None):
+
+    #print('PARSING ARGS:', cli_args)
+    del integer_log_level[:]
+    options = parser.parse_args(cli_args, namespace)
+    #print('PARSED OPTIONS:', options)
+
+    # warn for deprecated options
+    deprecated_args = [a for a in cli_args if a in deprecated_opts]
+    for da in deprecated_args:
+        # verify if it was a real option by looking into
+        # parsed values for the actual destination
+        hint = deprecated_opts[da]
+        dest = (hint or da).lstrip('-')
+        default = parser.get_default(dest)
+        if da == '--interfaces':
+            hint = '--interface'
+        if getattr(options, dest) is not default:
+            # the option has been specified
+            msg = "%s is deprecated" % da
+            if hint:
+                msg += ", use %s instead" % hint
+            warn(msg)
+    # warn for deprecated values
+    if integer_log_level and '--debug' not in deprecated_args:
+        warn('integer argument for -D/--log_level is deprecated, '
+             'use label instead')
+    # fix schedulers and die if all were skipped
+    if None in options.schedulers:
+        options.schedulers = [i for i in options.schedulers if i is not None]
+        if not options.schedulers:
+            die('no scheduler left')
+    # fix crontabs and die if all were skipped
+    if None in options.crontabs:
+        options.crontabs = [i for i in options.crontabs if i is not None]
+        if not options.crontabs:
+            die('no crontab left')
+    # taskbar
+    if options.taskbar and os.name != 'nt':
+        warn('--taskbar not supported on this platform, skipped')
+        options.taskbar = False
+    # options consistency checkings
+    if options.run and not options.shell:
+        die('-R/--run requires -S/--shell', exit_status=2)
+    if options.args and not options.run:
+        die('-A/--args requires -R/--run', exit_status=2)
+    if options.with_scheduler and not options.schedulers:
+        die('-X/--with_scheduler requires -K/--scheduler', exit_status=2)
+    if options.soft_cron and not options.with_cron:
+        die('--soft_cron requires -Y/--with_cron', exit_status=2)
+    if options.shell:
+        for o, os in dict(with_scheduler='-X/--with_scheduler',
+                          schedulers='-K/--scheduler',
+                          with_cron='-Y/--with_cron',
+                          cron_run='-C/--cron_run',
+                          run_doctests='-T/--run_doctests',
+                          run_system_tests='--run_system_tests').items():
+            if getattr(options, o):
+                die("-S/--shell and %s are conflicting options" % os,
+                    exit_status=2)
+    if options.bpython and options.plain:
+        die('-B/--bpython and -P/--plain are conflicting options',
+            exit_status=2)
+    if options.cron_run:
+        for o, os in dict(with_cron='-Y/--with_cron',
+                          run_doctests='-T/--run_doctests',
+                          run_system_tests='--run_system_tests').items():
+            if getattr(options, o):
+                die("-C/--cron_run and %s are conflicting options" % os,
+                    exit_status=2)
+    if options.run_doctests and options.run_system_tests:
+        die('-T/--run_doctests and --run_system_tests are conflicting options',
+            exit_status=2)
+
+    if options.config:
+        # load options from file,
+        # all options sourced from file that evaluates to False
+        # are skipped, the special IGNORE value is used for this
+        store_true = lambda v: True if v else IGNORE
+        str_or_default = lambda v : str(v) if v else IGNORE
+        list_or_default = lambda v : (
+            [str(i) for i in v] if isinstance(v, list) else [str(v)]) if v \
+            else IGNORE
+        # NOTE: 'help', 'version', 'folder', 'cron_job' and 'GAE' are not
+        #       sourced from file, the same applies to deprecated options
+        opt_map = {
+            # global options
+            'config': str_or_default,
+            'add_options': store_true,
+            'password': str_or_default,
+            'errors_to_console': store_true,
+            'no_banner': store_true,
+            'quiet': store_true,
+            'log_level': str_or_default,
+            # GUI options
+            'no_gui': store_true,
+            'taskbar': store_true,
+            # console options
+            'shell': str_or_default,
+            'bpython': store_true,
+            'plain': store_true,
+            'import_models': store_true,
+            'run': str_or_default,
+            'args': list_or_default,
+            # web server options
+            'server_name': str_or_default,
+            'ip': str_or_default,
+            'port': str_or_default,
+            'server_key': str_or_default,
+            'server_cert': str_or_default,
+            'ca_cert': str_or_default,
+            'interface': list_or_default,
+            'pid_filename': str_or_default,
+            'log_filename': str_or_default,
+            'min_threads': str_or_default,
+            'max_threads': str_or_default,
+            'request_queue_size': str_or_default,
+            'timeout': str_or_default,
+            'socket_timeout': str_or_default,
+            'profiler_dir': str_or_default,
+            # scheduler options
+            'with_scheduler': store_true,
+            'scheduler': list_or_default,
+            # cron options
+            'with_cron': store_true,
+            'crontab': list_or_default,
+            'soft_cron': store_true,
+            'cron_run': store_true,
+            # test options
+            'verbose': store_true,
+            'run_doctests': str_or_default,
+            'run_system_tests': store_true,
+            'with_coverage': store_true,
+        }
+        od = load_config(options.config, opt_map)
+        #print("LOADED FROM %s:" % options.config, od)
+        # convert loaded options dict as retuned by load_config
+        # into a list of arguments for further parsing by parse_args
+        file_args = []; args_args = [] # '--args' must be the last
+        for key, val in od.items():
+            if key != 'args':
+                file_args.append('--' + key)
+                if isinstance(val, list): file_args.extend(val)
+                elif not isinstance(val, bool): file_args.append(val)
+            else:
+                args_args = ['--args'] + val
+        file_args += args_args
+
+        if options.add_options:
+            # add options to existing ones,
+            # must clear config to avoid infinite recursion
+            options.config = options.add_options = None
+            return parse_args(parser, file_args,
+                deprecated_opts, integer_log_level, options)
+        return parse_args(parser, file_args,
+            deprecated_opts, integer_log_level)
+
+    return options

gluon/fileutils.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -19,10 +18,12 @@ import time
 import datetime
 import logging
 import shutil
+
 from gluon.http import HTTP
 from gzip import open as gzopen
 from gluon.recfile import generate
 from gluon._compat import PY2
+from gluon.settings import global_settings
 
 __all__ = (
     'parse_version',
@@ -255,12 +256,23 @@ def w2p_pack(filename, path, compiled=False, filenames=None):
     os.unlink(tarname)
 
 
+def missing_app_folders(path):
+    for subfolder in ('models', 'views', 'controllers', 'databases',
+                      'modules', 'cron', 'errors', 'sessions',
+                      'languages', 'static', 'private', 'uploads'):
+        yield os.path.join(path, subfolder)
+
+
 def create_welcome_w2p():
     is_newinstall = os.path.exists('NEWINSTALL')
     if not os.path.exists('welcome.w2p') or is_newinstall:
         logger = logging.getLogger("web2py")
         try:
-            w2p_pack('welcome.w2p', 'applications/welcome')
+            app_path = 'applications/welcome'
+            for amf in missing_app_folders(app_path):
+                if not os.path.exists(amf):
+                    os.mkdir(amf)
+            w2p_pack('welcome.w2p', app_path)
             logger.info("New installation: created welcome.w2p file")
         except:
             logger.exception("New installation error: unable to create welcome.w2p file")
@@ -295,12 +307,6 @@ def w2p_unpack(filename, path, delete_tar=True):
 
 def create_app(path):
     w2p_unpack('welcome.w2p', path)
-    for subfolder in ('models', 'views', 'controllers', 'databases',
-                      'modules', 'cron', 'errors', 'sessions',
-                      'languages', 'static', 'private', 'uploads'):
-        subpath = os.path.join(path, subfolder)
-        if not os.path.exists(subpath):
-            os.mkdir(subpath)
 
 
 def w2p_pack_plugin(filename, path, plugin_name):
@@ -374,7 +380,7 @@ def get_session(request, other_application='admin'):
         if not os.path.exists(session_filename):
             session_filename = generate(session_filename)
         osession = storage.load_storage(session_filename)
-    except:
+    except Exception:
         osession = storage.Storage()
     return osession
 
@@ -424,45 +430,14 @@ def fix_newlines(path):
             write_file(filename, wdata, 'w')
 
 
-def copystream(
-    src,
-    dest,
-    size,
-    chunk_size=10 ** 5,
-):
-    """
-    this is here because I think there is a bug in shutil.copyfileobj
-    """
-    while size > 0:
-        if size < chunk_size:
-            data = src.read(size)
-        else:
-            data = src.read(chunk_size)
-        length = len(data)
-        if length > size:
-            (data, length) = (data[:size], size)
-        size -= length
-        if length == 0:
-            break
-        dest.write(data)
-        if length < chunk_size:
-            break
-    dest.seek(0)
-    return
-
-
-from gluon.settings import global_settings  # we need to import settings here because
-                                      # settings imports fileutils too
-
-
-def abspath(*relpath, **base):
+# NOTE: same name as os.path.abspath (but signature is different)
+def abspath(*relpath, **kwargs):
     """Converts relative path to absolute path based (by default) on
     applications_parent
     """
     path = os.path.join(*relpath)
-    gluon = base.get('gluon', False)
     if os.path.isabs(path):
         return path
-    if gluon:
+    if kwargs.get('gluon', False):
         return os.path.join(global_settings.gluon_parent, path)
     return os.path.join(global_settings.applications_parent, path)

gluon/globals.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -27,7 +26,6 @@ from gluon.utils import web2py_uuid, secure_dumps, secure_loads
 from gluon.settings import global_settings
 from gluon import recfile
 from gluon.cache import CacheInRam
-from gluon.fileutils import copystream
 import hashlib
 from pydal.contrib import portalocker
 from pickle import Pickler, MARK, DICT, EMPTY_DICT
@@ -103,6 +101,26 @@ def sorting_dumps(obj, protocol=None):
 # END #####################################################################
 
 
+def copystream(src, dest, size, chunk_size, cache_inc=None):
+    while size > 0:
+        if size < chunk_size:
+            data = src.read(size)
+            callable(cache_inc) and cache_inc(size)
+        else:
+            data = src.read(chunk_size)
+            callable(cache_inc) and cache_inc(chunk_size)
+        length = len(data)
+        if length > size:
+            (data, length) = (data[:size], size)
+        size -= length
+        if length == 0:
+            break
+        dest.write(data)
+        if length < chunk_size:
+            break
+    dest.seek(0)
+    return
+
 def copystream_progress(request, chunk_size=10 ** 5):
     """
     Copies request.env.wsgi_input into request.body
@@ -128,23 +146,8 @@ def copystream_progress(request, chunk_size=10 ** 5):
     cache_ram = CacheInRam(request)  # same as cache.ram because meta_storage
     cache_ram(cache_key + ':length', lambda: size, 0)
     cache_ram(cache_key + ':uploaded', lambda: 0, 0)
-    while size > 0:
-        if size < chunk_size:
-            data = source.read(size)
-            cache_ram.increment(cache_key + ':uploaded', size)
-        else:
-            data = source.read(chunk_size)
-            cache_ram.increment(cache_key + ':uploaded', chunk_size)
-        length = len(data)
-        if length > size:
-            (data, length) = (data[:size], size)
-        size -= length
-        if length == 0:
-            break
-        dest.write(data)
-        if length < chunk_size:
-            break
-    dest.seek(0)
+    copystream(source, dest, size, chunk_size,
+               lambda v : cache_ram.increment(cache_key + ':uploaded', v))
     cache_ram(cache_key + ':length', None)
     cache_ram(cache_key + ':uploaded', None)
     return dest
@@ -355,11 +358,9 @@ class Request(Storage):
         """
         cmd_opts = global_settings.cmd_options
         # checking if this is called within the scheduler or within the shell
-        # in addition to checking if it's a cronjob
-        # FIXME: cmd_opts.scheduler does not imply that
-        #        we are running in the scheduler
-        if (self.is_https or cmd_opts and (
-                cmd_opts.shell or cmd_opts.scheduler or cmd_opts.cronjob)):
+        # in addition to checking if it's a cron job
+        if (self.is_https or self.is_scheduler or cmd_opts and (
+                cmd_opts.shell or cmd_opts.cron_job)):
             current.session.secure()
         else:
             current.session.forget()

gluon/main.py

@@ -1,4 +1,3 @@
-#!/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -117,8 +116,8 @@ load_routes()
 HTTPS_SCHEMES = set(('https', 'HTTPS'))
 
 
-# pattern used to validate client address
-REGEX_CLIENT = re.compile(r'[\w:-]+(\.[\w-]+)*\.?')  # ## to account for IPV6
+# pattern used to match client IP address
+REGEX_CLIENT = re.compile(r'[\w:]+(\.\w+)*')
 
 def get_client(env):
     """
@@ -557,7 +556,8 @@ def wsgibase(environ, responder):
         return wsgibase(new_environ, responder)
     if global_settings.web2py_crontype == 'soft':
         cmd_opts = global_settings.cmd_options
-        newcron.softcron(global_settings.applications_parent).start()
+        newcron.softcron(global_settings.applications_parent,
+                         apps=cmd_opts and cmd_opts.crontabs).start()
     return http_response.to(responder, env=env)
 
 

gluon/myregex.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-| This file is part of the web2py Web Framework
-| Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
-| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
-
-Useful regexes
----------------
-"""
-
-import re
-
-# pattern to find defined tables
-
-regex_tables = re.compile(
-    """^[\w]+\.define_table\(\s*[\'\"](?P<name>\w+)[\'\"]""",
-    flags=re.M)
-
-# pattern to find exposed functions in controller
-
-regex_expose = re.compile(
-    '^def\s+(?P<name>_?[a-zA-Z0-9]\w*)\( *\)\s*:',
-    flags=re.M)
-
-regex_longcomments = re.compile('(""".*?"""|'+"'''.*?''')", re.DOTALL)
-
-regex_include = re.compile(
-    '(?P<all>\{\{\s*include\s+[\'"](?P<name>[^\'"]*)[\'"]\s*\}\})')
-
-regex_extend = re.compile(
-    '^\s*(?P<all>\{\{\s*extend\s+[\'"](?P<name>[^\'"]+)[\'"]\s*\}\})', re.MULTILINE)

gluon/newcron.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
 
 """
 | This file is part of the web2py Web Framework
@@ -9,13 +10,15 @@
 Cron-style interface
 """
 
-import sys
-import os
 import threading
-import logging
+import os
+from logging import getLogger, DEBUG
 import time
 import sched
+import sys
 import re
+import shlex
+
 import datetime
 from functools import reduce
 from gluon.settings import global_settings
@@ -23,10 +26,18 @@ from gluon import fileutils
 from gluon._compat import to_bytes, pickle
 from pydal.contrib import portalocker
 
-logger = logging.getLogger("web2py.cron")
+logger_name = 'web2py.cron'
+
+
 _cron_stopping = False
+
+_cron_subprocs_lock = threading.RLock()
 _cron_subprocs = []
 
+def subprocess_count():
+    with _cron_subprocs_lock:
+        return len(_cron_subprocs)
+
 
 def absolute_path_link(path):
     """
@@ -45,14 +56,14 @@ def stopcron():
     """Graceful shutdown of cron"""
     global _cron_stopping
     _cron_stopping = True
-    while _cron_subprocs:
-        proc = _cron_subprocs.pop()
+    while subprocess_count():
+        with _cron_subprocs_lock:
+            proc = _cron_subprocs.pop()
         if proc.poll() is None:
             try:
                 proc.terminate()
-            except:
-                import traceback
-                traceback.print_exc()
+            except Exception:
+                getLogger(logger_name).exception('error in stopcron')
 
 
 class extcron(threading.Thread):
@@ -62,11 +73,10 @@ class extcron(threading.Thread):
         self.setDaemon(False)
         self.path = applications_parent
         self.apps = apps
-        # crondance(self.path, 'external', startup=True, apps=self.apps)
 
     def run(self):
         if not _cron_stopping:
-            logger.debug('external cron invocation')
+            getLogger(logger_name).debug('external cron invocation')
             crondance(self.path, 'external', startup=False, apps=self.apps)
 
 
@@ -77,16 +87,19 @@ class hardcron(threading.Thread):
         self.setDaemon(True)
         self.path = applications_parent
         self.apps = apps
+        # processing of '@reboot' entries in crontab (startup=True)
+        getLogger(logger_name).info('hard cron bootstrap')
         crondance(self.path, 'hard', startup=True, apps=self.apps)
 
     def launch(self):
         if not _cron_stopping:
-            logger.debug('hard cron invocation')
+            self.logger.debug('hard cron invocation')
             crondance(self.path, 'hard', startup=False, apps=self.apps)
 
     def run(self):
+        self.logger = getLogger(logger_name)
+        self.logger.info('hard cron daemon started')
         s = sched.scheduler(time.time, time.sleep)
-        logger.info('hard cron daemon started')
         while not _cron_stopping:
             now = time.time()
             s.enter(60 - now % 60, 1, self.launch, ())
@@ -99,14 +112,14 @@ class softcron(threading.Thread):
         threading.Thread.__init__(self)
         self.path = applications_parent
         self.apps = apps
-        # crondance(self.path, 'soft', startup=True, apps=self.apps)
 
     def run(self):
         if not _cron_stopping:
-            logger.debug('soft cron invocation')
+            getLogger(logger_name).debug('soft cron invocation')
             crondance(self.path, 'soft', startup=False, apps=self.apps)
 
 
+# TODO: context manager
 class Token(object):
 
     def __init__(self, path):
@@ -115,6 +128,7 @@ class Token(object):
             fileutils.write_file(self.path, to_bytes(''), 'wb')
         self.master = None
         self.now = time.time()
+        self.logger = getLogger(logger_name)
 
     def acquire(self, startup=False):
         """
@@ -133,7 +147,7 @@ class Token(object):
         else:
             locktime = 59.99
         if portalocker.LOCK_EX is None:
-            logger.warning('cron disabled because no file locking')
+            self.logger.warning('cron disabled because no file locking')
             return None
         self.master = fileutils.open_file(self.path, 'rb+')
         try:
@@ -148,8 +162,8 @@ class Token(object):
                 ret = self.now
                 if not stop:
                     # this happens if previous cron job longer than 1 minute
-                    logger.warning('stale cron.master detected')
-                logger.debug('acquiring lock')
+                    self.logger.warning('stale cron.master detected')
+                self.logger.debug('acquiring lock')
                 self.master.seek(0)
                 pickle.dump((self.now, 0), self.master)
                 self.master.flush()
@@ -167,7 +181,7 @@ class Token(object):
         ret = self.master.closed
         if not self.master.closed:
             portalocker.lock(self.master, portalocker.LOCK_EX)
-            logger.debug('releasing cron lock')
+            self.logger.debug('releasing cron lock')
             self.master.seek(0)
             (start, stop) = pickle.load(self.master)
             if start == self.now:  # if this is my lock
@@ -248,26 +262,25 @@ class cronlauncher(threading.Thread):
 
     def run(self):
         import subprocess
-        global _cron_subprocs
-        if isinstance(self.cmd, (list, tuple)):
-            cmd = self.cmd
-        else:
-            cmd = self.cmd.split()
-        proc = subprocess.Popen(cmd,
+        logger = getLogger(logger_name)
+        proc = subprocess.Popen(self.cmd,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
-        _cron_subprocs.append(proc)
+        with _cron_subprocs_lock:
+            _cron_subprocs.append(proc)
         (stdoutdata, stderrdata) = proc.communicate()
         try:
-            _cron_subprocs.remove(proc)
+            with _cron_subprocs_lock:
+                _cron_subprocs.remove(proc)
         except ValueError:
             pass
         if proc.returncode != 0:
-            logger.warning('call returned code %s:\n%s\n%s',
-                proc.returncode, stdoutdata, stderrdata)
-        else:
-            logger.debug('call returned success:\n%s', stdoutdata)
+            logger.warning('%r call returned code %s:\n%s\n%s',
+                ' '.join(self.cmd), proc.returncode, stdoutdata, stderrdata)
+        elif logger.isEnabledFor(DEBUG):
+            logger.debug('%r call returned success:\n%s',
+                ' '.join(self.cmd), stdoutdata)
 
 
 def crondance(applications_parent, ctype='soft', startup=False, apps=None):
@@ -287,7 +300,9 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
               ('dom', now_s.tm_mday),
               ('dow', (now_s.tm_wday + 1) % 7))
 
-    if apps is None:
+    logger = getLogger(logger_name)
+
+    if not apps:
         apps = [x for x in os.listdir(apppath)
                 if os.path.isdir(os.path.join(apppath, x))]
 
@@ -303,10 +318,7 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
         base_commands.append(w2p_path)
     if applications_parent != global_settings.gluon_parent:
         base_commands.extend(('-f', applications_parent))
-    base_commands.extend(('--cronjob', '--no-banner', '--nogui', '--plain',
-                          # FIXME: this should not be needed since we are
-                          #        not launching the web server
-                          '-a', '"<recycle>"'))
+    base_commands.extend(('--cron_job', '--no_banner', '--no_gui', '--plain'))
 
     for app in apps:
         if _cron_stopping:
@@ -335,15 +347,16 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
         for task in tasks:
             if _cron_stopping:
                 break
-            citems = [(k in task and not v in task[k]) for k, v in checks]
-            task_min = task.get('min', [])
             if not task:
                 continue
-            elif not startup and task_min == [-1]:
+            task_min = task.get('min', [])
+            if not startup and task_min == [-1]:
                 continue
-            elif task_min != [-1] and reduce(lambda a, b: a or b, citems):
+            citems = [(k in task and not v in task[k]) for k, v in checks]
+            if task_min != [-1] and reduce(lambda a, b: a or b, citems):
                 continue
-            logger.info('%s cron: %s executing %s in %s at %s',
+
+            logger.info('%s cron: %s executing %r in %s at %s',
                 ctype, app, task.get('cmd'),
                 os.getcwd(), datetime.datetime.now())
             action = models = False
@@ -364,11 +377,12 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
                 if models:
                     commands.append('-M')
             else:
-                commands = command
+                commands = shlex.split(command)
 
             try:
+                # FIXME: using a new thread every time there is a task to
+                #        launch is not a good idea in a long running process
                 cronlauncher(commands).start()
-            except Exception as e:
-                logger.warning('execution error for %s: %s',
-                    task.get('cmd'), e)
+            except Exception:
+                logger.exception('error starting %r', task['cmd'])
     token.release()

gluon/serializers.py

@@ -119,13 +119,43 @@ def xml(value, encoding='UTF-8', key='document', quote=True):
     return ('<?xml version="1.0" encoding="%s"?>' % encoding) + str(xml_rec(value, key, quote))
 
 
-def json(value, default=custom_json, indent=None, sort_keys=False):
-    value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent)
-    # replace JavaScript incompatible spacing
-    # http://timelessrepo.com/json-isnt-a-javascript-subset
-    # PY3 FIXME
-    # return value.replace(ur'\u2028', '\\u2028').replace(ur'\2029', '\\u2029')
-    return value
+class JSONEncoderForHTML(json_parser.JSONEncoder):
+    """An encoder that produces JSON safe to embed in HTML.
+    To embed JSON content in, say, a script tag on a web page, the
+    characters &, < and > should be escaped. They cannot be escaped
+    with the usual entities (e.g. &amp;) because they are not expanded
+    within <script> tags.
+    This class also escapes the line separator and paragraph separator
+    characters U+2028 and U+2029, irrespective of the ensure_ascii setting,
+    as these characters are not valid in JavaScript strings (see
+    http://timelessrepo.com/json-isnt-a-javascript-subset).
+    """
+
+    def encode(self, o):
+        # Override JSONEncoder.encode because it has hacks for
+        # performance that make things more complicated.
+        chunks = self.iterencode(o, True)
+        if self.ensure_ascii:
+            return ''.join(chunks)
+        else:
+            return u''.join(chunks)
+
+    def iterencode(self, o, _one_shot=False):
+        chunks = super(JSONEncoderForHTML, self).iterencode(o, _one_shot)
+        for chunk in chunks:
+            chunk = chunk.replace('&', '\\u0026')
+            chunk = chunk.replace('<', '\\u003c')
+            chunk = chunk.replace('>', '\\u003e')
+
+            if not self.ensure_ascii:
+                chunk = chunk.replace(u'\u2028', '\\u2028')
+                chunk = chunk.replace(u'\u2029', '\\u2029')
+
+            yield chunk
+
+
+def json(value, default=custom_json, indent=None, sort_keys=False, cls=JSONEncoderForHTML):
+    return json_parser.dumps(value, default=default, cls=cls, sort_keys=sort_keys, indent=indent)
 
 def csv(value):
     return ''

gluon/shell.py

@@ -59,6 +59,8 @@ def enable_autocomplete_and_history(adir, env):
         readline.set_completer(rlcompleter.Completer(env).complete)
 
 
+REGEX_APP_PATH = '(?:.*/)?applications/(?P<a>[^/]+)'
+
 def exec_environment(
     pyfile='',
     request=None,
@@ -83,10 +85,10 @@ def exec_environment(
         session = Session()
 
     if request.folder is None:
-        mo = re.match(r'(|.*/)applications/(?P<appname>[^/]+)', pyfile)
+        mo = re.match(REGEX_APP_PATH, pyfile)
         if mo:
-            appname = mo.group('appname')
-            request.folder = os.path.abspath(os.path.join('applications', appname))
+            a = mo.group('a')
+            request.folder = os.path.abspath(os.path.join('applications', a))
         else:
             request.folder = ''
     env = build_environment(request, response, session, store_current=False)
@@ -146,7 +148,7 @@ def env(
         request.is_shell = cmd_opts.shell is not None
     else:
         ip = '127.0.0.1'; port = 8000
-        # FIXME: what about request.is_shell ?
+        request.is_shell = False
     request.is_scheduler = False
     request.env.http_host = '%s:%s' % (ip, port)
     request.env.remote_addr = '127.0.0.1'
@@ -213,7 +215,7 @@ def run(
     startfile=None,
     bpython=False,
     python_code=None,
-    cronjob=False,
+    cron_job=False,
     scheduler_job=False,
     force_migrate=False):
     """
@@ -234,7 +236,7 @@ def run(
     adir = os.path.join('applications', a)
 
     if not os.path.exists(adir):
-        if not cronjob and not scheduler_job and \
+        if not cron_job and not scheduler_job and \
             sys.stdin and not sys.stdin.name == '/dev/null':
             confirm = raw_input(
                 'application %s does not exist, create (y/n)?' % a)
@@ -273,7 +275,7 @@ def run(
         pyfile = os.path.join('applications', a, 'controllers', c + '.py')
         pycfile = os.path.join('applications', a, 'compiled',
                                  "controllers_%s_%s.pyc" % (c, f))
-        if ((cronjob and os.path.isfile(pycfile))
+        if ((cron_job and os.path.isfile(pycfile))
             or not os.path.isfile(pyfile)):
             exec(read_pyc(pycfile), _env)
         elif os.path.isfile(pyfile):
@@ -403,7 +405,7 @@ def test(testpath, import_models=True, verbose=False):
 
     import doctest
     if os.path.isfile(testpath):
-        mo = re.match(r'(|.*/)applications/(?P<a>[^/]+)', testpath)
+        mo = re.match(REGEX_APP_PATH, testpath)
         if not mo:
             die('test file is not in application directory: %s'
                 % testpath)

gluon/tests/test_compileapp.py

@@ -1,58 +1,75 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-""" Unit tests for utils.py """
+""" Unit tests for compileapp.py """
 
 import unittest
 import os
+import tempfile
 import shutil
 
+from gluon.fileutils import create_app, w2p_pack, w2p_unpack
 from gluon.compileapp import compile_application, remove_compiled_application
-from gluon.fileutils import w2p_pack, w2p_unpack
 from gluon.globals import Request
-from gluon.admin import app_compile, app_create, app_cleanup, check_new_version
-from gluon.admin import app_uninstall
+from gluon.admin import (app_compile, app_create, app_cleanup,
+    app_uninstall, check_new_version)
 from gluon.main import global_settings
 
+test_app_name = '_test_compileapp'
+test_app2_name = '_test_compileapp_admin'
+test_unpack_dir = None
 
 WEB2PY_VERSION_URL = "http://web2py.com/examples/default/version"
 
-
 class TestPack(unittest.TestCase):
     """ Tests the compileapp.py module """
 
+    @classmethod
+    def setUpClass(cls):
+        appdir = os.path.join('applications', test_app_name)
+        if not os.path.exists(appdir):
+            os.mkdir(appdir)
+            create_app(appdir)
+
+    @classmethod
+    def tearDownClass(cls):
+        appdir = os.path.join('applications', test_app_name)
+        if os.path.exists(appdir):
+            shutil.rmtree(appdir)
+        test_pack = "%s.w2p" % test_app_name
+        if os.path.exists(test_pack):
+            os.unlink(test_pack)
+        if test_unpack_dir:
+            shutil.rmtree(test_unpack_dir)
+
     def test_compile(self):
-        #apps = ['welcome', 'admin', 'examples']
-        apps = ['welcome']
-        for appname in apps:
-            appname_path = os.path.join(os.getcwd(), 'applications', appname)
-            compile_application(appname_path)
-            remove_compiled_application(appname_path)
-            test_path = os.path.join(os.getcwd(), "%s.w2p" % appname)
-            unpack_path = os.path.join(os.getcwd(), 'unpack', appname)
-            w2p_pack(test_path, appname_path, compiled=True, filenames=None)
-            w2p_pack(test_path, appname_path, compiled=False, filenames=None)
-            w2p_unpack(test_path, unpack_path)
-        return
+        cwd = os.getcwd()
+        app_path = os.path.join(cwd, 'applications', test_app_name)
+        self.assertIsNone(compile_application(app_path))
+        remove_compiled_application(app_path)
+        test_pack = os.path.join(cwd, "%s.w2p" % test_app_name)
+        w2p_pack(test_pack, app_path, compiled=True, filenames=None)
+        w2p_pack(test_pack, app_path, compiled=False, filenames=None)
+        global test_unpack_dir
+        test_unpack_dir = tempfile.mkdtemp()
+        w2p_unpack(test_pack, test_unpack_dir)
 
     def test_admin_compile(self):
-        #apps = ['welcome', 'admin', 'examples']
         request = Request(env={})
         request.application = 'a'
         request.controller = 'c'
         request.function = 'f'
         request.folder = 'applications/admin'
-        apps = ['welcome']
-        for appname in apps:
-            appname_path = os.path.join(os.getcwd(), 'applications', appname)
-            self.assertEqual(app_compile(appname_path, request), None)
-            # remove any existing test_app
-            new_app = 'test_app_%s' % (appname)
-            if(os.path.exists('applications/%s' % (new_app))):
-                shutil.rmtree('applications/%s' % (new_app))
-            self.assertEqual(app_create(new_app, request), True)
-            self.assertEqual(os.path.exists('applications/test_app_%s/controllers/default.py' % (appname)), True)
-            self.assertEqual(app_cleanup(new_app, request), True)
-            self.assertEqual(app_uninstall(new_app, request), True)
-        self.assertNotEqual(check_new_version(global_settings.web2py_version, WEB2PY_VERSION_URL), -1)
-        return
+        # remove any existing test app
+        app_path = os.path.join('applications', test_app2_name)
+        if os.path.exists(app_path):
+            shutil.rmtree(app_path)
+
+        self.assertTrue(app_create(test_app2_name, request))
+        self.assertTrue(os.path.exists('applications/%s/controllers/default.py' % test_app2_name))
+        self.assertIsNone(app_compile(test_app2_name, request))
+        self.assertTrue(app_cleanup(test_app2_name, request))
+        self.assertTrue(app_uninstall(test_app2_name, request))
+
+    def test_check_new_version(self):
+        vert = check_new_version(global_settings.web2py_version, WEB2PY_VERSION_URL)
+        self.assertNotEqual(vert[0], -1)

gluon/tests/test_cron.py

@@ -1,22 +1,67 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
+"""
+    Unit tests for cron
+"""
 
-""" Unit tests for cron """
-import unittest, os
-from gluon.newcron import Token, crondance
+import unittest
+import os
+import shutil
+import time
+
+from gluon.newcron import Token, crondance, subprocess_count
+from gluon.fileutils import create_app, write_file
+
+
+test_app_name = '_test_cron'
+appdir = os.path.join('applications', test_app_name)
+
+def setUpModule():
+    if not os.path.exists(appdir):
+        os.mkdir(appdir)
+        create_app(appdir)
+
+def tearDownModule():
+    if os.path.exists(appdir):
+        shutil.rmtree(appdir)
+
+
+TEST_CRONTAB = """@reboot peppe **applications/%s/cron/test.py
+""" %  test_app_name
+TEST_SCRIPT = """
+from os.path import join as pjoin
+
+with open(pjoin(request.folder, 'private', 'cron_req'), 'w') as f:
+    f.write(str(request))
+"""
+TARGET = os.path.join(appdir, 'private', 'cron_req')
 
 class TestCron(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        for master in (os.path.join(appdir, 'cron.master'), 'cron.master'):
+            if os.path.exists(master):
+                os.unlink(master)
 
     def test_Token(self):
-        appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
-        t = Token(path=appname_path)
-        self.assertNotEqual(t.acquire(), None)
+        app_path = os.path.join(os.getcwd(), 'applications', test_app_name)
+        t = Token(path=app_path)
+        self.assertEqual(t.acquire(), t.now)
         self.assertFalse(t.release())
-        self.assertEqual(t.acquire(), None)
+        self.assertIsNone(t.acquire())
         self.assertTrue(t.release())
-        return
 
     def test_crondance(self):
-        #TODO update crondance to return something
-        crondance(os.getcwd())
+        base = os.path.join(appdir, 'cron')
+        write_file(os.path.join(base, 'crontab'), TEST_CRONTAB)
+        write_file(os.path.join(base, 'test.py'), TEST_SCRIPT)
+        if os.path.exists(TARGET):
+            os.unlink(TARGET)
+        crondance(os.getcwd(), 'hard', startup=True, apps=[test_app_name])
+        # must wait for the cron task, not very reliable
+        time.sleep(1)
+        while subprocess_count():
+            time.sleep(1)
+        time.sleep(1)
+        self.assertTrue(os.path.exists(TARGET))

gluon/tests/test_router.py

@@ -1,86 +1,76 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """Unit tests for rewrite.py routers option"""
 
 import os
-import unittest
 import tempfile
 import logging
+import unittest
+import shutil
 
-from gluon.rewrite import load, filter_url, filter_err, get_effective_router, map_url_out
+from gluon.rewrite import (load, filter_url, filter_err,
+    get_effective_router, map_url_out)
 from gluon.html import URL
-from gluon.fileutils import abspath
 from gluon.settings import global_settings
 from gluon.http import HTTP
 from gluon.storage import Storage
 from gluon._compat import to_bytes, PY2
 
-logger = None
-oldcwd = None
-root = None
 
-def norm_root(root):
-    return root.replace('/', os.sep)
+logger = None
+
+old_root = root = None
 
 def setUpModule():
-    def make_apptree():
+    def make_apptree(root):
         "build a temporary applications tree"
-        #  applications/
-        os.mkdir(abspath('applications'))
-
-        #  applications/app/
+        # applications/
+        os.mkdir(os.path.join(root, 'applications'))
+        # applications/app/
         for app in ('admin', 'examples', 'welcome'):
-            os.mkdir(abspath('applications', app))
+            os.mkdir(os.path.join(root, 'applications', app))
             #  applications/app/(controllers, static)
             for subdir in ('controllers', 'static'):
-                os.mkdir(abspath('applications', app, subdir))
-
-        #  applications/admin/controllers/*.py
+                os.mkdir(os.path.join(root, 'applications', app, subdir))
+        # applications/admin/controllers/*.py
+        base = os.path.join(root, 'applications', 'admin', 'controllers')
         for ctr in ('appadmin', 'default', 'gae', 'mercurial', 'shell', 'wizard'):
-            open(abspath('applications', 'admin',
-                 'controllers', '%s.py' % ctr), 'w').close()
-        #  applications/examples/controllers/*.py
+            open(os.path.join(base, '%s.py' % ctr), 'w').close()
+        # applications/examples/controllers/*.py
+        base = os.path.join(root, 'applications', 'examples', 'controllers')
         for ctr in ('ajax_examples', 'appadmin', 'default', 'global', 'spreadsheet'):
-            open(abspath('applications', 'examples',
-                 'controllers', '%s.py' % ctr), 'w').close()
-        #  applications/welcome/controllers/*.py
-        #  (include controller that collides with another app)
+            open(os.path.join(base, '%s.py' % ctr), 'w').close()
+        # applications/welcome/controllers/*.py
+        # (include controller that collides with another app)
+        base = os.path.join(root, 'applications', 'welcome', 'controllers')
         for ctr in ('appadmin', 'default', 'other', 'admin'):
-            open(abspath('applications', 'welcome',
-                 'controllers', '%s.py' % ctr), 'w').close()
-
-        #  create an app-specific routes.py for examples app
-        routes = open(abspath('applications', 'examples', 'routes.py'), 'w')
-        routes.write("routers=dict(examples=dict(default_function='exdef'))")
-        routes.close()
-
-        #  create language files for examples app
+            open(os.path.join(base, '%s.py' % ctr), 'w').close()
+        # create an app-specific routes.py for examples app
+        routes = os.path.join(root, 'applications', 'examples', 'routes.py')
+        with open(routes, 'w') as r:
+            r.write("routers=dict(examples=dict(default_function='exdef'))")
+        # create language files for examples app
+        base = os.path.join(root, 'applications', 'examples', 'static')
         for lang in ('en', 'it'):
-            os.mkdir(abspath('applications', 'examples', 'static', lang))
-            open(abspath('applications', 'examples', 'static',
-                 lang, 'file'), 'w').close()
+            os.mkdir(os.path.join(base, lang))
+            open(os.path.join(base, lang, 'file'), 'w').close()
 
-    global oldcwd
-    if oldcwd is None:  # do this only once
-        oldcwd = os.getcwd()
-        if not os.path.isdir('gluon'):
-            os.chdir(os.path.realpath(
-                '../../'))    # run from web2py base directory
-        import gluon.main     # for initialization after chdir
-        global logger
+    global old_root, root, logger
+    if old_root is None:  # do this only once
+        old_root = global_settings.applications_parent
+        root = global_settings.applications_parent = tempfile.mkdtemp()
+        make_apptree(root)
         logger = logging.getLogger('web2py.rewrite')
-        global_settings.applications_parent = tempfile.mkdtemp()
-        global root
-        root = global_settings.applications_parent
-        make_apptree()
 
 
 def tearDownModule():
-    global oldcwd
-    if oldcwd is not None:
-        os.chdir(oldcwd)
-        oldcwd = None
+    if old_root is not None:
+        global_settings.applications_parent = old_root
+        shutil.rmtree(root)
+
+
+def norm_root(root):
+    return root.replace('/', os.sep)
 
 
 class TestRouter(unittest.TestCase):
@@ -1037,12 +1027,14 @@ class TestRouter(unittest.TestCase):
                 'http://domain.com/app2/static/filename-with_underscore'),
             norm_root("%s/applications/app2/static/filename-with_underscore" % root))
 
-        from gluon.globals import current
-        current.response.static_version = None
-
         self.assertEqual(str(URL(a='init', c='default', f='a_b')), "/a_b")
         self.assertEqual(str(URL(a='app1', c='default', f='a_b')), "/app1/a-b")
         self.assertEqual(str(URL(a='app2', c='default', f='a_b')), "/app2/a_b")
+
+        from gluon.globals import current
+        if hasattr(current, 'response'):
+            current.response.static_version = None
+
         self.assertEqual(
             str(URL(a='app1', c='static', f='a/b_c')), "/app1/static/a/b_c")
         self.assertEqual(

gluon/tests/test_routes.py

@@ -1,81 +1,67 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """Unit tests for rewrite.py regex routing option"""
 
-import sys
 import os
-import unittest
 import tempfile
-import logging
+import unittest
+import shutil
 
-from gluon.rewrite import load, filter_url, filter_err, get_effective_router, regex_filter_out, regex_select
+from gluon.rewrite import load, filter_url, filter_err, regex_filter_out
 from gluon.html import URL
-from gluon.fileutils import abspath
 from gluon.settings import global_settings
 from gluon.http import HTTP
 from gluon.storage import Storage
 from gluon._compat import to_bytes
 
-logger = None
-oldcwd = None
-root = None
+
+old_root = root = None
+
+def setUpModule():
+    def make_apptree(root):
+        "build a temporary applications tree"
+        # applications/
+        os.mkdir(os.path.join(root, 'applications'))
+        # applications/app/
+        for app in ('admin', 'examples', 'welcome'):
+            os.mkdir(os.path.join(root, 'applications', app))
+            #  applications/app/(controllers, static)
+            for subdir in ('controllers', 'static'):
+                os.mkdir(os.path.join(root, 'applications', app, subdir))
+        # applications/admin/controllers/*.py
+        base = os.path.join(root, 'applications', 'admin', 'controllers')
+        for ctr in ('appadmin', 'default', 'gae', 'mercurial', 'shell', 'wizard'):
+            open(os.path.join(base, '%s.py' % ctr), 'w').close()
+        # applications/examples/controllers/*.py
+        base = os.path.join(root, 'applications', 'examples', 'controllers')
+        for ctr in ('ajax_examples', 'appadmin', 'default', 'global', 'spreadsheet'):
+            open(os.path.join(base, '%s.py' % ctr), 'w').close()
+        # applications/welcome/controllers/*.py
+        base = os.path.join(root, 'applications', 'welcome', 'controllers')
+        for ctr in ('appadmin', 'default'):
+            open(os.path.join(base, '%s.py' % ctr), 'w').close()
+        # create an app-specific routes.py for examples app
+        routes = os.path.join(root, 'applications', 'examples', 'routes.py')
+        with open(routes, 'w') as r:
+            r.write("default_function='exdef'\n")
+
+    global old_root, root
+    if old_root is None:  # do this only once
+        old_root = global_settings.applications_parent
+        root = global_settings.applications_parent = tempfile.mkdtemp()
+        make_apptree(root)
+
+
+def tearDownModule():
+    if old_root is not None:
+        global_settings.applications_parent = old_root
+        shutil.rmtree(root)
 
 
 def norm_root(root):
     return root.replace('/', os.sep)
 
 
-def setUpModule():
-    def make_apptree():
-        "build a temporary applications tree"
-        #  applications/
-        os.mkdir(abspath('applications'))
-        #  applications/app/
-        for app in ('admin', 'examples', 'welcome'):
-            os.mkdir(abspath('applications', app))
-            #  applications/app/(controllers, static)
-            for subdir in ('controllers', 'static'):
-                os.mkdir(abspath('applications', app, subdir))
-        #  applications/admin/controllers/*.py
-        for ctr in ('appadmin', 'default', 'gae', 'mercurial', 'shell', 'wizard'):
-            open(abspath('applications', 'admin',
-                 'controllers', '%s.py' % ctr), 'w').close()
-        #  applications/examples/controllers/*.py
-        for ctr in ('ajax_examples', 'appadmin', 'default', 'global', 'spreadsheet'):
-            open(abspath('applications', 'examples',
-                 'controllers', '%s.py' % ctr), 'w').close()
-        #  applications/welcome/controllers/*.py
-        for ctr in ('appadmin', 'default'):
-            open(abspath('applications', 'welcome',
-                 'controllers', '%s.py' % ctr), 'w').close()
-        #  create an app-specific routes.py for examples app
-        routes = open(abspath('applications', 'examples', 'routes.py'), 'w')
-        routes.write("default_function='exdef'\n")
-        routes.close()
-
-    global oldcwd
-    if oldcwd is None:  # do this only once
-        oldcwd = os.getcwd()
-        if not os.path.isdir('gluon'):
-            os.chdir(os.path.realpath(
-                '../../'))    # run from web2py base directory
-        from gluon import main   # for initialization after chdir
-        global logger
-        logger = logging.getLogger('web2py.rewrite')
-        global_settings.applications_parent = tempfile.mkdtemp()
-        global root
-        root = global_settings.applications_parent
-        make_apptree()
-
-
-def tearDownModule():
-    global oldcwd
-    if oldcwd is not None:
-        os.chdir(oldcwd)
-        oldcwd = None
-
-
 class TestRoutes(unittest.TestCase):
     """ Tests the regex routing logic from gluon.rewrite """
 

gluon/tests/test_scheduler.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -636,7 +635,7 @@ def termination():
 
     def exec_sched(self):
         import subprocess
-        call_args = [sys.executable, 'web2py.py', '--no-banner', '-D', '20','-K', 'welcome']
+        call_args = [sys.executable, 'web2py.py', '--no_banner', '-D', 'INFO','-K', 'welcome']
         ret = subprocess.call(call_args, env=dict(os.environ))
         return ret
 

gluon/tests/test_serializers.py

@@ -50,7 +50,8 @@ class TestSerializers(unittest.TestCase):
         lazy_translation = T('abc')
         self.assertEqual(json(lazy_translation), u'"abc"')
         # html helpers are xml()ed before too
-        self.assertEqual(json(SPAN('abc')), u'"<span>abc</span>"')
+        self.assertEqual(json(SPAN('abc'), cls=None), u'"<span>abc</span>"')
+        self.assertEqual(json(SPAN('abc')), u'"\\u003cspan\\u003eabc\\u003c/span\\u003e"')
         # unicode keys make a difference with loads_json
         base = {u'è': 1, 'b': 2}
         base_enc = json(base)

gluon/tests/test_web.py

@@ -1,23 +1,25 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """
     Unit tests for running web2py
 """
 
+from __future__ import print_function
+
 import sys
 import os
 import unittest
 import subprocess
 import time
-
-
+import shutil
 
 from gluon.contrib.webclient import WebClient
 from gluon._compat import urllib2, PY2
+from gluon.fileutils import create_app
+
+test_app_name = '_test_web'
 
 webserverprocess = None
 
-
 def startwebserver():
     global webserverprocess
     path = path = os.path.dirname(os.path.abspath(__file__))
@@ -33,10 +35,10 @@ def startwebserver():
     print('Sleeping before web2py starts...')
     for a in range(1, 11):
         time.sleep(1)
-        print(a, '...')
+        print("%d..." % a)
         try:
-            c = WebClient('http://127.0.0.1:8000')
-            c.get('/')
+            c = WebClient('http://127.0.0.1:8000/')
+            c.get(test_app_name)
             break
         except:
             continue
@@ -71,17 +73,24 @@ class LiveTest(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        appdir = os.path.join('applications', test_app_name)
+        if not os.path.exists(appdir):
+            os.mkdir(appdir)
+            create_app(appdir)
         startwebserver()
 
     @classmethod
     def tearDownClass(cls):
         stopwebserver()
+        appdir = os.path.join('applications', test_app_name)
+        if os.path.exists(appdir):
+            shutil.rmtree(appdir)
 
 
 @unittest.skipIf("datastore" in os.getenv("DB", ""), "TODO: setup web test for app engine")
 class TestWeb(LiveTest):
     def testRegisterAndLogin(self):
-        client = WebClient('http://127.0.0.1:8000/welcome/default/')
+        client = WebClient("http://127.0.0.1:8000/%s/default/" % test_app_name)
 
         client.get('index')
 
@@ -102,28 +111,28 @@ class TestWeb(LiveTest):
                     password='test',
                     _formname='login')
         client.post('user/login', data=data)
-        self.assertTrue('Homer' in client.text)
+        self.assertIn('Homer', client.text)
 
         # check registration and login were successful
         client.get('index')
 
-        self.assertTrue('Homer' in client.text)
+        self.assertIn('Homer', client.text)
 
         client = WebClient('http://127.0.0.1:8000/admin/default/')
         client.post('index', data=dict(password='testpass'))
         client.get('site')
-        client.get('design/welcome')
+        client.get('design/' + test_app_name)
 
     def testStaticCache(self):
-        s = WebClient('http://127.0.0.1:8000/welcome/')
+        s = WebClient("http://127.0.0.1:8000/%s/" % test_app_name)
         s.get('static/js/web2py.js')
-        assert('expires' not in s.headers)
-        assert(not s.headers['cache-control'].startswith('max-age'))
+        self.assertNotIn('expires', s.headers)
+        self.assertFalse(s.headers['cache-control'].startswith('max-age'))
         text = s.text
         s.get('static/_1.2.3/js/web2py.js')
-        assert(text == s.text)
-        assert('expires' in s.headers)
-        assert(s.headers['cache-control'].startswith('max-age'))
+        self.assertEqual(text, s.text)
+        self.assertIn('expires', s.headers)
+        self.assertTrue(s.headers['cache-control'].startswith('max-age'))
 
     @unittest.skipIf(not(PY2), 'skip PY3 testSoap')
     def testSoap(self):
@@ -133,15 +142,15 @@ class TestWeb(LiveTest):
         client = SoapClient(wsdl=url)
         ret = client.SubIntegers(a=3, b=2)
         # check that the value returned is ok
-        assert('SubResult' in ret)
-        assert(ret['SubResult'] == 1)
+        self.assertIn('SubResult', ret)
+        self.assertEqual(ret['SubResult'], 1)
 
         try:
             ret = client.Division(a=3, b=0)
         except SoapFault as sf:
             # verify the exception value is ok
-            # assert(sf.faultstring == "float division by zero") # true only in 2.7
-            assert(sf.faultcode == "Server.ZeroDivisionError")
+            # self.assertEqual(sf.faultstring, "float division by zero") # true only in 2.7
+            self.assertEqual(sf.faultcode, "Server.ZeroDivisionError")
 
         # store sent and received xml for low level test
         xml_request = client.xml_request
@@ -152,7 +161,7 @@ class TestWeb(LiveTest):
         try:
             s.post('examples/soap_examples/call/soap', data=xml_request, method="POST")
         except urllib2.HTTPError as e:
-            assert(e.msg == 'INTERNAL SERVER ERROR')
+            self.assertEqual(e.msg, 'INTERNAL SERVER ERROR')
         # check internal server error returned (issue 153)
-        assert(s.status == 500)
-        assert(s.text == xml_response)
+        self.assertEqual(s.status, 500)
+        self.assertEqual(s.text, xml_response)

gluon/widget.py

@@ -5,8 +5,8 @@
 | Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
 
-The widget is called from web2py
-----------------------------------
+GUI widget and services start function
+--------------------------------------
 """
 
 from __future__ import print_function
@@ -16,15 +16,15 @@ from gluon._compat import thread, xrange, PY2
 import time
 import threading
 import os
-import copy
 import socket
 import signal
 import math
 import logging
 import getpass
-from gluon import main, newcron
 
-from gluon.fileutils import read_file, write_file, create_welcome_w2p
+from gluon import main, newcron
+from gluon.fileutils import read_file, create_welcome_w2p
+from gluon.console import console
 from gluon.settings import global_settings
 from gluon.shell import die, run, test
 from gluon.utils import is_valid_ip_address, is_loopback_ip_address, getipaddrinfo
@@ -314,22 +314,20 @@ class web2pyDialog(object):
             self.tb = None
 
     def update_schedulers(self, start=False):
-        applications_folder = os.path.join(self.options.folder, 'applications')
-        apps = []
-        available_apps = [
-            arq for arq in os.listdir(applications_folder)
-            if os.path.isdir(os.path.join(applications_folder, arq))
-        ]
-        if start:
-            # the widget takes care of starting the scheduler
-            if self.options.scheduler and self.options.with_scheduler:
-                apps = [app for app
-                        in map(lambda ag : ag.split(':', 1)[0].strip(), self.options.scheduler.split(','))
-                        if app in available_apps]
+        if start and self.options.with_scheduler and self.options.schedulers:
+            # the widget takes care of starting the schedulers
+            apps = [ag.split(':', 1)[0] for ag in self.options.schedulers]
+        else:
+            apps = []
         for app in apps:
             self.try_start_scheduler(app)
 
         # reset the menu
+        applications_folder = os.path.join(self.options.folder, 'applications')
+        available_apps = [
+            arq for arq in os.listdir(applications_folder)
+            if os.path.isdir(os.path.join(applications_folder, arq))
+        ]
         self.schedmenu.delete(0, len(available_apps))
 
         for arq in available_apps:
@@ -351,7 +349,7 @@ class web2pyDialog(object):
         code = "from gluon.globals import current;current._scheduler.loop()"
         print('starting scheduler from widget for "%s"...' % app)
         args = (app, True, True, None, False, code, False, True)
-        logging.getLogger().setLevel(self.options.debuglevel)
+        logging.getLogger().setLevel(self.options.log_level)
         p = Process(target=run, args=args)
         self.scheduler_processes[app] = p
         self.update_schedulers()
@@ -473,7 +471,7 @@ class web2pyDialog(object):
         except:
             return self.error('invalid port number')
 
-        if self.options.ssl_certificate and self.options.ssl_private_key:
+        if self.options.server_key and self.options.server_cert:
             proto = 'https'
         else:
             proto = 'http'
@@ -492,11 +490,11 @@ class web2pyDialog(object):
                 pid_filename=options.pid_filename,
                 log_filename=options.log_filename,
                 profiler_dir=options.profiler_dir,
-                ssl_certificate=options.ssl_certificate,
-                ssl_private_key=options.ssl_private_key,
+                ssl_certificate=options.server_cert,
+                ssl_private_key=options.server_key,
                 ssl_ca_certificate=options.ca_cert,
-                min_threads=options.minthreads,
-                max_threads=options.maxthreads,
+                min_threads=options.min_threads,
+                max_threads=options.max_threads,
                 server_name=options.server_name,
                 request_queue_size=req_queue_size,
                 timeout=options.timeout,
@@ -582,328 +580,6 @@ class web2pyDialog(object):
         self.canvas.after(1000, self.update_canvas)
 
 
-def console():
-    """ Defines the behavior of the console web2py execution """
-    import optparse
-
-    parser = optparse.OptionParser(
-        usage='python %prog [options]',
-        version=ProgramVersion,
-        description='web2py Web Framework startup script.',
-        epilog='''NOTE: unless a password is specified (-a 'passwd')
-web2py will attempt to run a GUI to ask for it when starting the web server
-(if not disabled with --nogui).''')
-
-    parser.add_option('-i', '--ip',
-                      default='127.0.0.1',
-                      metavar='IP_ADDR', help=\
-        'IP address of the server (e.g., 127.0.0.1 or ::1); ' \
-        'Note: This value is ignored when using the --interfaces option')
-
-    parser.add_option('-p', '--port',
-                      default=8000,
-                      type='int', metavar='NUM', help=\
-        'port of server (%default); ' \
-        'Note: This value is ignored when using the --interfaces option')
-
-    parser.add_option('-G', '--GAE', dest='gae',
-                      default=None,
-                      metavar='APP_NAME', help=\
-        'will create app.yaml and gaehandler.py and exit')
-
-    parser.add_option('-a', '--password',
-                      default='<ask>',
-                      help=\
-        'password to be used for administration ' \
-        '(use "<recycle>" to reuse the last password), ' \
-        'when no password is available the administrative ' \
-        'interface will be disabled')
-
-    parser.add_option('-c', '--ssl_certificate',
-                      default=None,
-                      metavar='FILE', help='server certificate file')
-
-    parser.add_option('-k', '--ssl_private_key',
-                      default=None,
-                      metavar='FILE', help='server private key file')
-
-    parser.add_option('--ca-cert', dest='ca_cert', # not needed
-                      default=None,
-                      metavar='FILE', help='CA certificate file')
-
-    parser.add_option('-d', '--pid_filename',
-                      default='httpserver.pid',
-                      metavar='FILE', help='server pid file (%default)')
-
-    parser.add_option('-l', '--log_filename',
-                      default='httpserver.log',
-                      metavar='FILE', help='server log file (%default)')
-
-    parser.add_option('-n', '--numthreads',
-                      default=None,
-                      type='int', metavar='NUM',
-                      help='number of threads (deprecated)')
-
-    parser.add_option('--minthreads',
-                      default=None,
-                      type='int', metavar='NUM',
-                      help='minimum number of server threads')
-
-    parser.add_option('--maxthreads',
-                      default=None,
-                      type='int', metavar='NUM',
-                      help='maximum number of server threads')
-
-    parser.add_option('-s', '--server_name',
-                      default=socket.gethostname(),
-                      help='web server name (%default)')
-
-    parser.add_option('-q', '--request_queue_size',
-                      default=5,
-                      type='int', metavar='NUM',
-                      help=\
-        'max number of queued requests when server unavailable (%default)')
-
-    parser.add_option('-o', '--timeout',
-                      default=10,
-                      type='int', metavar='SECONDS',
-                      help='timeout for individual request (%default seconds)')
-
-    parser.add_option('-z', '--shutdown_timeout',
-                      default=None,
-                      type='int', metavar='SECONDS',
-                      help=\
-        'timeout on server shutdown; this value is not used by ' \
-        'Rocket web server')
-
-    parser.add_option('--socket-timeout', dest='socket_timeout', # not needed
-                      default=5,
-                      type='int', metavar='SECONDS',
-                      help='timeout for socket (%default seconds)')
-
-    parser.add_option('-f', '--folder',
-                      default=os.getcwd(), metavar='WEB2PY_DIR',
-                      help='folder from which to run web2py')
-
-    parser.add_option('-v', '--verbose',
-                      default=False,
-                      action='store_true',
-                      help='increase --test and --run_system_tests verbosity')
-
-    parser.add_option('-Q', '--quiet',
-                      default=False,
-                      action='store_true',
-                      help='disable all output')
-
-    parser.add_option('-e', '--errors_to_console',
-                      default=False,
-                      action='store_true',
-                      help='log all errors to console')
-
-    parser.add_option('-D', '--debug', dest='debuglevel',
-                      default=30,
-                      type='int',
-                      metavar='LOG_LEVEL', help=\
-        'set log level (0-100, 0 means all, 100 means none; ' \
-        'default is %default)')
-
-    parser.add_option('-S', '--shell',
-                      default=None,
-                      metavar='APPNAME', help=\
-        'run web2py in interactive shell or IPython (if installed) with ' \
-        'specified appname (if app does not exist it will be created). ' \
-        'APPNAME like a/c/f?x=y (c, f and vars optional)')
-
-    parser.add_option('-B', '--bpython',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'run web2py in interactive shell or bpython (if installed) with ' \
-        'specified appname (if app does not exist it will be created). ' \
-        'Use combined with --shell')
-
-    parser.add_option('-P', '--plain',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'only use plain python shell; should be used with --shell option')
-
-    parser.add_option('-M', '--import_models',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'auto import model files (default is %default); should be used ' \
-        'with --shell option')
-
-    parser.add_option('-R', '--run',
-                      default='', # NOTE: used for sys.argv[0] if --shell
-                      metavar='PYTHON_FILE', help=\
-        'run PYTHON_FILE in web2py environment; ' \
-        'should be used with --shell option')
-
-    parser.add_option('-K', '--scheduler',
-                      default=None,
-                      metavar='APP_LIST', help=\
-        'run scheduled tasks for the specified apps: expects a list of ' \
-        'app names as app1,app2,app3 ' \
-        'or a list of app:groups as app1:group1:group2,app2:group1 ' \
-        '(only strings, no spaces allowed). NOTE: ' \
-        'Requires a scheduler defined in the models')
-
-    parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed
-                      default=False,
-                      action='store_true',
-                      help=\
-        'run schedulers alongside webserver, needs -K')
-
-    parser.add_option('-T', '--test',
-                      default=None,
-                      metavar='TEST_PATH', help=\
-        'run doctests in web2py environment; ' \
-        'TEST_PATH like a/c/f (c, f optional)')
-
-    parser.add_option('-C', '--cron', dest='extcron',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'trigger a cron run and exit; usually used when invoked ' \
-        'from a system crontab')
-
-    parser.add_option('--softcron',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'use software cron emulation instead of separate cron process, '\
-        'needs -Y; NOTE: use of software cron emulation is strongly '
-        'discouraged')
-
-    parser.add_option('-Y', '--run-cron', dest='runcron',
-                      default=False,
-                      action='store_true',
-                      help='start the background cron process')
-
-    parser.add_option('-J', '--cronjob',
-                      default=False,
-                      action='store_true',
-                      # NOTE: help suppressed because this option is
-                      #       intended for internal use only
-                      help=optparse.SUPPRESS_HELP)
-
-    parser.add_option('-L', '--config',
-                      default='',
-                      help='config file')
-
-    parser.add_option('-F', '--profiler', dest='profiler_dir',
-                      default=None,
-                      help='profiler dir')
-
-    parser.add_option('-t', '--taskbar',
-                      default=False,
-                      action='store_true',
-                      help='use web2py GUI and run in taskbar (system tray)')
-
-    parser.add_option('--nogui',
-                      default=False,
-                      action='store_true',
-                      help='do not run GUI')
-
-    parser.add_option('-A', '--args',
-                      default=None,
-                      help=\
-        'should be followed by a list of arguments to be passed to script, ' \
-        'to be used with -S; NOTE: must be the last option because eat all ' \
-        'remaining arguments')
-
-    parser.add_option('--no-banner', dest='no_banner', # not needed
-                      default=False,
-                      action='store_true',
-                      help='do not print header banner')
-
-    parser.add_option('--interfaces',
-                      default=None,
-                      help=\
-        'listen on multiple addresses: ' \
-        '"ip1:port1:key1:cert1:ca_cert1;ip2:port2:key2:cert2:ca_cert2;..." ' \
-        '(:key:cert:ca_cert optional; no spaces; IPv6 addresses must be in ' \
-        'square [] brackets)')
-
-    parser.add_option('--run_system_tests',
-                      default=False,
-                      action='store_true',
-                      help='run web2py tests')
-
-    parser.add_option('--with_coverage',
-                      default=False,
-                      action='store_true',
-                      help=\
-        'collect coverage data when used with --run_system_tests; ' \
-        'require Python 2.7+ and the coverage module installed')
-
-    parser.add_option('--force_migrate',
-                      default=False,
-                      action='store_true',
-                      help= \
-                          'force DAL to migrate all tables that should be migrated when enabled; ' \
-                          'monkeypatch in the DAL class to force _migrate_enabled=True')
-
-    if '-A' in sys.argv:
-        k = sys.argv.index('-A')
-    elif '--args' in sys.argv:
-        k = sys.argv.index('--args')
-    else:
-        k = len(sys.argv)
-    sys.argv, other_args = sys.argv[:k], sys.argv[k + 1:]
-    (options, args) = parser.parse_args()
-    # TODO: warn or error if args (should be no unparsed arguments)
-    options.args = other_args
-
-    if options.taskbar and os.name != 'nt':
-        # TODO: warn and disable taskbar instead of exit
-        die('taskbar not supported on this platform')
-
-    if options.config.endswith('.py'):
-        options.config = options.config[:-3]
-    if options.config:
-        # import options from options.config file
-        try:
-            # FIXME: avoid __import__
-            options2 = __import__(options.config)
-        except:
-            die("cannot import config file %s" % options.config)
-        for key in dir(options2):
-            if hasattr(options, key):
-                setattr(options, key, getattr(options2, key))
-
-    # transform options.interfaces, in the form
-    # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3"
-    # (no spaces; optional key:cert:ca_cert indicate SSL), into
-    # a list of tuples
-    if options.interfaces:
-        interfaces = options.interfaces.split(';')
-        options.interfaces = []
-        for interface in interfaces:
-            if interface.startswith('['):
-                # IPv6
-                ip, if_remainder = interface.split(']', 1)
-                ip = ip[1:]
-                interface = if_remainder[1:].split(':')
-                interface.insert(0, ip)
-            else:
-                # IPv4
-                interface = interface.split(':')
-            interface[1] = int(interface[1])  # numeric port
-            options.interfaces.append(tuple(interface))
-
-    if options.numthreads is not None and options.minthreads is None:
-        options.minthreads = options.numthreads  # legacy
-
-    copy_options = copy.deepcopy(options)
-    copy_options.password = '******'
-    global_settings.cmd_options = copy_options
-
-    return options, args
-
-
 def check_existent_app(options, appname):
     if os.path.isdir(os.path.join(options.folder, 'applications', appname)):
         return True
@@ -928,11 +604,10 @@ def start_schedulers(options):
     except:
         sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
         return
-    logging.getLogger().setLevel(options.debuglevel)
+    logging.getLogger().setLevel(options.log_level)
 
-    apps = [[n.strip() for n in sched_app.split(':')]
-            for sched_app in options.scheduler.split(',')]
-    if len(apps) == 1 and not options.with_scheduler:
+    apps = [ag.split(':') for ag in options.schedulers]
+    if not options.with_scheduler and len(apps) == 1:
         app, code = get_code_for_scheduler(apps[0], options)
         if not app:
             return
@@ -975,7 +650,7 @@ def start():
     """ Starts server and other services """
 
     # get command line arguments
-    (options, args) = console()
+    options = console(version=ProgramVersion)
 
     if options.gae:
         # write app.yaml, gaehandler.py, and exit
@@ -997,7 +672,7 @@ def start():
         return
 
     logger = logging.getLogger("web2py")
-    logger.setLevel(options.debuglevel)
+    logger.setLevel(options.log_level)
 
     # on new installation build the scaffolding app
     create_welcome_w2p()
@@ -1034,36 +709,29 @@ def start():
         from pydal.drivers import DRIVERS
         print('Database drivers available: %s' % ', '.join(DRIVERS))
 
-    if options.test:
+    if options.run_doctests:
         # run doctests and exit
-        test(options.test, verbose=options.verbose)
+        test(options.run_doctests, verbose=options.verbose)
         return
 
     if options.shell:
         # run interactive shell and exit
-        sys.argv = [options.run] + options.args
+        sys.argv = [options.run or ''] + options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
             import_models=options.import_models, startfile=options.run,
-            cronjob=options.cronjob, force_migrate=options.force_migrate)
+            cron_job=options.cron_job, force_migrate=options.force_migrate)
         return
 
-    if options.extcron:
+    if options.cron_run:
         # run cron (extcron) and exit
         logger.debug('Starting extcron...')
         global_settings.web2py_crontype = 'external'
-        if options.scheduler:
-            # run cron for applications listed with --scheduler (-K)
-            apps = [app for app
-                    in map(lambda ag : ag.split(':', 1)[0].strip(), options.scheduler.split(','))
-                    if check_existent_app(options, app)]
-        else:
-            apps = None
-        extcron = newcron.extcron(options.folder, apps=apps)
+        extcron = newcron.extcron(options.folder, apps=options.crontabs)
         extcron.start()
         extcron.join()
         return
 
-    if options.scheduler and not options.with_scheduler:
+    if not options.with_scheduler and options.schedulers:
         # run schedulers and exit
         try:
             start_schedulers(options)
@@ -1071,22 +739,22 @@ def start():
             pass
         return
 
-    if options.runcron:
-        if options.softcron:
-            print('Using softcron (but this is not very efficient)')
+    if options.with_cron:
+        if options.soft_cron:
+            print('Using cron software emulation (but this is not very efficient)')
             global_settings.web2py_crontype = 'soft'
         else:
             # start hardcron thread
             logger.debug('Starting hardcron...')
             global_settings.web2py_crontype = 'hard'
-            newcron.hardcron(options.folder).start()
+            newcron.hardcron(options.folder, apps=options.crontabs).start()
 
     # if no password provided and have Tk library start GUI (when not
     # explicitly disabled), we also need a GUI to put in taskbar (system tray)
     # when requested
     root = None
 
-    if (not options.nogui and options.password == '<ask>') or options.taskbar:
+    if (not options.no_gui and options.password == '<ask>') or options.taskbar:
         try:
             if PY2:
                 import Tkinter as tkinter
@@ -1096,10 +764,10 @@ def start():
         except (ImportError, OSError):
             logger.warn(
                 'GUI not available because Tk library is not installed')
-            options.nogui = True
+            options.no_gui = True
         except:
             logger.exception('cannot get Tk root window, GUI disabled')
-            options.nogui = True
+            options.no_gui = True
 
     if root:
         # run GUI and exit
@@ -1128,7 +796,7 @@ end tell
 
     spt = None
 
-    if options.scheduler and options.with_scheduler:
+    if options.with_scheduler and options.schedulers:
         # start schedulers in a separate thread
         spt = threading.Thread(target=start_schedulers, args=(options,))
         spt.start()
@@ -1151,7 +819,7 @@ end tell
         ip = first_if[0]
         port = first_if[1]
 
-    if options.ssl_certificate and options.ssl_private_key:
+    if options.server_key and options.server_cert:
         proto = 'https'
     else:
         proto = 'http'
@@ -1193,11 +861,11 @@ end tell
                              pid_filename=options.pid_filename,
                              log_filename=options.log_filename,
                              profiler_dir=options.profiler_dir,
-                             ssl_certificate=options.ssl_certificate,
-                             ssl_private_key=options.ssl_private_key,
+                             ssl_certificate=options.server_cert,
+                             ssl_private_key=options.server_key,
                              ssl_ca_certificate=options.ca_cert,
-                             min_threads=options.minthreads,
-                             max_threads=options.maxthreads,
+                             min_threads=options.min_threads,
+                             max_threads=options.max_threads,
                              server_name=options.server_name,
                              request_queue_size=options.request_queue_size,
                              timeout=options.timeout,

scripts/web2py.fedora.sh

@@ -29,7 +29,7 @@ cd $DAEMON_DIR
 
 start() {
         echo -n $"Starting $DESC ($NAME): "
-        daemon --check $NAME $PYTHON $DAEMON_DIR/web2py.py -Q --nogui -a $ADMINPASS -d $PIDFILE -p $PORT &
+        daemon --check $NAME $PYTHON $DAEMON_DIR/web2py.py -Q --no_gui -a $ADMINPASS -d $PIDFILE -p $PORT &
         RETVAL=$?
         if [ $RETVAL -eq 0 ]; then
                 touch /var/lock/subsys/$NAME