diff --git a/CHANGELOG b/CHANGELOG index b5909b91..47b185a4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,24 @@ +## 2.19.0 +- new command line options (Thanks Paolo Pastori) + +OLD NAME NEW NAME +================== ================== +--debug --log_level +--nogui --no_gui +--ssl_private_key --server_key +--ssl_certificate --server_cert +--minthreads --min_threads +--maxthreads --max_threads +--profiler --profiler_dir +--run-cron --with_cron +--softcron --soft_cron +--cron --cron_run +--cronjob * --cron_job * +--test --run_doctests + --add_options + --interface + --crontab + ## 2.18.1-2.18.5 - pydal 19.04 - made template its own module (Yet Another Template Language) diff --git a/Makefile b/Makefile index 2e33adf9..beed2957 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,11 @@ clean: find applications/welcome/ -name '.*' -exec rm -f {} \; find . -name '*.pyc' -exec rm -f {} \; tests: - python web2py.py --run_system_tests + python web2py.py --verbose --run_system_tests coverage: coverage erase --rcfile=gluon/tests/coverage.ini export COVERAGE_PROCESS_START=gluon/tests/coverage.ini - python web2py.py --run_system_tests --with_coverage + python web2py.py --verbose --run_system_tests --with_coverage coverage combine --rcfile=gluon/tests/coverage.ini sleep 1 coverage html --rcfile=gluon/tests/coverage.ini diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index eb2d32f2..bf06d401 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -166,9 +166,9 @@ def check_version(): new_version, version = check_new_version(request.env.web2py_version, WEB2PY_VERSION_URL) - if new_version == -1: + if new_version in (-1, -2): return A(T('Unable to check for upgrades'), _href=WEB2PY_URL) - elif new_version != True: + elif not new_version: return A(T('web2py is up to date'), _href=WEB2PY_URL) elif platform.system().lower() in ('windows', 'win32', 'win64') and os.path.exists("web2py.exe"): return SPAN('You should upgrade to %s' % version.split('(')[0]) @@ -1120,7 +1120,7 @@ def design(): defines = {} for m in models: data = safe_read(apath('%s/models/%s' % (app, m), r=request)) - defines[m] = regex_tables.findall(data) + defines[m] = re.findall(REGEX_DEFINE_TABLE, data, re.MULTILINE) defines[m].sort() # Get all controllers @@ -1144,12 +1144,12 @@ def design(): include = {} for c in views: data = safe_read(apath('%s/views/%s' % (app, c), r=request)) - items = regex_extend.findall(data) + items = re.findall(REGEX_EXTEND, data, re.MULTILINE) if items: extend[c] = items[0][1] - items = regex_include.findall(data) + items = re.findall(REGEX_INCLUDE, data) include[c] = [i[1] for i in items] # Get all modules @@ -1285,11 +1285,11 @@ def plugin(): include = {} for c in views: data = safe_read(apath('%s/views/%s' % (app, c), r=request)) - items = regex_extend.findall(data) + items = re.findall(REGEX_EXTEND, data, re.MULTILINE) if items: extend[c] = items[0][1] - items = regex_include.findall(data) + items = re.findall(REGEX_INCLUDE, data) include[c] = [i[1] for i in items] # Get all modules diff --git a/applications/admin/models/0_imports.py b/applications/admin/models/0_imports.py index b911c0e7..6b550b83 100644 --- a/applications/admin/models/0_imports.py +++ b/applications/admin/models/0_imports.py @@ -25,6 +25,5 @@ from gluon.utils import md5_hash from gluon.fileutils import listdir, cleanpath, up from gluon.fileutils import tar, tar_compiled, untar, fix_newlines from gluon.languages import findT, update_all_languages -from gluon.myregex import * from gluon.restricted import * from gluon.compileapp import compile_application, remove_compiled_application diff --git a/docker/alpine/web2py-rocket-ssl/Dockerfile b/docker/alpine/web2py-rocket-ssl/Dockerfile index 05a8c47d..f0d34fd1 100755 --- a/docker/alpine/web2py-rocket-ssl/Dockerfile +++ b/docker/alpine/web2py-rocket-ssl/Dockerfile @@ -19,4 +19,4 @@ WORKDIR /web2py EXPOSE 443 -CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443 +CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443 diff --git a/docker/alpine/web2py-rocket/Dockerfile b/docker/alpine/web2py-rocket/Dockerfile index 404e0f45..dd08901a 100755 --- a/docker/alpine/web2py-rocket/Dockerfile +++ b/docker/alpine/web2py-rocket/Dockerfile @@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/centos/web2py-rocket/Dockerfile b/docker/centos/web2py-rocket/Dockerfile index 212f84bb..bd86aad4 100755 --- a/docker/centos/web2py-rocket/Dockerfile +++ b/docker/centos/web2py-rocket/Dockerfile @@ -25,4 +25,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/debian/web2py-rocket/Dockerfile b/docker/debian/web2py-rocket/Dockerfile index 802491cc..43c3164d 100755 --- a/docker/debian/web2py-rocket/Dockerfile +++ b/docker/debian/web2py-rocket/Dockerfile @@ -25,4 +25,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/fedora/web2py-rocket/Dockerfile b/docker/fedora/web2py-rocket/Dockerfile index 3fa20078..bdd1db99 100755 --- a/docker/fedora/web2py-rocket/Dockerfile +++ b/docker/fedora/web2py-rocket/Dockerfile @@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/opensuse/web2py-rocket/Dockerfile b/docker/opensuse/web2py-rocket/Dockerfile index a8fca1e2..24797491 100755 --- a/docker/opensuse/web2py-rocket/Dockerfile +++ b/docker/opensuse/web2py-rocket/Dockerfile @@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/python/web2py-rocket-ssl/Dockerfile b/docker/python/web2py-rocket-ssl/Dockerfile index 9308ca4b..8f238582 100755 --- a/docker/python/web2py-rocket-ssl/Dockerfile +++ b/docker/python/web2py-rocket-ssl/Dockerfile @@ -18,4 +18,4 @@ WORKDIR /web2py EXPOSE 443 -CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443 +CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443 diff --git a/docker/python/web2py-rocket/Dockerfile b/docker/python/web2py-rocket/Dockerfile index 28058453..94fdf95e 100755 --- a/docker/python/web2py-rocket/Dockerfile +++ b/docker/python/web2py-rocket/Dockerfile @@ -20,4 +20,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/stack/web2py-rocket-nginx/web2py-rocket b/docker/stack/web2py-rocket-nginx/web2py-rocket index ed228fcf..5b64b076 100644 --- a/docker/stack/web2py-rocket-nginx/web2py-rocket +++ b/docker/stack/web2py-rocket-nginx/web2py-rocket @@ -22,4 +22,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl index 5a7ffe54..1d6629b7 100644 --- a/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl +++ b/docker/stack/web2py-rocket-ssl-nginx-db-adminer/web2py-rocket-ssl @@ -17,4 +17,4 @@ WORKDIR /web2py EXPOSE 443 -CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443 +CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443 diff --git a/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl index 6720178a..6f271006 100644 --- a/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl +++ b/docker/stack/web2py-rocket-ssl-nginx-memcached/web2py-rocket-ssl @@ -17,4 +17,4 @@ WORKDIR /web2py EXPOSE 443 -CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443 +CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443 diff --git a/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl index 0b8c82f9..c6bb03fc 100644 --- a/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl +++ b/docker/stack/web2py-rocket-ssl-nginx-redis/web2py-rocket-ssl @@ -17,4 +17,4 @@ WORKDIR /web2py EXPOSE 443 -CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443 +CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443 diff --git a/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl b/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl index 6720178a..6f271006 100644 --- a/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl +++ b/docker/stack/web2py-rocket-ssl-nginx/web2py-rocket-ssl @@ -17,4 +17,4 @@ WORKDIR /web2py EXPOSE 443 -CMD python /web2py/web2py.py --nogui --no-banner -a 'a' -c web2py.crt -k web2py.key -i 0.0.0.0 -p 443 +CMD python /web2py/web2py.py --no_gui --no_banner -a 'a' -k web2py.key -c web2py.crt -i 0.0.0.0 -p 443 diff --git a/docker/ubuntu/web2py-rocket/Dockerfile b/docker/ubuntu/web2py-rocket/Dockerfile index ea48283d..ab96dca2 100755 --- a/docker/ubuntu/web2py-rocket/Dockerfile +++ b/docker/ubuntu/web2py-rocket/Dockerfile @@ -24,4 +24,4 @@ WORKDIR /home/web2py/web2py EXPOSE 8000 -CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --nogui --no-banner -a 'a' -i 0.0.0.0 -p 8000 +CMD . /home/web2py/bin/activate && python /home/web2py/web2py/web2py.py --no_gui --no_banner -a 'a' -i 0.0.0.0 -p 8000 diff --git a/examples/options_std.py b/examples/options_std.py deleted file mode 100644 index 6a7d822f..00000000 --- a/examples/options_std.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- - -# when web2py is run as a windows service (web2py.py -W) -# it does not load the command line options but it -# expects to find configuration settings in a file called -# -# web2py/options.py -# -# this file is an example for options.py - -import socket -import os - -ip = '0.0.0.0' -port = 80 -interfaces = [('0.0.0.0', 80)] - #,('0.0.0.0',443,'ssl_private_key.pem','ssl_certificate.pem')] -password = '' # ## means use the previous password -pid_filename = 'httpserver.pid' -log_filename = 'httpserver.log' -profiler_filename = None -ssl_certificate = '' # 'ssl_certificate.pem' # ## path to certificate file -ssl_private_key = '' # 'ssl_private_key.pem' # ## path to private key file -#numthreads = 50 # ## deprecated; remove -minthreads = None -maxthreads = None -server_name = socket.gethostname() -request_queue_size = 5 -timeout = 30 -shutdown_timeout = 5 -folder = os.getcwd() -extcron = None -nocron = None diff --git a/gluon/admin.py b/gluon/admin.py index 0c09bb69..79c8e7ff 100644 --- a/gluon/admin.py +++ b/gluon/admin.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- +# vim: set ts=4 sw=4 et ai: """ | This file is part of the web2py Web Framework @@ -10,24 +10,34 @@ Utility functions for the Admin application ------------------------------------------- """ +from __future__ import print_function + import os -import sys import traceback -import zipfile from shutil import rmtree, copyfileobj +import zipfile +import sys + from gluon.fileutils import (w2p_pack, create_app, w2p_unpack, w2p_pack_plugin, w2p_unpack_plugin, up, fix_newlines, abspath, recursive_unlink, - read_file, write_file, parse_version) + read_file, write_file, parse_version, missing_app_folders) from gluon.restricted import RestrictedError from gluon.settings import global_settings from gluon.cache import CacheOnDisk from gluon._compat import urlopen, to_native +# TODO: move into add_path_first if not global_settings.web2py_runtime_gae: import site +REGEX_DEFINE_TABLE = r"""^\w+\.define_table\(\s*['"](?P\w+)['"]""" +REGEX_EXTEND = r"""^\s*(?P\{\{\s*extend\s+['"](?P[^'"]+)['"]\s*\}\})""" +REGEX_INCLUDE = r"""(?P\{\{\s*include\s+['"](?P[^'"]+)['"]\s*\}\})""" + + +# TODO: swap arguments, let first ('r' or whatever) be mandatory def apath(path='', r=None): """Builds a path inside an application folder @@ -38,8 +48,9 @@ def apath(path='', r=None): """ opath = up(r.folder) - while path[:3] == '../': - (opath, path) = (up(opath), path[3:]) + while path.startswith('../'): + opath = up(opath) + path = path[3:] return os.path.join(opath, path).replace('\\', '/') @@ -81,7 +92,7 @@ def app_pack_compiled(app, request, raise_ex=False): filename = apath('../deposit/%s.w2p' % app, request) w2p_pack(filename, apath(app, request), compiled=True) return filename - except Exception as e: + except Exception: if raise_ex: raise return None @@ -105,7 +116,7 @@ def app_cleanup(app, request): if os.path.exists(path): for f in os.listdir(path): try: - if f[:1] != '.': + if not f.startswith('.'): os.unlink(os.path.join(path, f)) except IOError: r = False @@ -115,7 +126,7 @@ def app_cleanup(app, request): if os.path.exists(path): for f in os.listdir(path): try: - if f[:1] != '.': + if not f.startswith('.'): recursive_unlink(os.path.join(path, f)) except (OSError, IOError): r = False @@ -126,7 +137,7 @@ def app_cleanup(app, request): CacheOnDisk(folder=path).clear() for f in os.listdir(path): try: - if f[:1] != '.': + if not f.startswith('.'): recursive_unlink(os.path.join(path, f)) except (OSError, IOError): r = False @@ -211,9 +222,9 @@ def app_install(app, fobj, request, filename, overwrite=None): """ did_mkdir = False - if filename[-4:] == '.w2p': + if filename.endswith('.w2p'): extension = 'w2p' - elif filename[-7:] == '.tar.gz': + elif filename.endswith('.tar.gz'): extension = 'tar.gz' else: extension = 'tar' @@ -300,7 +311,8 @@ def plugin_install(app, fobj, request, filename): upname = apath('../deposit/%s' % filename, request) try: - write_file(upname, fobj.read(), 'wb') + with open(upname, 'wb') as appfp: + copyfileobj(fobj, appfp, 4194304) # 4 MB buffer path = apath(app, request) w2p_unpack_plugin(upname, path) fix_newlines(path) @@ -322,7 +334,9 @@ def check_new_version(myversion, version_url): tuple: state, version - state : `True` if upgrade available, `False` if current - version is up-to-date, -1 on error + version is up-to-date, -1 on error, + -2 when the system is likely to be offline (no + internet link available) - version : the most up-to-version available """ @@ -330,10 +344,19 @@ def check_new_version(myversion, version_url): version = to_native(urlopen(version_url).read()) pversion = parse_version(version) pmyversion = parse_version(myversion) - except IOError: - import traceback - print(traceback.format_exc()) - return -1, myversion + except IOError as e: + from socket import gaierror + if isinstance(getattr(e, 'reason', None), gaierror) and \ + e.reason.errno == -2: + # assuming the version_url is ok the socket.gaierror + # (gaierror stands for getaddrinfo() error) that + # originates the exception is probably due to a + # missing internet link (i.e. the system is offline) + print('system is offline, cannot retrieve latest web2py version') + return -2, myversion + else: + print(traceback.format_exc()) + return -1, myversion if pversion[:3]+pversion[-6:] > pmyversion[:3]+pmyversion[-6:]: return True, version @@ -360,7 +383,7 @@ def unzip(filename, dir, subfolder=''): for name in sorted(zf.namelist()): if not name.startswith(subfolder): continue - # print name[n:] + # print(name[n:]) if name.endswith('/'): folder = os.path.join(dir, name[n:]) if not os.path.exists(folder): @@ -421,6 +444,7 @@ def upgrade(request, url='http://web2py.com'): return False, e +# TODO: move to fileutils def add_path_first(path): sys.path = [path] + [p for p in sys.path if ( not p == path and not p == (path + '/'))] @@ -429,21 +453,24 @@ def add_path_first(path): site.addsitedir(path) +# TODO: move to fileutils def try_mkdir(path): if not os.path.exists(path): try: if os.path.islink(path): - # path is a broken link, try to mkdir the target of the link instead of the link itself. + # path is a broken link, try to mkdir the target of the link + # instead of the link itself. os.mkdir(os.path.realpath(path)) else: os.mkdir(path) except OSError as e: - if e.strerror == 'File exists': # In case of race condition. + if e.errno == 17: # "File exists" (race condition). pass else: - raise e + raise +# TODO: move to fileutils def create_missing_folders(): if not global_settings.web2py_runtime_gae: for path in ('applications', 'deposit', 'site-packages', 'logs'): @@ -453,16 +480,16 @@ def create_missing_folders(): paths = (global_settings.gluon_parent, abspath( 'site-packages', gluon=True), abspath('gluon', gluon=True), '') """ - paths = (global_settings.gluon_parent, abspath( - 'site-packages', gluon=True), '') - [add_path_first(p) for p in paths] + for p in (global_settings.gluon_parent, + abspath('site-packages', gluon=True), + ''): + add_path_first(p) +# TODO: move to fileutils def create_missing_app_folders(request): if not global_settings.web2py_runtime_gae: if request.folder not in global_settings.app_folders: - for subfolder in ('models', 'views', 'controllers', 'databases', - 'modules', 'cron', 'errors', 'sessions', - 'languages', 'static', 'private', 'uploads'): - try_mkdir(os.path.join(request.folder, subfolder)) + for amf in missing_app_folders(request.folder): + try_mkdir(amf) global_settings.app_folders.add(request.folder) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index e7c2b7e8..7d190e30 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -14,93 +13,43 @@ Note: """ import re -import fnmatch -import os, sys -import copy import random -from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, integer_types, basestring, reduce, xrange, long, reload -from gluon.storage import Storage, List -from gluon.template import parse_template -from gluon.restricted import restricted, compile2 -from gluon.fileutils import mktree, listdir, read_file, write_file -from gluon.myregex import regex_expose, regex_longcomments -from gluon.languages import TranslatorFactory -from gluon.dal import DAL, Field -from gluon.validators import Validator -from pydal.base import BaseAdapter -from gluon.sqlhtml import SQLFORM, SQLTABLE -from gluon.cache import Cache -from gluon.globals import current, Response -from gluon import settings -from gluon.cfs import getcfs -from gluon import html -from gluon import validators -from gluon.http import HTTP, redirect -import marshal -import shutil -import imp -import logging import types +import copy from functools import reduce -from gluon import rewrite -from gluon.custom_import import custom_import_install import py_compile +import os +from os.path import join as pjoin, exists +import sys +import imp +import marshal +import fnmatch +import shutil -logger = logging.getLogger("web2py") +from gluon._compat import (builtin, PY2, unicodeT, to_native, to_bytes, + iteritems, integer_types, basestring, xrange, reload) +from gluon.storage import Storage, List +from gluon.globals import current, Response +from gluon import html, validators, rewrite +from gluon.http import HTTP, redirect +from gluon.dal import DAL, Field +from gluon.sqlhtml import SQLFORM, SQLTABLE +from gluon.languages import TranslatorFactory +from gluon.cache import Cache +from gluon.validators import Validator +from gluon.settings import global_settings +from pydal.base import BaseAdapter +from gluon.custom_import import custom_import_install +from gluon.fileutils import mktree, listdir, read_file, write_file, abspath +from gluon.template import parse_template +from gluon.cfs import getcfs +from gluon.restricted import restricted, compile2 +from gluon.admin import add_path_first -is_pypy = settings.global_settings.is_pypy -is_gae = settings.global_settings.web2py_runtime_gae -is_jython = settings.global_settings.is_jython -pjoin = os.path.join - -if PY2: - marshal_header_size = 8 -else: - marshal_header_size = 16 if sys.version_info[1] >= 7 else 12 - -TEST_CODE = \ - r""" -def _TEST(): - import doctest, sys, cStringIO, types, cgi, gluon.fileutils - if not gluon.fileutils.check_credentials(request): - raise HTTP(401, web2py_error='invalid credentials') - stdout = sys.stdout - html = '

Testing controller "%s.py" ... done.


\n' \ - % request.controller - for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]): - eval_key = eval(key) - if type(eval_key) == types.FunctionType: - number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)]) - if number_doctests>0: - sys.stdout = cStringIO.StringIO() - name = '%s/controllers/%s.py in %s.__doc__' \ - % (request.folder, request.controller, key) - doctest.run_docstring_examples(eval_key, - globals(), False, name=name) - report = sys.stdout.getvalue().strip() - if report: - pf = 'failed' - else: - pf = 'passed' - html += '

Function %s [%s]

\n' \ - % (pf, key, pf) - if report: - html += CODE(report, language='web2py', \ - link='/examples/global/vars/').xml() - html += '
\n' - else: - html += \ - '

Function %s [no doctests]


\n' \ - % (key) - response._vars = html - sys.stdout = stdout -_TEST() -""" CACHED_REGEXES = {} CACHED_REGEXES_MAX_SIZE = 1000 - def re_compile(regex): try: return CACHED_REGEXES[regex] @@ -111,22 +60,6 @@ def re_compile(regex): return compiled_regex -class mybuiltin(object): - """ - NOTE could simple use a dict and populate it, - NOTE not sure if this changes things though if monkey patching import..... - """ - # __builtins__ - def __getitem__(self, key): - try: - return getattr(builtin, key) - except AttributeError: - raise KeyError(key) - - def __setitem__(self, key, value): - setattr(self, key, value) - - def LOAD(c=None, f='index', args=None, vars=None, extension=None, target=None, ajax=False, ajax_trap=False, url=None, user_signature=False, timeout=None, times=1, @@ -140,10 +73,10 @@ def LOAD(c=None, f='index', args=None, vars=None, vars(dict): vars extension(str): extension target(str): id of the target - ajax(bool): True to enable AJAX bahaviour + ajax(bool): True to enable AJAX behaviour ajax_trap(bool): True if `ajax` is set to `True`, traps both links and forms "inside" the target - url(str): overrides `c`,`f`,`args` and `vars` + url(str): overrides `c`, `f`, `args` and `vars` user_signature(bool): adds hmac signature to all links with a key that is different for every user timeout(int): in milliseconds, specifies the time to wait before @@ -153,7 +86,6 @@ def LOAD(c=None, f='index', args=None, vars=None, "infinity" or "continuous" are accepted to reload indefinitely the component """ - from gluon.html import TAG, DIV, URL, SCRIPT, XML if args is None: args = [] vars = Storage(vars or {}) @@ -163,19 +95,21 @@ def LOAD(c=None, f='index', args=None, vars=None, if '.' in f: f, extension = f.rsplit('.', 1) if url or ajax: - url = url or URL(request.application, c, f, r=request, - args=args, vars=vars, extension=extension, - user_signature=user_signature) + url = url or html.URL(request.application, c, f, r=request, + args=args, vars=vars, extension=extension, + user_signature=user_signature) # timing options if isinstance(times, basestring): if times.upper() in ("INFINITY", "CONTINUOUS"): times = "Infinity" else: + # FIXME: should be a ValueError raise TypeError("Unsupported times argument %s" % times) elif isinstance(times, int): if times <= 0: raise ValueError("Times argument must be greater than zero, 'Infinity' or None") else: + # NOTE: why do not use ValueError only? raise TypeError("Unsupported times argument type %s" % type(times)) if timeout is not None: if not isinstance(timeout, integer_types): @@ -191,7 +125,7 @@ def LOAD(c=None, f='index', args=None, vars=None, statement = "$.web2py.component('%s','%s');" % (url, target) attr['_data-w2p_remote'] = url if target is not None: - return DIV(content, **attr) + return html.DIV(content, **attr) else: if not isinstance(args, (list, tuple)): @@ -211,7 +145,7 @@ def LOAD(c=None, f='index', args=None, vars=None, '/'.join([request.application, c, f] + [str(a) for a in other_request.args]) other_request.env.query_string = \ - vars and URL(vars=vars).split('?')[1] or '' + vars and html.URL(vars=vars).split('?')[1] or '' other_request.env.http_web2py_component_location = \ request.env.path_info other_request.cid = target @@ -220,7 +154,7 @@ def LOAD(c=None, f='index', args=None, vars=None, # A bit nasty but needed to use LOAD on action decorates with @request.restful() other_response.view = '%s/%s.%s' % (c, f, other_request.extension) - other_environment = copy.copy(current.globalenv) # NASTY + other_environment = copy.copy(current.globalenv) # FIXME: NASTY other_response._view_environment = other_environment other_response.generic_patterns = \ @@ -241,12 +175,12 @@ def LOAD(c=None, f='index', args=None, vars=None, current.request, current.response = original_request, original_response js = None if ajax_trap: - link = URL(request.application, c, f, r=request, - args=args, vars=vars, extension=extension, - user_signature=user_signature) + link = html.URL(request.application, c, f, r=request, + args=args, vars=vars, extension=extension, + user_signature=user_signature) js = "$.web2py.trap_form('%s','%s');" % (link, target) - script = js and SCRIPT(js, _type="text/javascript") or '' - return TAG[''](DIV(XML(page), **attr), script) + script = js and html.SCRIPT(js, _type="text/javascript") or '' + return html.TAG[''](html.DIV(html.XML(page), **attr), script) class LoadFactory(object): @@ -262,7 +196,6 @@ class LoadFactory(object): if args is None: args = [] vars = Storage(vars or {}) - from . import globals target = target or 'c' + str(random.random())[2:] attr['_id'] = target request = current.request @@ -289,7 +222,7 @@ class LoadFactory(object): other_request.vars = vars other_request.get_vars = vars other_request.post_vars = Storage() - other_response = globals.Response() + other_response = Response() other_request.env.path_info = '/' + \ '/'.join([request.application, c, f] + [str(a) for a in other_request.args]) @@ -354,37 +287,36 @@ def local_import_aux(name, reload_force=False, app='welcome'): if reload_force: reload(module) return module +# +# OLD IMPLEMENTATION: +# +# items = name.replace('/','.').split('.') +# filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1]) +# imp.acquire_lock() +# try: +# file=None +# (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path) +# if not path in sys.modules or reload: +# if is_gae: +# module={} +# execfile(path,{},module) +# module=Storage(module) +# else: +# module = imp.load_module(path,file,path,desc) +# sys.modules[path] = module +# else: +# module = sys.modules[path] +# except Exception, e: +# module = None +# if file: +# file.close() +# imp.release_lock() +# if not module: +# raise ImportError, "cannot find module %s in %s" % ( +# filename, modulepath) +# return module -""" -OLD IMPLEMENTATION: - items = name.replace('/','.').split('.') - filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1]) - imp.acquire_lock() - try: - file=None - (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path) - if not path in sys.modules or reload: - if is_gae: - module={} - execfile(path,{},module) - module=Storage(module) - else: - module = imp.load_module(path,file,path,desc) - sys.modules[path] = module - else: - module = sys.modules[path] - except Exception, e: - module = None - if file: - file.close() - imp.release_lock() - if not module: - raise ImportError, "cannot find module %s in %s" % ( - filename, modulepath) - return module -""" - _base_environment_ = dict((k, getattr(html, k)) for k in html.__all__) _base_environment_.update( (k, getattr(validators, k)) for k in validators.__all__) @@ -424,7 +356,7 @@ def build_environment(request, response, session, store_current=True): r'^%s/%s/\w+\.py$' % (request.controller, request.function) ] - T = environment['T'] = TranslatorFactory(os.path.join(request.folder, 'languages'), + T = environment['T'] = TranslatorFactory(pjoin(request.folder, 'languages'), request.env.http_accept_language) c = environment['cache'] = Cache(request) @@ -439,7 +371,23 @@ def build_environment(request, response, session, store_current=True): current.T = T current.cache = c - if is_jython: # jython hack + if global_settings.is_jython: + # jython hack + class mybuiltin(object): + """ + NOTE could simple use a dict and populate it, + NOTE not sure if this changes things though if monkey patching import..... + """ + # __builtins__ + def __getitem__(self, key): + try: + return getattr(builtin, key) + except AttributeError: + raise KeyError(key) + + def __setitem__(self, key, value): + setattr(self, key, value) + global __builtins__ __builtins__ = mybuiltin() @@ -462,6 +410,11 @@ def save_pyc(filename): py_compile.compile(filename, cfile=cfile) +if PY2: + MARSHAL_HEADER_SIZE = 8 +else: + MARSHAL_HEADER_SIZE = 16 if sys.version_info[1] >= 7 else 12 + def read_pyc(filename): """ Read the code inside a bytecode compiled file if the MAGIC number is @@ -471,44 +424,49 @@ def read_pyc(filename): a code object """ data = read_file(filename, 'rb') - if not is_gae and data[:4] != imp.get_magic(): + if not global_settings.web2py_runtime_gae and \ + not data.startswith(imp.get_magic()): raise SystemError('compiled code is incompatible') - return marshal.loads(data[marshal_header_size:]) + return marshal.loads(data[MARSHAL_HEADER_SIZE:]) +REGEX_PATH_CLASS = r"[\w%s-]" % os.sep if os.sep != '\\' else r'[\w\\-]' +REGEX_VIEW_PATH = r"%s+(?:\.\w+)*$" % REGEX_PATH_CLASS + def compile_views(folder, skip_failed_views=False): """ Compiles all the views in the application specified by `folder` """ - path = pjoin(folder, 'views') failed_views = [] - for fname in listdir(path, '^[\w/\-]+(\.\w+)*$'): + for fname in listdir(path, REGEX_VIEW_PATH): try: data = parse_template(fname, path) except Exception as e: if skip_failed_views: failed_views.append(fname) else: + # FIXME: should raise something more specific than Exception raise Exception("%s in %s" % (e, fname)) else: - filename = 'views.%s.py' % fname.replace(os.path.sep, '.') + filename = 'views.%s.py' % fname.replace(os.sep, '.') filename = pjoin(folder, 'compiled', filename) write_file(filename, data) save_pyc(filename) os.unlink(filename) - return failed_views if failed_views else None + return failed_views or None +REGEX_MODEL_PATH = r"%s+\.py$" % REGEX_PATH_CLASS + def compile_models(folder): """ Compiles all the models in the application specified by `folder` """ - path = pjoin(folder, 'models') - for fname in listdir(path, '.+\.py$'): + for fname in listdir(path, REGEX_MODEL_PATH): data = read_file(pjoin(path, fname)) - modelfile = 'models.'+fname.replace(os.path.sep, '.') + modelfile = 'models.'+fname.replace(os.sep, '.') filename = pjoin(folder, 'compiled', modelfile) mktree(filename) write_file(filename, data) @@ -516,19 +474,22 @@ def compile_models(folder): os.unlink(filename) -def find_exposed_functions(data): - data = regex_longcomments.sub('', data) - return regex_expose.findall(data) +REGEX_LONG_STRING = re.compile('(""".*?"""|' "'''.*?''')", re.DOTALL) +REGEX_EXPOSED = re.compile(r'^def\s+(_?[a-zA-Z0-9]\w*)\( *\)\s*:', re.MULTILINE) +def find_exposed_functions(data): + data = REGEX_LONG_STRING.sub('', data) + return REGEX_EXPOSED.findall(data) + + +REGEX_CONTROLLER = r'[\w-]+\.py$' def compile_controllers(folder): """ Compiles all the controllers in the application specified by `folder` """ - path = pjoin(folder, 'controllers') - for fname in listdir(path, '.+\.py$'): - ### why is this here? save_pyc(pjoin(path, file)) + for fname in listdir(path, REGEX_CONTROLLER): data = read_file(pjoin(path, fname)) exposed = find_exposed_functions(data) for function in exposed: @@ -541,20 +502,22 @@ def compile_controllers(folder): os.unlink(filename) -def model_cmp(a, b, sep='.'): - return cmp(a.count(sep), b.count(sep)) or cmp(a, b) +if PY2: + def model_cmp(a, b, sep='.'): + return cmp(a.count(sep), b.count(sep)) or cmp(a, b) + + def model_cmp_sep(a, b, sep=os.sep): + return model_cmp(a, b, sep) -def model_cmp_sep(a, b, sep=os.path.sep): - return model_cmp(a, b, sep) - +REGEX_COMPILED_MODEL = r'models[_.][\w.-]+\.pyc$' +REGEX_MODEL = r'[\w-]+\.py$' def run_models_in(environment): """ Runs all models (in the app specified by the current folder) It tries pre-compiled models first before compiling them. """ - request = current.request folder = request.folder c = request.controller @@ -563,19 +526,21 @@ def run_models_in(environment): path = pjoin(folder, 'models') cpath = pjoin(folder, 'compiled') - compiled = os.path.exists(cpath) + compiled = exists(cpath) if PY2: if compiled: - models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), model_cmp) + models = sorted(listdir(cpath, REGEX_COMPILED_MODEL, 0), + model_cmp) else: - models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep) + models = sorted(listdir(path, REGEX_MODEL, 0, sort=False), + model_cmp_sep) else: if compiled: - models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), + models = sorted(listdir(cpath, REGEX_COMPILED_MODEL, 0), key=lambda f: '{0:03d}'.format(f.count('.')) + f) else: - models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), - key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f) + models = sorted(listdir(path, REGEX_MODEL, 0, sort=False), + key=lambda f: '{0:03d}'.format(f.count(os.sep)) + f) models_to_run = None for model in models: @@ -589,7 +554,7 @@ def run_models_in(environment): fname = model[n:-4].replace('.', '/')+'.py' else: n = len(path)+1 - fname = model[n:].replace(os.path.sep, '/') + fname = model[n:].replace(os.sep, '/') if not regex.search(fname) and c != 'appadmin': continue elif compiled: @@ -600,19 +565,56 @@ def run_models_in(environment): restricted(ccode, environment, layer=model) +TEST_CODE = r""" +def _TEST(): + import doctest, sys, cStringIO, types, cgi, gluon.fileutils + if not gluon.fileutils.check_credentials(request): + raise HTTP(401, web2py_error='invalid credentials') + stdout = sys.stdout + html = '

Testing controller "%s.py" ... done.


\n' \ + % request.controller + for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]): + eval_key = eval(key) + if type(eval_key) == types.FunctionType: + number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)]) + if number_doctests>0: + sys.stdout = cStringIO.StringIO() + name = '%s/controllers/%s.py in %s.__doc__' \ + % (request.folder, request.controller, key) + doctest.run_docstring_examples(eval_key, + globals(), False, name=name) + report = sys.stdout.getvalue().strip() + if report: + pf = 'failed' + else: + pf = 'passed' + html += '

Function %s [%s]

\n' \ + % (pf, key, pf) + if report: + html += CODE(report, language='web2py', \ + link='/examples/global/vars/').xml() + html += '
\n' + else: + html += \ + '

Function %s [no doctests]


\n' \ + % (key) + response._vars = html + sys.stdout = stdout +_TEST() +""" + def run_controller_in(controller, function, environment): """ Runs the controller.function() (for the app specified by the current folder). It tries pre-compiled controller.function.pyc first before compiling it. """ - # if compiled should run compiled! folder = current.request.folder cpath = pjoin(folder, 'compiled') badc = 'invalid controller (%s/%s)' % (controller, function) badf = 'invalid function (%s/%s)' % (controller, function) - if os.path.exists(cpath): + if exists(cpath): filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function)) try: ccode = getcfs(filename, filename, lambda: read_pyc(filename)) @@ -622,8 +624,6 @@ def run_controller_in(controller, function, environment): web2py_error=badf) elif function == '_TEST': # TESTING: adjust the path to include site packages - from gluon.settings import global_settings - from gluon.admin import abspath, add_path_first paths = (global_settings.gluon_parent, abspath( 'site-packages', gluon=True), abspath('gluon', gluon=True), '') [add_path_first(path) for path in paths] @@ -631,7 +631,7 @@ def run_controller_in(controller, function, environment): filename = pjoin(folder, 'controllers/%s.py' % controller) - if not os.path.exists(filename): + if not exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badc, web2py_error=badc) @@ -696,12 +696,12 @@ def run_view_in(environment): layer = 'file stream' else: filename = pjoin(folder, 'views', view) - if os.path.exists(cpath): # compiled views + if exists(cpath): # compiled views x = view.replace('/', '.') files = ['views.%s.pyc' % x] - is_compiled = os.path.exists(pjoin(cpath, files[0])) + is_compiled = exists(pjoin(cpath, files[0])) # Don't use a generic view if the non-compiled view exists. - if is_compiled or (not is_compiled and not os.path.exists(filename)): + if is_compiled or (not is_compiled and not exists(filename)): if allow_generic: files.append('views.generic.%s.pyc' % request.extension) # for backward compatibility @@ -712,16 +712,16 @@ def run_view_in(environment): # end backward compatibility code for f in files: compiled = pjoin(cpath, f) - if os.path.exists(compiled): + if exists(compiled): ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled)) layer = compiled break # if the view is not compiled if not layer: - if not os.path.exists(filename) and allow_generic: + if not exists(filename) and allow_generic: view = 'generic.' + request.extension filename = pjoin(folder, 'views', view) - if not os.path.exists(filename): + if not exists(filename): raise HTTP(404, rewrite.THREAD_LOCAL.routes.error_message % badv, web2py_error=badv) @@ -737,6 +737,8 @@ def run_view_in(environment): return environment['response'].body.getvalue() +REGEX_COMPILED_CONTROLLER = r'[\w-]+\.pyc$' + def remove_compiled_application(folder): """ Deletes the folder `compiled` containing the compiled application. @@ -744,7 +746,7 @@ def remove_compiled_application(folder): try: shutil.rmtree(pjoin(folder, 'compiled')) path = pjoin(folder, 'controllers') - for file in listdir(path, '.*\.pyc$', drop=False): + for file in listdir(path, REGEX_COMPILED_CONTROLLER, drop=False): os.unlink(file) except OSError: pass diff --git a/gluon/console.py b/gluon/console.py new file mode 100644 index 00000000..e3d519a2 --- /dev/null +++ b/gluon/console.py @@ -0,0 +1,722 @@ +# -*- coding: utf-8 -*- +# vim: set ts=4 sw=4 et ai: +""" +| This file is part of the web2py Web Framework +| Copyrighted by Massimo Di Pierro +| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) + +Command line interface +---------------------- + +The processing of all command line arguments is done using +the argparse library in the console function. + +The basic principle is to process and check for all options +in a single place, this place is the parse_args function. +Notice that when I say all options I mean really all, +options sourced from a configuration file are included. + +A brief summary of options style follows, +for the benefit of code maintainers/developers: + +- use the underscore to split words in long names (as in + '--run_system_tests') +- remember to allow the '-' too as word separator (e.g. + '--run-system-tests') but do not use this form on help +- prefer short names on help messages, instead use + all options names in warning/error messages (e.g. + '-R/--run requires -S/--shell') +""" + +from __future__ import print_function + +__author__ = 'Paolo Pastori' + +import os.path +import argparse +import logging +import socket +import sys +import re +import ast +from collections import OrderedDict +import copy + +from gluon._compat import PY2 +from gluon.shell import die +from gluon.utils import is_valid_ip_address +from gluon.settings import global_settings + + +def warn(msg): + print("%s: warning: %s" % (sys.argv[0], msg), file=sys.stderr) + +def is_appdir(applications_parent, app): + return os.path.isdir(os.path.join(applications_parent, 'applications', app)) + + +def console(version): + """ + Load command line options. + Trivial -h/--help and --version options are also processed. + + Returns a namespace object (in the sense of argparse) + with all options loaded. + """ + + # replacement hints for deprecated options + deprecated_opts = { + '--debug': '--log_level', + '--nogui': '--no_gui', + '--ssl_private_key': '--server_key', + '--ssl_certificate': '--server_cert', + '--interfaces': None, # dest is 'interfaces', hint is '--interface' + '-n': '--min_threads', '--numthreads': '--min_threads', + '--minthreads': '--min_threads', + '--maxthreads': '--max_threads', + '-z': None, '--shutdown_timeout': None, + '--profiler': '--profiler_dir', + '--run-cron': '--with_cron', + '--softcron': '--soft_cron', + '--cron': '--cron_run', + '--test': '--run_doctests' + } + + class HelpFormatter2(argparse.HelpFormatter): + """Hides the options listed in _hidden_options in usage help.""" + + # NOTE: preferred style for long options name is to use '_' + # between words (as in 'no_gui'), also accept the '-' in + # most of the options but do not show both versions on help + _omitted_opts = ('--add-options', '--errors-to-console', + '--no-banner', '--log-level', '--no-gui', '--import-models', + '--server-name', '--server-key', '--server-cert', '--ca-cert', + '--pid-filename', '--log-filename', '--min-threads', + '--max-threads', '--request-queue-size', '--socket-timeout', + '--profiler-dir', '--with-scheduler', '--with-cron', + '--soft-cron', '--cron-run', + '--run-doctests', '--run-system-tests', '--with-coverage') + + _hidden_options = _omitted_opts + tuple(deprecated_opts.keys()) + + def _format_action_invocation(self, action): + if not action.option_strings: + return super(HelpFormatter2, self)._format_action_invocation(action) + parts = [] + if action.nargs == 0: + parts.extend(filter(lambda o : o not in self._hidden_options, + action.option_strings)) + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + for option_string in action.option_strings: + if option_string in self._hidden_options: + continue + parts.append('%s %s' % (option_string, args_string)) + return ', '.join(parts) + + class ExtendAction(argparse._AppendAction): + """Action to accumulate values in a flat list.""" + + def __call__(self, parser, namespace, values, option_string=None): + if isinstance(values, list): + # must copy to avoid altering the option default value + value = getattr(namespace, self.dest, None) + if value is None: + value = [] + setattr(namespace, self.dest, value) + items = value[:] + # for options that allows multiple args (i.e. those declared + # with add_argument(..., nargs='+', ...)) the values are + # always placed into a list + while len(values) == 1 and isinstance(values[0], list): + values = values[0] + items.extend(values) + setattr(namespace, self.dest, items) + else: + super(ExtendAction, self).__call__(parser, namespace, values, option_string) + + parser = argparse.ArgumentParser( + usage='python %(prog)s [options]', + description='web2py Web Framework startup script.', + epilog='''NOTE: unless a password is specified (-a 'passwd') +web2py will attempt to run a GUI to ask for it when starting the web server +(if not disabled with --no_gui).''', + formatter_class=HelpFormatter2, + add_help=False) # do not add -h/--help option + + # global options + g = parser.add_argument_group('global options') + g.add_argument('-h', '--help', action='help', + help='show this help message and exit') + g.add_argument('--version', action='version', + version=version, + help="show program's version and exit") + folder = os.getcwd() + g.add_argument('-f', '--folder', + default=folder, metavar='WEB2PY_DIR', + help='web2py installation directory (%(default)s)') + def existing_file(v): + if not v: + raise argparse.ArgumentTypeError('empty argument') + if not os.path.exists(v): + raise argparse.ArgumentTypeError("file %r not found" % v) + return v + g.add_argument('-L', '--config', + type=existing_file, + metavar='PYTHON_FILE', + help='read all options from PYTHON_FILE') + g.add_argument('--add_options', '--add-options', + default=False, + action='store_true', help= + 'add options to existing ones, useful with -L only') + g.add_argument('-a', '--password', + default='', help= + 'password to be used for administration (use "" ' + 'to reuse the last password), when no password is available ' + 'the administrative web interface will be disabled') + g.add_argument('-e', '--errors_to_console', '--errors-to-console', + default=False, + action='store_true', + help='log application errors to console') + g.add_argument('--no_banner', '--no-banner', + default=False, + action='store_true', + help='do not print header banner') + g.add_argument('-Q', '--quiet', + default=False, + action='store_true', + help='disable all output') + integer_log_level = [] + def log_level(v): + # try to convert a lgging level name to its numeric value, + # could use logging.getLevelName but not with + # 3.4 <= Python < 3.4.2, see + # https://docs.python.org/3/library/logging.html#logging.getLevelName) + try: + name2level = logging._levelNames + except AttributeError: + # logging._levelNames has gone with Python 3.4, see + # https://github.com/python/cpython/commit/3b84eae03ebd8122fdbdced3d85999dd9aedfc7e + name2level = logging._nameToLevel + try: + return name2level[v.upper()] + except KeyError: + pass + try: + ill = int(v) + # value deprecated: integer in range(101) + if 0 <= ill <= 100: + integer_log_level.append(ill) + return ill + except ValueError: + pass + raise argparse.ArgumentTypeError("bad level %r" % v) + g.add_argument('-D', '--log_level', '--log-level', + '--debug', # deprecated + default='WARNING', + type=log_level, + metavar='LOG_LEVEL', help= + 'set log level, allowed values are: NOTSET, DEBUG, INFO, WARN, ' + 'WARNING, ERROR, and CRITICAL, also lowercase (default is ' + '%(default)s)') + + # GUI options + g = parser.add_argument_group('GUI options') + g.add_argument('--no_gui', '--no-gui', + '--nogui', # deprecated + default=False, + action='store_true', + help='do not run GUI') + g.add_argument('-t', '--taskbar', + default=False, + action='store_true', + help='run in taskbar (system tray)') + + # console options + g = parser.add_argument_group('console options') + g.add_argument('-S', '--shell', + metavar='APP_ENV', help= + 'run web2py in Python interactive shell or IPython (if installed) ' + 'with specified application environment (if application does not ' + 'exist it will be created). APP_ENV like a/c/f?x=y (c, f and vars ' + 'optional), if APP_ENV include the action f then after the ' + 'action execution the interpreter is exited') + g.add_argument('-B', '--bpython', + default=False, + action='store_true', help= + 'use bpython (if installed) when running in interactive shell, ' + 'see -S above') + g.add_argument('-P', '--plain', + default=False, + action='store_true', help= + 'use plain Python shell when running in interactive shell, ' + 'see -S above') + g.add_argument('-M', '--import_models', '--import-models', + default=False, + action='store_true', help= + 'auto import model files when running in interactive shell ' + '(default is %(default)s), see -S above. NOTE: when the APP_ENV ' + 'argument of -S include a controller c automatic import of ' + 'models is always enabled') + g.add_argument('--force_migrate', + default=False, + action='store_true', + help= + 'force DAL to migrate all tables that should be migrated when enabled; ' + 'monkeypatch in the DAL class to force _migrate_enabled=True') + g.add_argument('-R', '--run', + type=existing_file, + metavar='PYTHON_FILE', help= + 'run PYTHON_FILE in web2py environment; require -S') + g.add_argument('-A', '--args', + default=[], + nargs=argparse.REMAINDER, help= + 'use this to pass arguments to the PYTHON_FILE above; require ' + '-R. NOTE: must be the last option because eat all remaining ' + 'arguments') + + # web server options + g = parser.add_argument_group('web server options') + g.add_argument('-s', '--server_name', '--server-name', + default=socket.gethostname(), + help='web server name (%(default)s)') + def ip_addr(v): + if not is_valid_ip_address(v): + raise argparse.ArgumentTypeError("bad IP address %s" % v) + return v + g.add_argument('-i', '--ip', + default='127.0.0.1', + type=ip_addr, metavar='IP_ADDR', help= + 'IP address of the server (%(default)s), accept either IPv4 or ' + 'IPv6 (e.g. ::1) addresses. NOTE: this option is ignored if ' + '--interface is specified') + def not_negative_int(v, err_label='value'): + try: + iv = int(v) + if iv < 0: raise ValueError() + return iv + except ValueError: + pass + raise argparse.ArgumentTypeError("bad %s %s" % (err_label, v)) + def port(v): + return not_negative_int(v, err_label='port') + g.add_argument('-p', '--port', + default=8000, + type=port, metavar='NUM', help= + 'port of server (%(default)d). ' + 'NOTE: this option is ignored if --interface is specified') + g.add_argument('-k', '--server_key', '--server-key', + '--ssl_private_key', # deprecated + type=existing_file, + metavar='FILE', help='server private key') + g.add_argument('-c', '--server_cert', '--server-cert', + '--ssl_certificate', # deprecated + type=existing_file, + metavar='FILE', help='server certificate') + g.add_argument('--ca_cert', '--ca-cert', + type=existing_file, + metavar='FILE', help='CA certificate') + def iface(v, sep=','): + if not v: + raise argparse.ArgumentTypeError('empty argument') + if sep == ':': + # deprecated --interfaces ip:port:key:cert:ca_cert + # IPv6 addresses in square brackets + if v.startswith('['): + # IPv6 + ip, v_remainder = v.split(']', 1) + ip = ip[1:] + ifp = v_remainder[1:].split(':') + ifp.insert(0, ip) + else: + # IPv4 + ifp = v.split(':') + else: + # --interface + ifp = v.split(sep, 5) + if not len(ifp) in (2, 4, 5): + raise argparse.ArgumentTypeError("bad interface %r" % v) + try: + ip_addr(ifp[0]) + ifp[1] = port(ifp[1]) + for fv in ifp[2:]: + existing_file(fv) + except argparse.ArgumentTypeError as ex: + raise argparse.ArgumentTypeError("bad interface %r (%s)" % (v, ex)) + return tuple(ifp) + g.add_argument('--interface', dest='interfaces', + default=[], action=ExtendAction, + type=iface, nargs='+', + metavar='IF_INFO', help= + 'listen on specified interface, IF_INFO = ' + 'IP_ADDR,PORT[,KEY_FILE,CERT_FILE[,CA_CERT_FILE]].' + ' NOTE: this option can be used multiple times to provide additional ' + 'interfaces to choose from but you can choose which one to listen to ' + 'only using the GUI otherwise the first interface specified is used') + def ifaces(v): + # deprecated --interfaces 'if1;if2;...' + if not v: + raise argparse.ArgumentTypeError('empty argument') + return [iface(i, ':') for i in v.split(';')] + g.add_argument('--interfaces', # deprecated + default=argparse.SUPPRESS, # do not set if absent + action=ExtendAction, + type=ifaces, + help=argparse.SUPPRESS) # do not show on help + g.add_argument('-d', '--pid_filename', '--pid-filename', + default='httpserver.pid', + metavar='FILE', help='server pid file (%(default)s)') + g.add_argument('-l', '--log_filename', '--log-filename', + default='httpserver.log', + metavar='FILE', help='server log file (%(default)s)') + g.add_argument('--min_threads', '--min-threads', + '--minthreads', '-n', '--numthreads', # deprecated + type=not_negative_int, metavar='NUM', + help='minimum number of server threads') + g.add_argument('--max_threads', '--max-threads', + '--maxthreads', # deprecated + type=not_negative_int, metavar='NUM', + help='maximum number of server threads') + g.add_argument('-q', '--request_queue_size', '--request-queue-size', + default=5, + type=not_negative_int, metavar='NUM', help= + 'max number of queued requests when server busy (%(default)d)') + g.add_argument('-o', '--timeout', + default=10, + type=not_negative_int, metavar='SECONDS', + help='timeout for individual request (%(default)d seconds)') + g.add_argument('--socket_timeout', '--socket-timeout', + default=5, + type=not_negative_int, metavar='SECONDS', + help='timeout for socket (%(default)d seconds)') + g.add_argument('-z', '--shutdown_timeout', # deprecated + type=not_negative_int, + help=argparse.SUPPRESS) # do not show on help + g.add_argument('-F', '--profiler_dir', '--profiler-dir', + '--profiler', # deprecated + help='profiler directory') + + # scheduler options + g = parser.add_argument_group('scheduler options') + g.add_argument('-X', '--with_scheduler', '--with-scheduler', + default=False, + action='store_true', help= + 'run schedulers alongside web server; require --K') + def is_app(app): + return is_appdir(folder, app) + def scheduler(v): + if not v: + raise argparse.ArgumentTypeError('empty argument') + if ',' in v: + # legacy "app1,..." + vl = [n.strip() for n in v.split(',')] + return [scheduler(iv) for iv in vl] + vp = [n.strip() for n in v.split(':')] + app = vp[0] + if not app: + raise argparse.ArgumentTypeError('empty application') + if not is_app(app): + warn("argument -K/--scheduler: bad application %r, skipped" % app) + return None + return ':'.join(filter(None, vp)) + g.add_argument('-K', '--scheduler', dest='schedulers', + default=[], action=ExtendAction, + type=scheduler, nargs='+', + metavar='APP_INFO', help= + 'run scheduler for the specified application(s), APP_INFO = ' + 'APP_NAME[:GROUPS], that is an optional list of groups can follow ' + 'the application name (e.g. app:group1:group2); require a scheduler ' + "to be defined in the application's models. NOTE: this option can " + 'be used multiple times to add schedulers') + + # cron options + g = parser.add_argument_group('cron options') + g.add_argument('-Y', '--with_cron', '--with-cron', + '--run-cron', # deprecated + default=False, + action='store_true', help= + 'run cron service alongside web server') + def crontab(v): + if not v: + raise argparse.ArgumentTypeError('empty argument') + if not is_app(v): + warn("argument --crontab: bad application %r, skipped" % v) + return None + return v + g.add_argument('--crontab', dest='crontabs', + default=[], action=ExtendAction, + type=crontab, nargs='+', + metavar='APP_NAME', help= + 'tell cron to read the crontab for the specified application(s) ' + 'only, the default behaviour is to read the crontab for all of the ' + 'installed applications. NOTE: this option can be used multiple ' + 'times to build the list of crontabs to be processed by cron') + g.add_argument('--soft_cron', '--soft-cron', + '--softcron', # deprecated + default=False, + action='store_true', help= + 'use cron software emulation instead of separate cron process; ' + 'require -Y. NOTE: use of cron software emulation is strongly ' + 'discouraged') + g.add_argument('-C', '--cron_run', '--cron-run', + '--cron', # deprecated + default=False, + action='store_true', help= + 'trigger a cron run and exit; usually used when invoked ' + 'from a system (external) crontab') + g.add_argument('--cron_job', # NOTE: this is intended for internal use only + default=False, + action='store_true', + help=argparse.SUPPRESS) # do not show on help + + # test options + g = parser.add_argument_group('test options') + g.add_argument('-v', '--verbose', + default=False, + action='store_true', help='increase verbosity') + g.add_argument('-T', '--run_doctests', '--run-doctests', + '--test', # deprecated + metavar='APP_ENV', help= + 'run doctests in application environment. APP_ENV like a/c/f (c, f ' + 'optional)') + g.add_argument('--run_system_tests', '--run-system-tests', + default=False, + action='store_true', help='run web2py test suite') + g.add_argument('--with_coverage', '--with-coverage', + default=False, + action='store_true', help= + 'collect coverage data when used with --run_system_tests; ' + 'require Python 2.7+ and the coverage module installed') + + # other options + g = parser.add_argument_group('other options') + g.add_argument('-G', '--GAE', dest='gae', + metavar='APP_NAME', help= + 'will create app.yaml and gaehandler.py and exit') + + options = parse_args(parser, sys.argv[1:], + deprecated_opts, integer_log_level) + + # make a copy of all options for global_settings + copy_options = copy.deepcopy(options) + copy_options.password = '******' + global_settings.cmd_options = copy_options + + return options + + +REGEX_PEP263 = r'^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)' + +def get_pep263_encoding(source): + """ + Read python source file encoding, according to PEP 263, see + https://www.python.org/dev/peps/pep-0263/ + """ + with open(source, 'r') as sf: + l12 = (sf.readline(), sf.readline()) + m12 = re.match(REGEX_PEP263, l12[0]) or re.match(REGEX_PEP263, l12[1]) + return m12 and m12.group(1) + + +IGNORE = lambda: None + +def load_config(config_file, opt_map): + """ + Load options from config file (a Python script). + + config_file(str): file name + opt_map(dict): mapping fom option name (key) to callable (val), + used to post-process parsed value for the option + + Notice that the configuring Python script is never executed/imported, + instead the ast library is used to evaluate each option assignment, + provided that it is written on a single line. + + Returns an OrderedDict with sourced options. + """ + REGEX_ASSIGN_EXP = re.compile(r'\s*=\s*(.+)') + map_items = opt_map.items() + # preserve the order of loaded options even though this is not needed + pl = OrderedDict() + config_encoding = get_pep263_encoding(config_file) + # NOTE: assume 'ascii' encoding when not explicitly stated (Python 2), + # this is not correct for Python 3 where the default is 'utf-8' + open_kwargs = dict() if PY2 else dict(encoding=config_encoding or 'ascii') + with open(config_file, 'r', **open_kwargs) as cfil: + for linenum, clin in enumerate(cfil, start=1): + if PY2 and config_encoding: + clin = unicode(clin, config_encoding) + clin = clin.strip() + for opt, mapr in map_items: + if clin.startswith(opt): + m = REGEX_ASSIGN_EXP.match(clin[len(opt):]) + if m is None: continue + try: + val = opt_map[opt](ast.literal_eval(m.group(1))) + except: + die("cannot parse config file %r at line %d" % (config_file, linenum)) + if val is not IGNORE: + pl[opt] = val + return pl + + +def parse_args(parser, cli_args, deprecated_opts, integer_log_level, + namespace=None): + + #print('PARSING ARGS:', cli_args) + del integer_log_level[:] + options = parser.parse_args(cli_args, namespace) + #print('PARSED OPTIONS:', options) + + # warn for deprecated options + deprecated_args = [a for a in cli_args if a in deprecated_opts] + for da in deprecated_args: + # verify if it was a real option by looking into + # parsed values for the actual destination + hint = deprecated_opts[da] + dest = (hint or da).lstrip('-') + default = parser.get_default(dest) + if da == '--interfaces': + hint = '--interface' + if getattr(options, dest) is not default: + # the option has been specified + msg = "%s is deprecated" % da + if hint: + msg += ", use %s instead" % hint + warn(msg) + # warn for deprecated values + if integer_log_level and '--debug' not in deprecated_args: + warn('integer argument for -D/--log_level is deprecated, ' + 'use label instead') + # fix schedulers and die if all were skipped + if None in options.schedulers: + options.schedulers = [i for i in options.schedulers if i is not None] + if not options.schedulers: + die('no scheduler left') + # fix crontabs and die if all were skipped + if None in options.crontabs: + options.crontabs = [i for i in options.crontabs if i is not None] + if not options.crontabs: + die('no crontab left') + # taskbar + if options.taskbar and os.name != 'nt': + warn('--taskbar not supported on this platform, skipped') + options.taskbar = False + # options consistency checkings + if options.run and not options.shell: + die('-R/--run requires -S/--shell', exit_status=2) + if options.args and not options.run: + die('-A/--args requires -R/--run', exit_status=2) + if options.with_scheduler and not options.schedulers: + die('-X/--with_scheduler requires -K/--scheduler', exit_status=2) + if options.soft_cron and not options.with_cron: + die('--soft_cron requires -Y/--with_cron', exit_status=2) + if options.shell: + for o, os in dict(with_scheduler='-X/--with_scheduler', + schedulers='-K/--scheduler', + with_cron='-Y/--with_cron', + cron_run='-C/--cron_run', + run_doctests='-T/--run_doctests', + run_system_tests='--run_system_tests').items(): + if getattr(options, o): + die("-S/--shell and %s are conflicting options" % os, + exit_status=2) + if options.bpython and options.plain: + die('-B/--bpython and -P/--plain are conflicting options', + exit_status=2) + if options.cron_run: + for o, os in dict(with_cron='-Y/--with_cron', + run_doctests='-T/--run_doctests', + run_system_tests='--run_system_tests').items(): + if getattr(options, o): + die("-C/--cron_run and %s are conflicting options" % os, + exit_status=2) + if options.run_doctests and options.run_system_tests: + die('-T/--run_doctests and --run_system_tests are conflicting options', + exit_status=2) + + if options.config: + # load options from file, + # all options sourced from file that evaluates to False + # are skipped, the special IGNORE value is used for this + store_true = lambda v: True if v else IGNORE + str_or_default = lambda v : str(v) if v else IGNORE + list_or_default = lambda v : ( + [str(i) for i in v] if isinstance(v, list) else [str(v)]) if v \ + else IGNORE + # NOTE: 'help', 'version', 'folder', 'cron_job' and 'GAE' are not + # sourced from file, the same applies to deprecated options + opt_map = { + # global options + 'config': str_or_default, + 'add_options': store_true, + 'password': str_or_default, + 'errors_to_console': store_true, + 'no_banner': store_true, + 'quiet': store_true, + 'log_level': str_or_default, + # GUI options + 'no_gui': store_true, + 'taskbar': store_true, + # console options + 'shell': str_or_default, + 'bpython': store_true, + 'plain': store_true, + 'import_models': store_true, + 'run': str_or_default, + 'args': list_or_default, + # web server options + 'server_name': str_or_default, + 'ip': str_or_default, + 'port': str_or_default, + 'server_key': str_or_default, + 'server_cert': str_or_default, + 'ca_cert': str_or_default, + 'interface': list_or_default, + 'pid_filename': str_or_default, + 'log_filename': str_or_default, + 'min_threads': str_or_default, + 'max_threads': str_or_default, + 'request_queue_size': str_or_default, + 'timeout': str_or_default, + 'socket_timeout': str_or_default, + 'profiler_dir': str_or_default, + # scheduler options + 'with_scheduler': store_true, + 'scheduler': list_or_default, + # cron options + 'with_cron': store_true, + 'crontab': list_or_default, + 'soft_cron': store_true, + 'cron_run': store_true, + # test options + 'verbose': store_true, + 'run_doctests': str_or_default, + 'run_system_tests': store_true, + 'with_coverage': store_true, + } + od = load_config(options.config, opt_map) + #print("LOADED FROM %s:" % options.config, od) + # convert loaded options dict as retuned by load_config + # into a list of arguments for further parsing by parse_args + file_args = []; args_args = [] # '--args' must be the last + for key, val in od.items(): + if key != 'args': + file_args.append('--' + key) + if isinstance(val, list): file_args.extend(val) + elif not isinstance(val, bool): file_args.append(val) + else: + args_args = ['--args'] + val + file_args += args_args + + if options.add_options: + # add options to existing ones, + # must clear config to avoid infinite recursion + options.config = options.add_options = None + return parse_args(parser, file_args, + deprecated_opts, integer_log_level, options) + return parse_args(parser, file_args, + deprecated_opts, integer_log_level) + + return options diff --git a/gluon/fileutils.py b/gluon/fileutils.py index 9424d32e..3c805ca4 100644 --- a/gluon/fileutils.py +++ b/gluon/fileutils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -19,10 +18,12 @@ import time import datetime import logging import shutil + from gluon.http import HTTP from gzip import open as gzopen from gluon.recfile import generate from gluon._compat import PY2 +from gluon.settings import global_settings __all__ = ( 'parse_version', @@ -255,12 +256,23 @@ def w2p_pack(filename, path, compiled=False, filenames=None): os.unlink(tarname) +def missing_app_folders(path): + for subfolder in ('models', 'views', 'controllers', 'databases', + 'modules', 'cron', 'errors', 'sessions', + 'languages', 'static', 'private', 'uploads'): + yield os.path.join(path, subfolder) + + def create_welcome_w2p(): is_newinstall = os.path.exists('NEWINSTALL') if not os.path.exists('welcome.w2p') or is_newinstall: logger = logging.getLogger("web2py") try: - w2p_pack('welcome.w2p', 'applications/welcome') + app_path = 'applications/welcome' + for amf in missing_app_folders(app_path): + if not os.path.exists(amf): + os.mkdir(amf) + w2p_pack('welcome.w2p', app_path) logger.info("New installation: created welcome.w2p file") except: logger.exception("New installation error: unable to create welcome.w2p file") @@ -295,12 +307,6 @@ def w2p_unpack(filename, path, delete_tar=True): def create_app(path): w2p_unpack('welcome.w2p', path) - for subfolder in ('models', 'views', 'controllers', 'databases', - 'modules', 'cron', 'errors', 'sessions', - 'languages', 'static', 'private', 'uploads'): - subpath = os.path.join(path, subfolder) - if not os.path.exists(subpath): - os.mkdir(subpath) def w2p_pack_plugin(filename, path, plugin_name): @@ -374,7 +380,7 @@ def get_session(request, other_application='admin'): if not os.path.exists(session_filename): session_filename = generate(session_filename) osession = storage.load_storage(session_filename) - except: + except Exception: osession = storage.Storage() return osession @@ -424,45 +430,14 @@ def fix_newlines(path): write_file(filename, wdata, 'w') -def copystream( - src, - dest, - size, - chunk_size=10 ** 5, -): - """ - this is here because I think there is a bug in shutil.copyfileobj - """ - while size > 0: - if size < chunk_size: - data = src.read(size) - else: - data = src.read(chunk_size) - length = len(data) - if length > size: - (data, length) = (data[:size], size) - size -= length - if length == 0: - break - dest.write(data) - if length < chunk_size: - break - dest.seek(0) - return - - -from gluon.settings import global_settings # we need to import settings here because - # settings imports fileutils too - - -def abspath(*relpath, **base): +# NOTE: same name as os.path.abspath (but signature is different) +def abspath(*relpath, **kwargs): """Converts relative path to absolute path based (by default) on applications_parent """ path = os.path.join(*relpath) - gluon = base.get('gluon', False) if os.path.isabs(path): return path - if gluon: + if kwargs.get('gluon', False): return os.path.join(global_settings.gluon_parent, path) return os.path.join(global_settings.applications_parent, path) diff --git a/gluon/globals.py b/gluon/globals.py index fc14ceff..a3b95d31 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -27,7 +26,6 @@ from gluon.utils import web2py_uuid, secure_dumps, secure_loads from gluon.settings import global_settings from gluon import recfile from gluon.cache import CacheInRam -from gluon.fileutils import copystream import hashlib from pydal.contrib import portalocker from pickle import Pickler, MARK, DICT, EMPTY_DICT @@ -103,6 +101,26 @@ def sorting_dumps(obj, protocol=None): # END ##################################################################### +def copystream(src, dest, size, chunk_size, cache_inc=None): + while size > 0: + if size < chunk_size: + data = src.read(size) + callable(cache_inc) and cache_inc(size) + else: + data = src.read(chunk_size) + callable(cache_inc) and cache_inc(chunk_size) + length = len(data) + if length > size: + (data, length) = (data[:size], size) + size -= length + if length == 0: + break + dest.write(data) + if length < chunk_size: + break + dest.seek(0) + return + def copystream_progress(request, chunk_size=10 ** 5): """ Copies request.env.wsgi_input into request.body @@ -128,23 +146,8 @@ def copystream_progress(request, chunk_size=10 ** 5): cache_ram = CacheInRam(request) # same as cache.ram because meta_storage cache_ram(cache_key + ':length', lambda: size, 0) cache_ram(cache_key + ':uploaded', lambda: 0, 0) - while size > 0: - if size < chunk_size: - data = source.read(size) - cache_ram.increment(cache_key + ':uploaded', size) - else: - data = source.read(chunk_size) - cache_ram.increment(cache_key + ':uploaded', chunk_size) - length = len(data) - if length > size: - (data, length) = (data[:size], size) - size -= length - if length == 0: - break - dest.write(data) - if length < chunk_size: - break - dest.seek(0) + copystream(source, dest, size, chunk_size, + lambda v : cache_ram.increment(cache_key + ':uploaded', v)) cache_ram(cache_key + ':length', None) cache_ram(cache_key + ':uploaded', None) return dest @@ -355,11 +358,9 @@ class Request(Storage): """ cmd_opts = global_settings.cmd_options # checking if this is called within the scheduler or within the shell - # in addition to checking if it's a cronjob - # FIXME: cmd_opts.scheduler does not imply that - # we are running in the scheduler - if (self.is_https or cmd_opts and ( - cmd_opts.shell or cmd_opts.scheduler or cmd_opts.cronjob)): + # in addition to checking if it's a cron job + if (self.is_https or self.is_scheduler or cmd_opts and ( + cmd_opts.shell or cmd_opts.cron_job)): current.session.secure() else: current.session.forget() diff --git a/gluon/main.py b/gluon/main.py index 68e4307d..08d04860 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -1,4 +1,3 @@ -#!/bin/env python # -*- coding: utf-8 -*- """ @@ -117,8 +116,8 @@ load_routes() HTTPS_SCHEMES = set(('https', 'HTTPS')) -# pattern used to validate client address -REGEX_CLIENT = re.compile(r'[\w:-]+(\.[\w-]+)*\.?') # ## to account for IPV6 +# pattern used to match client IP address +REGEX_CLIENT = re.compile(r'[\w:]+(\.\w+)*') def get_client(env): """ @@ -557,7 +556,8 @@ def wsgibase(environ, responder): return wsgibase(new_environ, responder) if global_settings.web2py_crontype == 'soft': cmd_opts = global_settings.cmd_options - newcron.softcron(global_settings.applications_parent).start() + newcron.softcron(global_settings.applications_parent, + apps=cmd_opts and cmd_opts.crontabs).start() return http_response.to(responder, env=env) diff --git a/gluon/myregex.py b/gluon/myregex.py deleted file mode 100644 index 5c3a7c99..00000000 --- a/gluon/myregex.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -| This file is part of the web2py Web Framework -| Copyrighted by Massimo Di Pierro -| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) - -Useful regexes ---------------- -""" - -import re - -# pattern to find defined tables - -regex_tables = re.compile( - """^[\w]+\.define_table\(\s*[\'\"](?P\w+)[\'\"]""", - flags=re.M) - -# pattern to find exposed functions in controller - -regex_expose = re.compile( - '^def\s+(?P_?[a-zA-Z0-9]\w*)\( *\)\s*:', - flags=re.M) - -regex_longcomments = re.compile('(""".*?"""|'+"'''.*?''')", re.DOTALL) - -regex_include = re.compile( - '(?P\{\{\s*include\s+[\'"](?P[^\'"]*)[\'"]\s*\}\})') - -regex_extend = re.compile( - '^\s*(?P\{\{\s*extend\s+[\'"](?P[^\'"]+)[\'"]\s*\}\})', re.MULTILINE) diff --git a/gluon/newcron.py b/gluon/newcron.py index 5eba24e0..4d208265 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# vim: set ts=4 sw=4 et ai: """ | This file is part of the web2py Web Framework @@ -9,13 +10,15 @@ Cron-style interface """ -import sys -import os import threading -import logging +import os +from logging import getLogger, DEBUG import time import sched +import sys import re +import shlex + import datetime from functools import reduce from gluon.settings import global_settings @@ -23,10 +26,18 @@ from gluon import fileutils from gluon._compat import to_bytes, pickle from pydal.contrib import portalocker -logger = logging.getLogger("web2py.cron") +logger_name = 'web2py.cron' + + _cron_stopping = False + +_cron_subprocs_lock = threading.RLock() _cron_subprocs = [] +def subprocess_count(): + with _cron_subprocs_lock: + return len(_cron_subprocs) + def absolute_path_link(path): """ @@ -45,14 +56,14 @@ def stopcron(): """Graceful shutdown of cron""" global _cron_stopping _cron_stopping = True - while _cron_subprocs: - proc = _cron_subprocs.pop() + while subprocess_count(): + with _cron_subprocs_lock: + proc = _cron_subprocs.pop() if proc.poll() is None: try: proc.terminate() - except: - import traceback - traceback.print_exc() + except Exception: + getLogger(logger_name).exception('error in stopcron') class extcron(threading.Thread): @@ -62,11 +73,10 @@ class extcron(threading.Thread): self.setDaemon(False) self.path = applications_parent self.apps = apps - # crondance(self.path, 'external', startup=True, apps=self.apps) def run(self): if not _cron_stopping: - logger.debug('external cron invocation') + getLogger(logger_name).debug('external cron invocation') crondance(self.path, 'external', startup=False, apps=self.apps) @@ -77,16 +87,19 @@ class hardcron(threading.Thread): self.setDaemon(True) self.path = applications_parent self.apps = apps + # processing of '@reboot' entries in crontab (startup=True) + getLogger(logger_name).info('hard cron bootstrap') crondance(self.path, 'hard', startup=True, apps=self.apps) def launch(self): if not _cron_stopping: - logger.debug('hard cron invocation') + self.logger.debug('hard cron invocation') crondance(self.path, 'hard', startup=False, apps=self.apps) def run(self): + self.logger = getLogger(logger_name) + self.logger.info('hard cron daemon started') s = sched.scheduler(time.time, time.sleep) - logger.info('hard cron daemon started') while not _cron_stopping: now = time.time() s.enter(60 - now % 60, 1, self.launch, ()) @@ -99,14 +112,14 @@ class softcron(threading.Thread): threading.Thread.__init__(self) self.path = applications_parent self.apps = apps - # crondance(self.path, 'soft', startup=True, apps=self.apps) def run(self): if not _cron_stopping: - logger.debug('soft cron invocation') + getLogger(logger_name).debug('soft cron invocation') crondance(self.path, 'soft', startup=False, apps=self.apps) +# TODO: context manager class Token(object): def __init__(self, path): @@ -115,6 +128,7 @@ class Token(object): fileutils.write_file(self.path, to_bytes(''), 'wb') self.master = None self.now = time.time() + self.logger = getLogger(logger_name) def acquire(self, startup=False): """ @@ -133,7 +147,7 @@ class Token(object): else: locktime = 59.99 if portalocker.LOCK_EX is None: - logger.warning('cron disabled because no file locking') + self.logger.warning('cron disabled because no file locking') return None self.master = fileutils.open_file(self.path, 'rb+') try: @@ -148,8 +162,8 @@ class Token(object): ret = self.now if not stop: # this happens if previous cron job longer than 1 minute - logger.warning('stale cron.master detected') - logger.debug('acquiring lock') + self.logger.warning('stale cron.master detected') + self.logger.debug('acquiring lock') self.master.seek(0) pickle.dump((self.now, 0), self.master) self.master.flush() @@ -167,7 +181,7 @@ class Token(object): ret = self.master.closed if not self.master.closed: portalocker.lock(self.master, portalocker.LOCK_EX) - logger.debug('releasing cron lock') + self.logger.debug('releasing cron lock') self.master.seek(0) (start, stop) = pickle.load(self.master) if start == self.now: # if this is my lock @@ -248,26 +262,25 @@ class cronlauncher(threading.Thread): def run(self): import subprocess - global _cron_subprocs - if isinstance(self.cmd, (list, tuple)): - cmd = self.cmd - else: - cmd = self.cmd.split() - proc = subprocess.Popen(cmd, + logger = getLogger(logger_name) + proc = subprocess.Popen(self.cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _cron_subprocs.append(proc) + with _cron_subprocs_lock: + _cron_subprocs.append(proc) (stdoutdata, stderrdata) = proc.communicate() try: - _cron_subprocs.remove(proc) + with _cron_subprocs_lock: + _cron_subprocs.remove(proc) except ValueError: pass if proc.returncode != 0: - logger.warning('call returned code %s:\n%s\n%s', - proc.returncode, stdoutdata, stderrdata) - else: - logger.debug('call returned success:\n%s', stdoutdata) + logger.warning('%r call returned code %s:\n%s\n%s', + ' '.join(self.cmd), proc.returncode, stdoutdata, stderrdata) + elif logger.isEnabledFor(DEBUG): + logger.debug('%r call returned success:\n%s', + ' '.join(self.cmd), stdoutdata) def crondance(applications_parent, ctype='soft', startup=False, apps=None): @@ -287,7 +300,9 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): ('dom', now_s.tm_mday), ('dow', (now_s.tm_wday + 1) % 7)) - if apps is None: + logger = getLogger(logger_name) + + if not apps: apps = [x for x in os.listdir(apppath) if os.path.isdir(os.path.join(apppath, x))] @@ -303,10 +318,7 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): base_commands.append(w2p_path) if applications_parent != global_settings.gluon_parent: base_commands.extend(('-f', applications_parent)) - base_commands.extend(('--cronjob', '--no-banner', '--nogui', '--plain', - # FIXME: this should not be needed since we are - # not launching the web server - '-a', '""')) + base_commands.extend(('--cron_job', '--no_banner', '--no_gui', '--plain')) for app in apps: if _cron_stopping: @@ -335,15 +347,16 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): for task in tasks: if _cron_stopping: break - citems = [(k in task and not v in task[k]) for k, v in checks] - task_min = task.get('min', []) if not task: continue - elif not startup and task_min == [-1]: + task_min = task.get('min', []) + if not startup and task_min == [-1]: continue - elif task_min != [-1] and reduce(lambda a, b: a or b, citems): + citems = [(k in task and not v in task[k]) for k, v in checks] + if task_min != [-1] and reduce(lambda a, b: a or b, citems): continue - logger.info('%s cron: %s executing %s in %s at %s', + + logger.info('%s cron: %s executing %r in %s at %s', ctype, app, task.get('cmd'), os.getcwd(), datetime.datetime.now()) action = models = False @@ -364,11 +377,12 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): if models: commands.append('-M') else: - commands = command + commands = shlex.split(command) try: + # FIXME: using a new thread every time there is a task to + # launch is not a good idea in a long running process cronlauncher(commands).start() - except Exception as e: - logger.warning('execution error for %s: %s', - task.get('cmd'), e) + except Exception: + logger.exception('error starting %r', task['cmd']) token.release() diff --git a/gluon/packages/dal b/gluon/packages/dal index f1cf5aab..3815fbe9 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit f1cf5aab12b839ec168cc194f10c33e026b3de6b +Subproject commit 3815fbe9b6f7af4e2fa28c509010fea10d072b02 diff --git a/gluon/packages/yatl b/gluon/packages/yatl index 8c6f1e1f..dd969365 160000 --- a/gluon/packages/yatl +++ b/gluon/packages/yatl @@ -1 +1 @@ -Subproject commit 8c6f1e1f17f08d5638490fb3dfe736953f585f84 +Subproject commit dd969365265bf329c73784d989bdaa6de23d9633 diff --git a/gluon/serializers.py b/gluon/serializers.py index 440b8657..a72bc216 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -119,13 +119,43 @@ def xml(value, encoding='UTF-8', key='document', quote=True): return ('' % encoding) + str(xml_rec(value, key, quote)) -def json(value, default=custom_json, indent=None, sort_keys=False): - value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent) - # replace JavaScript incompatible spacing - # http://timelessrepo.com/json-isnt-a-javascript-subset - # PY3 FIXME - # return value.replace(ur'\u2028', '\\u2028').replace(ur'\2029', '\\u2029') - return value +class JSONEncoderForHTML(json_parser.JSONEncoder): + """An encoder that produces JSON safe to embed in HTML. + To embed JSON content in, say, a script tag on a web page, the + characters &, < and > should be escaped. They cannot be escaped + with the usual entities (e.g. &) because they are not expanded + within