From 066d9c9ab54e9e88e42e58f7effb19afaade4f88 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 11 Apr 2019 21:15:48 -0700 Subject: [PATCH] better newcrow and widget, fixed some ssl related bugs, thanks Paolo --- gluon/main.py | 5 +- gluon/newcron.py | 106 ++++++++++++++----------------- gluon/packages/dal | 2 +- gluon/widget.py | 155 +++++++++++++++++++++++++-------------------- 4 files changed, 140 insertions(+), 128 deletions(-) diff --git a/gluon/main.py b/gluon/main.py index aeb7f76e..89ef709a 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -55,7 +55,6 @@ web2py_path = global_settings.applications_parent # backward compatibility create_missing_folders() # set up logging for subsequent imports -import logging import logging.config # This needed to prevent exception on Python 2.5: @@ -765,8 +764,8 @@ class HttpServer(object): app_info=app_info, min_threads=min_threads, max_threads=max_threads, - queue_size=int(request_queue_size), - timeout=int(timeout), + queue_size=request_queue_size, + timeout=timeout, handle_signals=False, ) diff --git a/gluon/newcron.py b/gluon/newcron.py index 4affed93..a415b530 100644 --- a/gluon/newcron.py +++ b/gluon/newcron.py @@ -17,7 +17,6 @@ import time import sched import re import datetime -import platform from functools import reduce from gluon.settings import global_settings from gluon import fileutils @@ -87,7 +86,7 @@ class hardcron(threading.Thread): def run(self): s = sched.scheduler(time.time, time.sleep) - logger.info('Hard cron daemon started') + logger.info('hard cron daemon started') while not _cron_stopping: now = time.time() s.enter(60 - now % 60, 1, self.launch, ()) @@ -133,7 +132,7 @@ class Token(object): else: locktime = 59.99 if portalocker.LOCK_EX is None: - logger.warning('WEB2PY CRON: Disabled because no file locking') + logger.warning('cron disabled because no file locking') return None self.master = fileutils.open_file(self.path, 'rb+') try: @@ -142,13 +141,14 @@ class Token(object): try: (start, stop) = pickle.load(self.master) except: - (start, stop) = (0, 1) + start = 0 + stop = 1 if startup or self.now - start > locktime: ret = self.now if not stop: # this happens if previous cron job longer than 1 minute - logger.warning('WEB2PY CRON: Stale cron.master detected') - logger.debug('WEB2PY CRON: Acquiring lock') + logger.warning('stale cron.master detected') + logger.debug('acquiring lock') self.master.seek(0) pickle.dump((self.now, 0), self.master) self.master.flush() @@ -166,7 +166,7 @@ class Token(object): ret = self.master.closed if not self.master.closed: portalocker.lock(self.master, portalocker.LOCK_EX) - logger.debug('WEB2PY CRON: Releasing cron lock') + logger.debug('releasing cron lock') self.master.seek(0) (start, stop) = pickle.load(self.master) if start == self.now: # if this is my lock @@ -241,12 +241,9 @@ def parsecronline(line): class cronlauncher(threading.Thread): - def __init__(self, cmd, shell=True): + def __init__(self, cmd): threading.Thread.__init__(self) - if platform.system() == 'Windows': - shell = False self.cmd = cmd - self.shell = shell def run(self): import subprocess @@ -258,8 +255,7 @@ class cronlauncher(threading.Thread): proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=self.shell) + stderr=subprocess.PIPE) _cron_subprocs.append(proc) (stdoutdata, stderrdata) = proc.communicate() try: @@ -267,15 +263,14 @@ class cronlauncher(threading.Thread): except ValueError: pass if proc.returncode != 0: - logger.warning( - 'WEB2PY CRON Call returned code %s:\n%s' % - (proc.returncode, stdoutdata + stderrdata)) + logger.warning('call returned code %s:\n%s\n%s', + proc.returncode, stdoutdata, stderrdata) else: - logger.debug('WEB2PY CRON Call returned success:\n%s' - % stdoutdata) + logger.debug('call returned success:\n%s', stdoutdata) def crondance(applications_parent, ctype='soft', startup=False, apps=None): + # TODO: docstring apppath = os.path.join(applications_parent, 'applications') token = Token(applications_parent) cronmaster = token.acquire(startup=startup) @@ -294,6 +289,21 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): full_apath_links = set() + if sys.executable.lower().endswith('pythonservice.exe'): + _python_exe = os.path.join(sys.exec_prefix, 'python.exe') + else: + _python_exe = sys.executable + base_commands = [_python_exe] + w2p_path = fileutils.abspath('web2py.py', gluon=True) + if os.path.exists(w2p_path): + base_commands.append(w2p_path) + if applications_parent != global_settings.gluon_parent: + base_commands.extend(('-f', applications_parent)) + base_commands.extend(('-J', + # FIXME: this should not be needed since we are + # not launching the web server + '-a', '""')) + for app in apps: if _cron_stopping: break @@ -315,22 +325,12 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): lines = [line for line in cronlines if line and not line.startswith('#')] tasks = [parsecronline(cline) for cline in lines] except Exception as e: - logger.error('WEB2PY CRON: crontab read error %s' % e) + logger.error('crontab read error %s', e) continue for task in tasks: if _cron_stopping: break - if sys.executable.lower().endswith('pythonservice.exe'): - _python_exe = os.path.join(sys.exec_prefix, 'python.exe') - else: - _python_exe = sys.executable - commands = [_python_exe] - w2p_path = fileutils.abspath('web2py.py', gluon=True) - if os.path.exists(w2p_path): - commands.append(w2p_path) - if applications_parent != global_settings.gluon_parent: - commands.extend(('-f', applications_parent)) citems = [(k in task and not v in task[k]) for k, v in checks] task_min = task.get('min', []) if not task: @@ -339,40 +339,32 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None): continue elif task_min != [-1] and reduce(lambda a, b: a or b, citems): continue - logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' - % (ctype, app, task.get('cmd'), - os.getcwd(), datetime.datetime.now())) - action, command, models = False, task['cmd'], '' + logger.info('%s cron: %s executing %s in %s at %s', + ctype, app, task.get('cmd'), + os.getcwd(), datetime.datetime.now()) + action = models = False + command = task['cmd'] if command.startswith('**'): - (action, models, command) = (True, '', command[2:]) + action = True + command = command[2:] elif command.startswith('*'): - (action, models, command) = (True, '-M', command[1:]) - else: - action = False + action = models = True + command = command[1:] - if action and command.endswith('.py'): - commands.extend(('-J', # cron job - models, # import models? - '-S', app, # app name - '-a', '""', # password - '-R', command)) # command - elif action: - commands.extend(('-J', # cron job - models, # import models? - '-S', app + '/' + command, # app name - '-a', '""')) # password + if action: + commands = base_commands[:] + if command.endswith('.py'): + commands.extend(('-S', app, '-R', command)) + else: + commands.extend(('-S', app + '/' + command)) + if models: + commands.append('-M') else: commands = command - # from python docs: - # You do not need shell=True to run a batch file or - # console-based executable. - shell = False - try: - cronlauncher(commands, shell=shell).start() + cronlauncher(commands).start() except Exception as e: - logger.warning( - 'WEB2PY CRON: Execution error for %s: %s' - % (task.get('cmd'), e)) + logger.warning('execution error for %s: %s', + task.get('cmd'), e) token.release() diff --git a/gluon/packages/dal b/gluon/packages/dal index cecd7712..37784cb6 160000 --- a/gluon/packages/dal +++ b/gluon/packages/dal @@ -1 +1 @@ -Subproject commit cecd77127c122404c1aee7f6377c6a0150d86d84 +Subproject commit 37784cb6aaa37340eb706eb550164a3f56be4186 diff --git a/gluon/widget.py b/gluon/widget.py index 33ef31df..c87028e9 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -10,7 +10,7 @@ The widget is called from web2py """ import sys -from gluon._compat import StringIO, thread, xrange, PY2 +from gluon._compat import thread, xrange, PY2 import time import threading import os @@ -465,7 +465,7 @@ class web2pyDialog(object): except: return self.error('invalid port number') - if self.options.ssl_certificate or self.options.ssl_private_key: + if self.options.ssl_certificate and self.options.ssl_private_key: proto = 'https' else: proto = 'http' @@ -583,18 +583,18 @@ def console(): version=ProgramVersion, description='web2py Web Framework startup script.', epilog='''NOTE: unless a password is specified (-a 'passwd') -web2py will attempt to run a GUI to ask for it +web2py will attempt to run a GUI to ask for it when starting the web server (if not disabled with --nogui).''') parser.add_option('-i', '--ip', default='127.0.0.1', - help=\ + metavar='IP_ADDR', help=\ 'IP address of the server (e.g., 127.0.0.1 or ::1); ' \ 'Note: This value is ignored when using the --interfaces option') parser.add_option('-p', '--port', default=8000, - type='int', help=\ + type='int', metavar='NUM', help=\ 'port of server (%default); ' \ 'Note: This value is ignored when using the --interfaces option') @@ -607,43 +607,43 @@ web2py will attempt to run a GUI to ask for it default='', help=\ 'password to be used for administration ' \ - '(use -a "" to reuse the last password))') + '(use "" to reuse the last password), ' \ + 'when no password is available the administrative ' \ + 'interface will be disabled') parser.add_option('-c', '--ssl_certificate', - default='', - help='file that contains ssl certificate') + default=None, + metavar='FILE', help='server certificate file') parser.add_option('-k', '--ssl_private_key', - default='', - help='file that contains ssl private key') + default=None, + metavar='FILE', help='server private key file') parser.add_option('--ca-cert', dest='ssl_ca_certificate', default=None, - help=\ - 'use this file containing the CA certificate to validate X509 ' \ - 'certificates from clients') + metavar='FILE', help='CA certificate file') parser.add_option('-d', '--pid_filename', default='httpserver.pid', - help='file to store the pid of the server') + metavar='FILE', help='server pid file (%default)') parser.add_option('-l', '--log_filename', default='httpserver.log', - help='name for the server log file') + metavar='FILE', help='server log file (%default)') parser.add_option('-n', '--numthreads', default=None, - type='int', + type='int', metavar='NUM', help='number of threads (deprecated)') parser.add_option('--minthreads', default=None, - type='int', + type='int', metavar='NUM', help='minimum number of server threads') parser.add_option('--maxthreads', default=None, - type='int', + type='int', metavar='NUM', help='maximum number of server threads') parser.add_option('-s', '--server_name', @@ -651,28 +651,30 @@ web2py will attempt to run a GUI to ask for it help='web server name (%default)') parser.add_option('-q', '--request_queue_size', - default='5', - type='int', + default=5, + type='int', metavar='NUM', help=\ - 'max number of queued requests when server unavailable') + 'max number of queued requests when server unavailable (%default)') parser.add_option('-o', '--timeout', - default='10', - type='int', + default=10, + type='int', metavar='SECONDS', help='timeout for individual request (%default seconds)') parser.add_option('-z', '--shutdown_timeout', - default='5', - type='int', - help='timeout on shutdown of server (%default seconds)') + default=None, + type='int', metavar='SECONDS', + help=\ + 'timeout on server shutdown; this value is not used by ' \ + 'Rocket web server') parser.add_option('--socket-timeout', dest='socket_timeout', # not needed default=5, - type='int', + type='int', metavar='SECONDS', help='timeout for socket (%default seconds)') parser.add_option('-f', '--folder', - default=os.getcwd(), + default=os.getcwd(), metavar='WEB2PY_DIR', help='folder from which to run web2py') parser.add_option('-v', '--verbose', @@ -693,8 +695,8 @@ web2py will attempt to run a GUI to ask for it parser.add_option('-D', '--debug', dest='debuglevel', default=30, type='int', - help=\ - 'set debug output level (0-100, 0 means all, 100 means none; ' \ + metavar='LOG_LEVEL', help=\ + 'set log level (0-100, 0 means all, 100 means none; ' \ 'default is %default)') parser.add_option('-S', '--shell', @@ -722,7 +724,7 @@ web2py will attempt to run a GUI to ask for it default=False, action='store_true', help=\ - 'auto import model files; default is %default; should be used ' \ + 'auto import model files (default is %default); should be used ' \ 'with --shell option') parser.add_option('-R', '--run', @@ -733,18 +735,18 @@ web2py will attempt to run a GUI to ask for it parser.add_option('-K', '--scheduler', default=None, - help=\ + metavar='APP_LIST', help=\ 'run scheduled tasks for the specified apps: expects a list of ' \ - 'app names as -K app1,app2,app3 ' \ - 'or a list of app:groups as -K app1:group1:group2,app2:group1 ' \ - 'to override specific group_names. (only strings, no spaces ' \ - 'allowed. Requires a scheduler defined in the models') + 'app names as app1,app2,app3 ' \ + 'or a list of app:groups as app1:group1:group2,app2:group1 ' \ + '(only strings, no spaces allowed). NOTE: ' \ + 'Requires a scheduler defined in the models') parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed default=False, action='store_true', help=\ - 'run schedulers alongside webserver, needs -K app1 and -a too') + 'run schedulers alongside webserver, needs -K') parser.add_option('-T', '--test', default=None, @@ -756,12 +758,16 @@ web2py will attempt to run a GUI to ask for it default=False, action='store_true', help=\ - 'trigger a cron run manually; usually invoked from a system crontab') + 'trigger a cron run and exit; usually used when invoked ' \ + 'from a system crontab') parser.add_option('--softcron', default=False, action='store_true', - help='triggers the use of softcron') + help=\ + 'use software cron emulation instead of separate cron process, '\ + 'needs -Y; NOTE: use of software cron emulation is strongly ' + 'discouraged') parser.add_option('-Y', '--run-cron', dest='runcron', default=False, @@ -795,7 +801,8 @@ web2py will attempt to run a GUI to ask for it default=None, help=\ 'should be followed by a list of arguments to be passed to script, ' \ - 'to be used with -S, -A must be the last option') + 'to be used with -S; NOTE: must be the last option because eat all ' \ + 'remaining arguments') parser.add_option('--no-banner', dest='nobanner', default=False, @@ -819,8 +826,8 @@ web2py will attempt to run a GUI to ask for it default=False, action='store_true', help=\ - 'adds coverage reporting (needs --run_system_tests), ' \ - 'python 2.7 and the coverage module installed. ' \ + 'adds coverage reporting (should be used with --run_system_tests), ' \ + 'needs Python 2.7+ and the coverage module installed. ' \ 'You can alter the default path setting the environment ' \ 'variable "COVERAGE_PROCESS_START" ' \ '(by default it takes gluon/tests/coverage.ini)') @@ -849,6 +856,7 @@ web2py will attempt to run a GUI to ask for it if hasattr(options, key): setattr(options, key, getattr(options2, key)) + # store in options.ips the list of server IP addresses try: options.ips = list(set( # no duplicates [addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn()) @@ -862,10 +870,11 @@ web2py will attempt to run a GUI to ask for it options.nobanner = True options.nogui = True - # accept --interfaces in the form - # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" - # (no spaces; optional key:cert:ca_cert indicate SSL) - if isinstance(options.interfaces, str): + # transform options.interfaces, in the form + # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" + # (no spaces; optional key:cert:ca_cert indicate SSL), into + # a list of tuples + if options.interfaces: interfaces = options.interfaces.split(';') options.interfaces = [] for interface in interfaces: @@ -881,16 +890,16 @@ web2py will attempt to run a GUI to ask for it interface[1] = int(interface[1]) # numeric port options.interfaces.append(tuple(interface)) - # accepts --scheduler in the form - # "app:group1:group2,app2:group1" - scheduler = [] - options.scheduler_groups = None - if isinstance(options.scheduler, str): - if ':' in options.scheduler: - for opt in options.scheduler.split(','): - scheduler.append(opt.split(':')) - options.scheduler = ','.join([app[0] for app in scheduler]) - options.scheduler_groups = scheduler + # strip group infos from options.scheduler, in the form + # "app:group1:group2,app2:group1", and put into a list of lists + # in options.scheduler_groups + if options.scheduler and ':' in options.scheduler: + sg = options.scheduler_groups = [] + for awg in options.scheduler.split(','): + sg.append(awg.split(':')) + options.scheduler = ','.join([app[0] for app in sg]) + else: + options.scheduler_groups = None if options.numthreads is not None and options.minthreads is None: options.minthreads = options.numthreads # legacy @@ -898,8 +907,6 @@ web2py will attempt to run a GUI to ask for it copy_options = copy.deepcopy(options) copy_options.password = '******' global_settings.cmd_options = copy_options - # FIXME: do we still really need this? - global_settings.cmd_args = args return options, args @@ -930,9 +937,8 @@ def start_schedulers(options): sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') return processes = [] - apps = [(app.strip(), None) for app in options.scheduler.split(',')] - if options.scheduler_groups: - apps = options.scheduler_groups + apps = options.scheduler_groups or \ + [(app.strip(), None) for app in options.scheduler.split(',')] code = "from gluon.globals import current;current._scheduler.loop()" logging.getLogger().setLevel(options.debuglevel) if options.folder: @@ -1007,11 +1013,26 @@ def start(cron=True): run_system_tests(options) if options.quiet: - capture = StringIO() - sys.stdout = capture - logger.setLevel(logging.CRITICAL + 1) - else: - logger.setLevel(options.debuglevel) + # to prevent writes on stdout set a null stream + class NullFile(object): + def write(self, x): + pass + sys.stdout = NullFile() + # but still has to mute existing loggers, to do that iterate + # over all existing loggers (root logger included) and remove + # all attached logging.StreamHandler instances currently + # streaming on sys.stdout or sys.stderr + loggers = [logging.getLogger()] + loggers.extend(logging.Logger.manager.loggerDict.values()) + for logger in loggers: + if isinstance(logger, logging.PlaceHolder): continue + for h in logger.handlers[:]: + if isinstance(h, logging.StreamHandler) and \ + h.stream in (sys.stdout, sys.stderr): + logger.removeHandler(h) + # NOTE: stderr.write() is still working + + logger.setLevel(options.debuglevel) if not options.nobanner: # banner @@ -1144,7 +1165,7 @@ end tell ip = first_if[0] port = first_if[1] - if options.ssl_certificate or options.ssl_private_key: + if options.ssl_certificate and options.ssl_private_key: proto = 'https' else: proto = 'http'