dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better newcrow and widget, fixed some ssl related bugs, thanks Paolo

Browse Source
This commit is contained in:
mdipierro
2019-04-11 21:15:48 -07:00
parent b96c54cef9
commit 066d9c9ab5
4 changed files with 140 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gluon/main.py
+2 -3
View File
@@ -55,7 +55,6 @@ web2py_path = global_settings.applications_parent # backward compatibility
create_missing_folders() create_missing_folders()
# set up logging for subsequent imports # set up logging for subsequent imports
import logging
import logging.config import logging.config
# This needed to prevent exception on Python 2.5: # This needed to prevent exception on Python 2.5:
@@ -765,8 +764,8 @@ class HttpServer(object):
app_info=app_info, app_info=app_info,
min_threads=min_threads, min_threads=min_threads,
max_threads=max_threads, max_threads=max_threads,
queue_size=int(request_queue_size), queue_size=request_queue_size,
timeout=int(timeout), timeout=timeout,
handle_signals=False, handle_signals=False,
) )
gluon/newcron.py
+49 -57
View File
@@ -17,7 +17,6 @@ import time
import sched import sched
import re import re
import datetime import datetime
import platform
from functools import reduce from functools import reduce
from gluon.settings import global_settings from gluon.settings import global_settings
from gluon import fileutils from gluon import fileutils
@@ -87,7 +86,7 @@ class hardcron(threading.Thread):
def run(self): def run(self):
s = sched.scheduler(time.time, time.sleep) s = sched.scheduler(time.time, time.sleep)
logger.info('Hard cron daemon started') logger.info('hard cron daemon started')
while not _cron_stopping: while not _cron_stopping:
now = time.time() now = time.time()
s.enter(60 - now % 60, 1, self.launch, ()) s.enter(60 - now % 60, 1, self.launch, ())
@@ -133,7 +132,7 @@ class Token(object):
else: else:
locktime = 59.99 locktime = 59.99
if portalocker.LOCK_EX is None: if portalocker.LOCK_EX is None:
logger.warning('WEB2PY CRON: Disabled because no file locking') logger.warning('cron disabled because no file locking')
return None return None
self.master = fileutils.open_file(self.path, 'rb+') self.master = fileutils.open_file(self.path, 'rb+')
try: try:
@@ -142,13 +141,14 @@ class Token(object):
try: try:
(start, stop) = pickle.load(self.master) (start, stop) = pickle.load(self.master)
except: except:
(start, stop) = (0, 1) start = 0
stop = 1
if startup or self.now - start > locktime: if startup or self.now - start > locktime:
ret = self.now ret = self.now
if not stop: if not stop:
# this happens if previous cron job longer than 1 minute # this happens if previous cron job longer than 1 minute
logger.warning('WEB2PY CRON: Stale cron.master detected') logger.warning('stale cron.master detected')
logger.debug('WEB2PY CRON: Acquiring lock') logger.debug('acquiring lock')
self.master.seek(0) self.master.seek(0)
pickle.dump((self.now, 0), self.master) pickle.dump((self.now, 0), self.master)
self.master.flush() self.master.flush()
@@ -166,7 +166,7 @@ class Token(object):
ret = self.master.closed ret = self.master.closed
if not self.master.closed: if not self.master.closed:
portalocker.lock(self.master, portalocker.LOCK_EX) portalocker.lock(self.master, portalocker.LOCK_EX)
logger.debug('WEB2PY CRON: Releasing cron lock') logger.debug('releasing cron lock')
self.master.seek(0) self.master.seek(0)
(start, stop) = pickle.load(self.master) (start, stop) = pickle.load(self.master)
if start == self.now: # if this is my lock if start == self.now: # if this is my lock
@@ -241,12 +241,9 @@ def parsecronline(line):
class cronlauncher(threading.Thread): class cronlauncher(threading.Thread):
def __init__(self, cmd, shell=True): def __init__(self, cmd):
threading.Thread.__init__(self) threading.Thread.__init__(self)
if platform.system() == 'Windows':
shell = False
self.cmd = cmd self.cmd = cmd
self.shell = shell
def run(self): def run(self):
import subprocess import subprocess
@@ -258,8 +255,7 @@ class cronlauncher(threading.Thread):
proc = subprocess.Popen(cmd, proc = subprocess.Popen(cmd,
stdin=subprocess.PIPE, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE)
shell=self.shell)
_cron_subprocs.append(proc) _cron_subprocs.append(proc)
(stdoutdata, stderrdata) = proc.communicate() (stdoutdata, stderrdata) = proc.communicate()
try: try:
@@ -267,15 +263,14 @@ class cronlauncher(threading.Thread):
except ValueError: except ValueError:
pass pass
if proc.returncode != 0: if proc.returncode != 0:
logger.warning( logger.warning('call returned code %s:\n%s\n%s',
'WEB2PY CRON Call returned code %s:\n%s' % proc.returncode, stdoutdata, stderrdata)
(proc.returncode, stdoutdata + stderrdata))
else: else:
logger.debug('WEB2PY CRON Call returned success:\n%s' logger.debug('call returned success:\n%s', stdoutdata)
% stdoutdata)
def crondance(applications_parent, ctype='soft', startup=False, apps=None): def crondance(applications_parent, ctype='soft', startup=False, apps=None):
# TODO: docstring
apppath = os.path.join(applications_parent, 'applications') apppath = os.path.join(applications_parent, 'applications')
token = Token(applications_parent) token = Token(applications_parent)
cronmaster = token.acquire(startup=startup) cronmaster = token.acquire(startup=startup)
@@ -294,6 +289,21 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
full_apath_links = set() full_apath_links = set()
if sys.executable.lower().endswith('pythonservice.exe'):
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
else:
_python_exe = sys.executable
base_commands = [_python_exe]
w2p_path = fileutils.abspath('web2py.py', gluon=True)
if os.path.exists(w2p_path):
base_commands.append(w2p_path)
if applications_parent != global_settings.gluon_parent:
base_commands.extend(('-f', applications_parent))
base_commands.extend(('-J',
# FIXME: this should not be needed since we are
# not launching the web server
'-a', '"<recycle>"'))
for app in apps: for app in apps:
if _cron_stopping: if _cron_stopping:
break break
@@ -315,22 +325,12 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
lines = [line for line in cronlines if line and not line.startswith('#')] lines = [line for line in cronlines if line and not line.startswith('#')]
tasks = [parsecronline(cline) for cline in lines] tasks = [parsecronline(cline) for cline in lines]
except Exception as e: except Exception as e:
logger.error('WEB2PY CRON: crontab read error %s' % e) logger.error('crontab read error %s', e)
continue continue
for task in tasks: for task in tasks:
if _cron_stopping: if _cron_stopping:
break break
if sys.executable.lower().endswith('pythonservice.exe'):
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
else:
_python_exe = sys.executable
commands = [_python_exe]
w2p_path = fileutils.abspath('web2py.py', gluon=True)
if os.path.exists(w2p_path):
commands.append(w2p_path)
if applications_parent != global_settings.gluon_parent:
commands.extend(('-f', applications_parent))
citems = [(k in task and not v in task[k]) for k, v in checks] citems = [(k in task and not v in task[k]) for k, v in checks]
task_min = task.get('min', []) task_min = task.get('min', [])
if not task: if not task:
@@ -339,40 +339,32 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
continue continue
elif task_min != [-1] and reduce(lambda a, b: a or b, citems): elif task_min != [-1] and reduce(lambda a, b: a or b, citems):
continue continue
logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' logger.info('%s cron: %s executing %s in %s at %s',
% (ctype, app, task.get('cmd'), ctype, app, task.get('cmd'),
os.getcwd(), datetime.datetime.now())) os.getcwd(), datetime.datetime.now())
action, command, models = False, task['cmd'], '' action = models = False
command = task['cmd']
if command.startswith('**'): if command.startswith('**'):
(action, models, command) = (True, '', command[2:]) action = True
command = command[2:]
elif command.startswith('*'): elif command.startswith('*'):
(action, models, command) = (True, '-M', command[1:]) action = models = True
else: command = command[1:]
action = False
if action and command.endswith('.py'): if action:
commands.extend(('-J', # cron job commands = base_commands[:]
models, # import models? if command.endswith('.py'):
'-S', app, # app name commands.extend(('-S', app, '-R', command))
'-a', '"<recycle>"', # password else:
'-R', command)) # command commands.extend(('-S', app + '/' + command))
elif action: if models:
commands.extend(('-J', # cron job commands.append('-M')
models, # import models?
'-S', app + '/' + command, # app name
'-a', '"<recycle>"')) # password
else: else:
commands = command commands = command
# from python docs:
# You do not need shell=True to run a batch file or
# console-based executable.
shell = False
try: try:
cronlauncher(commands, shell=shell).start() cronlauncher(commands).start()
except Exception as e: except Exception as e:
logger.warning( logger.warning('execution error for %s: %s',
'WEB2PY CRON: Execution error for %s: %s' task.get('cmd'), e)
% (task.get('cmd'), e))
token.release() token.release()
gluon/packages/dal
+1 -1
Submodule gluon/packages/dal updated: cecd77127c...37784cb6aa
gluon/widget.py
+88 -67
View File
@@ -10,7 +10,7 @@ The widget is called from web2py
""" """
import sys import sys
from gluon._compat import StringIO, thread, xrange, PY2 from gluon._compat import thread, xrange, PY2
import time import time
import threading import threading
import os import os
@@ -465,7 +465,7 @@ class web2pyDialog(object):
except: except:
return self.error('invalid port number') return self.error('invalid port number')
if self.options.ssl_certificate or self.options.ssl_private_key: if self.options.ssl_certificate and self.options.ssl_private_key:
proto = 'https' proto = 'https'
else: else:
proto = 'http' proto = 'http'
@@ -583,18 +583,18 @@ def console():
version=ProgramVersion, version=ProgramVersion,
description='web2py Web Framework startup script.', description='web2py Web Framework startup script.',
epilog='''NOTE: unless a password is specified (-a 'passwd') epilog='''NOTE: unless a password is specified (-a 'passwd')
web2py will attempt to run a GUI to ask for it web2py will attempt to run a GUI to ask for it when starting the web server
(if not disabled with --nogui).''') (if not disabled with --nogui).''')
parser.add_option('-i', '--ip', parser.add_option('-i', '--ip',
default='127.0.0.1', default='127.0.0.1',
help=\ metavar='IP_ADDR', help=\
'IP address of the server (e.g., 127.0.0.1 or ::1); ' \ 'IP address of the server (e.g., 127.0.0.1 or ::1); ' \
'Note: This value is ignored when using the --interfaces option') 'Note: This value is ignored when using the --interfaces option')
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
default=8000, default=8000,
type='int', help=\ type='int', metavar='NUM', help=\
'port of server (%default); ' \ 'port of server (%default); ' \
'Note: This value is ignored when using the --interfaces option') 'Note: This value is ignored when using the --interfaces option')
@@ -607,43 +607,43 @@ web2py will attempt to run a GUI to ask for it
default='<ask>', default='<ask>',
help=\ help=\
'password to be used for administration ' \ 'password to be used for administration ' \
'(use -a "<recycle>" to reuse the last password))') '(use "<recycle>" to reuse the last password), ' \
'when no password is available the administrative ' \
'interface will be disabled')
parser.add_option('-c', '--ssl_certificate', parser.add_option('-c', '--ssl_certificate',
default='', default=None,
help='file that contains ssl certificate') metavar='FILE', help='server certificate file')
parser.add_option('-k', '--ssl_private_key', parser.add_option('-k', '--ssl_private_key',
default='', default=None,
help='file that contains ssl private key') metavar='FILE', help='server private key file')
parser.add_option('--ca-cert', dest='ssl_ca_certificate', parser.add_option('--ca-cert', dest='ssl_ca_certificate',
default=None, default=None,
help=\ metavar='FILE', help='CA certificate file')
'use this file containing the CA certificate to validate X509 ' \
'certificates from clients')
parser.add_option('-d', '--pid_filename', parser.add_option('-d', '--pid_filename',
default='httpserver.pid', default='httpserver.pid',
help='file to store the pid of the server') metavar='FILE', help='server pid file (%default)')
parser.add_option('-l', '--log_filename', parser.add_option('-l', '--log_filename',
default='httpserver.log', default='httpserver.log',
help='name for the server log file') metavar='FILE', help='server log file (%default)')
parser.add_option('-n', '--numthreads', parser.add_option('-n', '--numthreads',
default=None, default=None,
type='int', type='int', metavar='NUM',
help='number of threads (deprecated)') help='number of threads (deprecated)')
parser.add_option('--minthreads', parser.add_option('--minthreads',
default=None, default=None,
type='int', type='int', metavar='NUM',
help='minimum number of server threads') help='minimum number of server threads')
parser.add_option('--maxthreads', parser.add_option('--maxthreads',
default=None, default=None,
type='int', type='int', metavar='NUM',
help='maximum number of server threads') help='maximum number of server threads')
parser.add_option('-s', '--server_name', parser.add_option('-s', '--server_name',
@@ -651,28 +651,30 @@ web2py will attempt to run a GUI to ask for it
help='web server name (%default)') help='web server name (%default)')
parser.add_option('-q', '--request_queue_size', parser.add_option('-q', '--request_queue_size',
default='5', default=5,
type='int', type='int', metavar='NUM',
help=\ help=\
'max number of queued requests when server unavailable') 'max number of queued requests when server unavailable (%default)')
parser.add_option('-o', '--timeout', parser.add_option('-o', '--timeout',
default='10', default=10,
type='int', type='int', metavar='SECONDS',
help='timeout for individual request (%default seconds)') help='timeout for individual request (%default seconds)')
parser.add_option('-z', '--shutdown_timeout', parser.add_option('-z', '--shutdown_timeout',
default='5', default=None,
type='int', type='int', metavar='SECONDS',
help='timeout on shutdown of server (%default seconds)') help=\
'timeout on server shutdown; this value is not used by ' \
'Rocket web server')
parser.add_option('--socket-timeout', dest='socket_timeout', # not needed parser.add_option('--socket-timeout', dest='socket_timeout', # not needed
default=5, default=5,
type='int', type='int', metavar='SECONDS',
help='timeout for socket (%default seconds)') help='timeout for socket (%default seconds)')
parser.add_option('-f', '--folder', parser.add_option('-f', '--folder',
default=os.getcwd(), default=os.getcwd(), metavar='WEB2PY_DIR',
help='folder from which to run web2py') help='folder from which to run web2py')
parser.add_option('-v', '--verbose', parser.add_option('-v', '--verbose',
@@ -693,8 +695,8 @@ web2py will attempt to run a GUI to ask for it
parser.add_option('-D', '--debug', dest='debuglevel', parser.add_option('-D', '--debug', dest='debuglevel',
default=30, default=30,
type='int', type='int',
help=\ metavar='LOG_LEVEL', help=\
'set debug output level (0-100, 0 means all, 100 means none; ' \ 'set log level (0-100, 0 means all, 100 means none; ' \
'default is %default)') 'default is %default)')
parser.add_option('-S', '--shell', parser.add_option('-S', '--shell',
@@ -722,7 +724,7 @@ web2py will attempt to run a GUI to ask for it
default=False, default=False,
action='store_true', action='store_true',
help=\ help=\
'auto import model files; default is %default; should be used ' \ 'auto import model files (default is %default); should be used ' \
'with --shell option') 'with --shell option')
parser.add_option('-R', '--run', parser.add_option('-R', '--run',
@@ -733,18 +735,18 @@ web2py will attempt to run a GUI to ask for it
parser.add_option('-K', '--scheduler', parser.add_option('-K', '--scheduler',
default=None, default=None,
help=\ metavar='APP_LIST', help=\
'run scheduled tasks for the specified apps: expects a list of ' \ 'run scheduled tasks for the specified apps: expects a list of ' \
'app names as -K app1,app2,app3 ' \ 'app names as app1,app2,app3 ' \
'or a list of app:groups as -K app1:group1:group2,app2:group1 ' \ 'or a list of app:groups as app1:group1:group2,app2:group1 ' \
'to override specific group_names. (only strings, no spaces ' \ '(only strings, no spaces allowed). NOTE: ' \
'allowed. Requires a scheduler defined in the models') 'Requires a scheduler defined in the models')
parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed
default=False, default=False,
action='store_true', action='store_true',
help=\ help=\
'run schedulers alongside webserver, needs -K app1 and -a too') 'run schedulers alongside webserver, needs -K')
parser.add_option('-T', '--test', parser.add_option('-T', '--test',
default=None, default=None,
@@ -756,12 +758,16 @@ web2py will attempt to run a GUI to ask for it
default=False, default=False,
action='store_true', action='store_true',
help=\ help=\
'trigger a cron run manually; usually invoked from a system crontab') 'trigger a cron run and exit; usually used when invoked ' \
'from a system crontab')
parser.add_option('--softcron', parser.add_option('--softcron',
default=False, default=False,
action='store_true', action='store_true',
help='triggers the use of softcron') help=\
'use software cron emulation instead of separate cron process, '\
'needs -Y; NOTE: use of software cron emulation is strongly '
'discouraged')
parser.add_option('-Y', '--run-cron', dest='runcron', parser.add_option('-Y', '--run-cron', dest='runcron',
default=False, default=False,
@@ -795,7 +801,8 @@ web2py will attempt to run a GUI to ask for it
default=None, default=None,
help=\ help=\
'should be followed by a list of arguments to be passed to script, ' \ 'should be followed by a list of arguments to be passed to script, ' \
'to be used with -S, -A must be the last option') 'to be used with -S; NOTE: must be the last option because eat all ' \
'remaining arguments')
parser.add_option('--no-banner', dest='nobanner', parser.add_option('--no-banner', dest='nobanner',
default=False, default=False,
@@ -819,8 +826,8 @@ web2py will attempt to run a GUI to ask for it
default=False, default=False,
action='store_true', action='store_true',
help=\ help=\
'adds coverage reporting (needs --run_system_tests), ' \ 'adds coverage reporting (should be used with --run_system_tests), ' \
'python 2.7 and the coverage module installed. ' \ 'needs Python 2.7+ and the coverage module installed. ' \
'You can alter the default path setting the environment ' \ 'You can alter the default path setting the environment ' \
'variable "COVERAGE_PROCESS_START" ' \ 'variable "COVERAGE_PROCESS_START" ' \
'(by default it takes gluon/tests/coverage.ini)') '(by default it takes gluon/tests/coverage.ini)')
@@ -849,6 +856,7 @@ web2py will attempt to run a GUI to ask for it
if hasattr(options, key): if hasattr(options, key):
setattr(options, key, getattr(options2, key)) setattr(options, key, getattr(options2, key))
# store in options.ips the list of server IP addresses
try: try:
options.ips = list(set( # no duplicates options.ips = list(set( # no duplicates
[addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn()) [addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn())
@@ -862,10 +870,11 @@ web2py will attempt to run a GUI to ask for it
options.nobanner = True options.nobanner = True
options.nogui = True options.nogui = True
# accept --interfaces in the form # transform options.interfaces, in the form
# "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3" # "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3"
# (no spaces; optional key:cert:ca_cert indicate SSL) # (no spaces; optional key:cert:ca_cert indicate SSL), into
if isinstance(options.interfaces, str): # a list of tuples
if options.interfaces:
interfaces = options.interfaces.split(';') interfaces = options.interfaces.split(';')
options.interfaces = [] options.interfaces = []
for interface in interfaces: for interface in interfaces:
@@ -881,16 +890,16 @@ web2py will attempt to run a GUI to ask for it
interface[1] = int(interface[1]) # numeric port interface[1] = int(interface[1]) # numeric port
options.interfaces.append(tuple(interface)) options.interfaces.append(tuple(interface))
# accepts --scheduler in the form # strip group infos from options.scheduler, in the form
# "app:group1:group2,app2:group1" # "app:group1:group2,app2:group1", and put into a list of lists
scheduler = [] # in options.scheduler_groups
options.scheduler_groups = None if options.scheduler and ':' in options.scheduler:
if isinstance(options.scheduler, str): sg = options.scheduler_groups = []
if ':' in options.scheduler: for awg in options.scheduler.split(','):
for opt in options.scheduler.split(','): sg.append(awg.split(':'))
scheduler.append(opt.split(':')) options.scheduler = ','.join([app[0] for app in sg])
options.scheduler = ','.join([app[0] for app in scheduler]) else:
options.scheduler_groups = scheduler options.scheduler_groups = None
if options.numthreads is not None and options.minthreads is None: if options.numthreads is not None and options.minthreads is None:
options.minthreads = options.numthreads # legacy options.minthreads = options.numthreads # legacy
@@ -898,8 +907,6 @@ web2py will attempt to run a GUI to ask for it
copy_options = copy.deepcopy(options) copy_options = copy.deepcopy(options)
copy_options.password = '******' copy_options.password = '******'
global_settings.cmd_options = copy_options global_settings.cmd_options = copy_options
# FIXME: do we still really need this?
global_settings.cmd_args = args
return options, args return options, args
@@ -930,9 +937,8 @@ def start_schedulers(options):
sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
return return
processes = [] processes = []
apps = [(app.strip(), None) for app in options.scheduler.split(',')] apps = options.scheduler_groups or \
if options.scheduler_groups: [(app.strip(), None) for app in options.scheduler.split(',')]
apps = options.scheduler_groups
code = "from gluon.globals import current;current._scheduler.loop()" code = "from gluon.globals import current;current._scheduler.loop()"
logging.getLogger().setLevel(options.debuglevel) logging.getLogger().setLevel(options.debuglevel)
if options.folder: if options.folder:
@@ -1007,11 +1013,26 @@ def start(cron=True):
run_system_tests(options) run_system_tests(options)
if options.quiet: if options.quiet:
capture = StringIO() # to prevent writes on stdout set a null stream
sys.stdout = capture class NullFile(object):
logger.setLevel(logging.CRITICAL + 1) def write(self, x):
else: pass
logger.setLevel(options.debuglevel) sys.stdout = NullFile()
# but still has to mute existing loggers, to do that iterate
# over all existing loggers (root logger included) and remove
# all attached logging.StreamHandler instances currently
# streaming on sys.stdout or sys.stderr
loggers = [logging.getLogger()]
loggers.extend(logging.Logger.manager.loggerDict.values())
for logger in loggers:
if isinstance(logger, logging.PlaceHolder): continue
for h in logger.handlers[:]:
if isinstance(h, logging.StreamHandler) and \
h.stream in (sys.stdout, sys.stderr):
logger.removeHandler(h)
# NOTE: stderr.write() is still working
logger.setLevel(options.debuglevel)
if not options.nobanner: if not options.nobanner:
# banner # banner
@@ -1144,7 +1165,7 @@ end tell
ip = first_if[0] ip = first_if[0]
port = first_if[1] port = first_if[1]
if options.ssl_certificate or options.ssl_private_key: if options.ssl_certificate and options.ssl_private_key:
proto = 'https' proto = 'https'
else: else:
proto = 'http' proto = 'http'