dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better newcrow and widget, fixed some ssl related bugs, thanks Paolo

Browse Source
This commit is contained in:
mdipierro
2019-04-11 21:15:48 -07:00
parent b96c54cef9
commit 066d9c9ab5
4 changed files with 140 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
gluon/main.py
View File

@@ -55,7 +55,6 @@ web2py_path = global_settings.applications_parent # backward compatibility
create_missing_folders()
# set up logging for subsequent imports
import logging
import logging.config
# This needed to prevent exception on Python 2.5:
@@ -765,8 +764,8 @@ class HttpServer(object):
app_info=app_info,
min_threads=min_threads,
max_threads=max_threads,
queue_size=int(request_queue_size),
timeout=int(timeout),
queue_size=request_queue_size,
timeout=timeout,
handle_signals=False,
)

106
gluon/newcron.py
View File

@@ -17,7 +17,6 @@ import time
import sched
import re
import datetime
import platform
from functools import reduce
from gluon.settings import global_settings
from gluon import fileutils
@@ -87,7 +86,7 @@ class hardcron(threading.Thread):
def run(self):
s = sched.scheduler(time.time, time.sleep)
logger.info('Hard cron daemon started')
logger.info('hard cron daemon started')
while not _cron_stopping:
now = time.time()
s.enter(60 - now % 60, 1, self.launch, ())
@@ -133,7 +132,7 @@ class Token(object):
else:
locktime = 59.99
if portalocker.LOCK_EX is None:
logger.warning('WEB2PY CRON: Disabled because no file locking')
logger.warning('cron disabled because no file locking')
return None
self.master = fileutils.open_file(self.path, 'rb+')
try:
@@ -142,13 +141,14 @@ class Token(object):
try:
(start, stop) = pickle.load(self.master)
except:
(start, stop) = (0, 1)
start = 0
stop = 1
if startup or self.now - start > locktime:
ret = self.now
if not stop:
# this happens if previous cron job longer than 1 minute
logger.warning('WEB2PY CRON: Stale cron.master detected')
logger.debug('WEB2PY CRON: Acquiring lock')
logger.warning('stale cron.master detected')
logger.debug('acquiring lock')
self.master.seek(0)
pickle.dump((self.now, 0), self.master)
self.master.flush()
@@ -166,7 +166,7 @@ class Token(object):
ret = self.master.closed
if not self.master.closed:
portalocker.lock(self.master, portalocker.LOCK_EX)
logger.debug('WEB2PY CRON: Releasing cron lock')
logger.debug('releasing cron lock')
self.master.seek(0)
(start, stop) = pickle.load(self.master)
if start == self.now: # if this is my lock
@@ -241,12 +241,9 @@ def parsecronline(line):
class cronlauncher(threading.Thread):
def __init__(self, cmd, shell=True):
def __init__(self, cmd):
threading.Thread.__init__(self)
if platform.system() == 'Windows':
shell = False
self.cmd = cmd
self.shell = shell
def run(self):
import subprocess
@@ -258,8 +255,7 @@ class cronlauncher(threading.Thread):
proc = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=self.shell)
stderr=subprocess.PIPE)
_cron_subprocs.append(proc)
(stdoutdata, stderrdata) = proc.communicate()
try:
@@ -267,15 +263,14 @@ class cronlauncher(threading.Thread):
except ValueError:
pass
if proc.returncode != 0:
logger.warning(
'WEB2PY CRON Call returned code %s:\n%s' %
(proc.returncode, stdoutdata + stderrdata))
logger.warning('call returned code %s:\n%s\n%s',
proc.returncode, stdoutdata, stderrdata)
else:
logger.debug('WEB2PY CRON Call returned success:\n%s'
% stdoutdata)
logger.debug('call returned success:\n%s', stdoutdata)
def crondance(applications_parent, ctype='soft', startup=False, apps=None):
# TODO: docstring
apppath = os.path.join(applications_parent, 'applications')
token = Token(applications_parent)
cronmaster = token.acquire(startup=startup)
@@ -294,6 +289,21 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
full_apath_links = set()
if sys.executable.lower().endswith('pythonservice.exe'):
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
else:
_python_exe = sys.executable
base_commands = [_python_exe]
w2p_path = fileutils.abspath('web2py.py', gluon=True)
if os.path.exists(w2p_path):
base_commands.append(w2p_path)
if applications_parent != global_settings.gluon_parent:
base_commands.extend(('-f', applications_parent))
base_commands.extend(('-J',
# FIXME: this should not be needed since we are
# not launching the web server
'-a', '"<recycle>"'))
for app in apps:
if _cron_stopping:
break
@@ -315,22 +325,12 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
lines = [line for line in cronlines if line and not line.startswith('#')]
tasks = [parsecronline(cline) for cline in lines]
except Exception as e:
logger.error('WEB2PY CRON: crontab read error %s' % e)
logger.error('crontab read error %s', e)
continue
for task in tasks:
if _cron_stopping:
break
if sys.executable.lower().endswith('pythonservice.exe'):
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
else:
_python_exe = sys.executable
commands = [_python_exe]
w2p_path = fileutils.abspath('web2py.py', gluon=True)
if os.path.exists(w2p_path):
commands.append(w2p_path)
if applications_parent != global_settings.gluon_parent:
commands.extend(('-f', applications_parent))
citems = [(k in task and not v in task[k]) for k, v in checks]
task_min = task.get('min', [])
if not task:
@@ -339,40 +339,32 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
continue
elif task_min != [-1] and reduce(lambda a, b: a or b, citems):
continue
logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s'
% (ctype, app, task.get('cmd'),
os.getcwd(), datetime.datetime.now()))
action, command, models = False, task['cmd'], ''
logger.info('%s cron: %s executing %s in %s at %s',
ctype, app, task.get('cmd'),
os.getcwd(), datetime.datetime.now())
action = models = False
command = task['cmd']
if command.startswith('**'):
(action, models, command) = (True, '', command[2:])
action = True
command = command[2:]
elif command.startswith('*'):
(action, models, command) = (True, '-M', command[1:])
else:
action = False
action = models = True
command = command[1:]
if action and command.endswith('.py'):
commands.extend(('-J', # cron job
models, # import models?
'-S', app, # app name
'-a', '"<recycle>"', # password
'-R', command)) # command
elif action:
commands.extend(('-J', # cron job
models, # import models?
'-S', app + '/' + command, # app name
'-a', '"<recycle>"')) # password
if action:
commands = base_commands[:]
if command.endswith('.py'):
commands.extend(('-S', app, '-R', command))
else:
commands.extend(('-S', app + '/' + command))
if models:
commands.append('-M')
else:
commands = command
# from python docs:
# You do not need shell=True to run a batch file or
# console-based executable.
shell = False
try:
cronlauncher(commands, shell=shell).start()
cronlauncher(commands).start()
except Exception as e:
logger.warning(
'WEB2PY CRON: Execution error for %s: %s'
% (task.get('cmd'), e))
logger.warning('execution error for %s: %s',
task.get('cmd'), e)
token.release()

2
gluon/packages/dal

Submodule gluon/packages/dal updated: cecd77127c...37784cb6aa

155
gluon/widget.py
View File

@@ -10,7 +10,7 @@ The widget is called from web2py
"""
import sys
from gluon._compat import StringIO, thread, xrange, PY2
from gluon._compat import thread, xrange, PY2
import time
import threading
import os
@@ -465,7 +465,7 @@ class web2pyDialog(object):
except:
return self.error('invalid port number')
if self.options.ssl_certificate or self.options.ssl_private_key:
if self.options.ssl_certificate and self.options.ssl_private_key:
proto = 'https'
else:
proto = 'http'
@@ -583,18 +583,18 @@ def console():
version=ProgramVersion,
description='web2py Web Framework startup script.',
epilog='''NOTE: unless a password is specified (-a 'passwd')
web2py will attempt to run a GUI to ask for it
web2py will attempt to run a GUI to ask for it when starting the web server
(if not disabled with --nogui).''')
parser.add_option('-i', '--ip',
default='127.0.0.1',
help=\
metavar='IP_ADDR', help=\
'IP address of the server (e.g., 127.0.0.1 or ::1); ' \
'Note: This value is ignored when using the --interfaces option')
parser.add_option('-p', '--port',
default=8000,
type='int', help=\
type='int', metavar='NUM', help=\
'port of server (%default); ' \
'Note: This value is ignored when using the --interfaces option')
@@ -607,43 +607,43 @@ web2py will attempt to run a GUI to ask for it
default='<ask>',
help=\
'password to be used for administration ' \
'(use -a "<recycle>" to reuse the last password))')
'(use "<recycle>" to reuse the last password), ' \
'when no password is available the administrative ' \
'interface will be disabled')
parser.add_option('-c', '--ssl_certificate',
default='',
help='file that contains ssl certificate')
default=None,
metavar='FILE', help='server certificate file')
parser.add_option('-k', '--ssl_private_key',
default='',
help='file that contains ssl private key')
default=None,
metavar='FILE', help='server private key file')
parser.add_option('--ca-cert', dest='ssl_ca_certificate',
default=None,
help=\
'use this file containing the CA certificate to validate X509 ' \
'certificates from clients')
metavar='FILE', help='CA certificate file')
parser.add_option('-d', '--pid_filename',
default='httpserver.pid',
help='file to store the pid of the server')
metavar='FILE', help='server pid file (%default)')
parser.add_option('-l', '--log_filename',
default='httpserver.log',
help='name for the server log file')
metavar='FILE', help='server log file (%default)')
parser.add_option('-n', '--numthreads',
default=None,
type='int',
type='int', metavar='NUM',
help='number of threads (deprecated)')
parser.add_option('--minthreads',
default=None,
type='int',
type='int', metavar='NUM',
help='minimum number of server threads')
parser.add_option('--maxthreads',
default=None,
type='int',
type='int', metavar='NUM',
help='maximum number of server threads')
parser.add_option('-s', '--server_name',
@@ -651,28 +651,30 @@ web2py will attempt to run a GUI to ask for it
help='web server name (%default)')
parser.add_option('-q', '--request_queue_size',
default='5',
type='int',
default=5,
type='int', metavar='NUM',
help=\
'max number of queued requests when server unavailable')
'max number of queued requests when server unavailable (%default)')
parser.add_option('-o', '--timeout',
default='10',
type='int',
default=10,
type='int', metavar='SECONDS',
help='timeout for individual request (%default seconds)')
parser.add_option('-z', '--shutdown_timeout',
default='5',
type='int',
help='timeout on shutdown of server (%default seconds)')
default=None,
type='int', metavar='SECONDS',
help=\
'timeout on server shutdown; this value is not used by ' \
'Rocket web server')
parser.add_option('--socket-timeout', dest='socket_timeout', # not needed
default=5,
type='int',
type='int', metavar='SECONDS',
help='timeout for socket (%default seconds)')
parser.add_option('-f', '--folder',
default=os.getcwd(),
default=os.getcwd(), metavar='WEB2PY_DIR',
help='folder from which to run web2py')
parser.add_option('-v', '--verbose',
@@ -693,8 +695,8 @@ web2py will attempt to run a GUI to ask for it
parser.add_option('-D', '--debug', dest='debuglevel',
default=30,
type='int',
help=\
'set debug output level (0-100, 0 means all, 100 means none; ' \
metavar='LOG_LEVEL', help=\
'set log level (0-100, 0 means all, 100 means none; ' \
'default is %default)')
parser.add_option('-S', '--shell',
@@ -722,7 +724,7 @@ web2py will attempt to run a GUI to ask for it
default=False,
action='store_true',
help=\
'auto import model files; default is %default; should be used ' \
'auto import model files (default is %default); should be used ' \
'with --shell option')
parser.add_option('-R', '--run',
@@ -733,18 +735,18 @@ web2py will attempt to run a GUI to ask for it
parser.add_option('-K', '--scheduler',
default=None,
help=\
metavar='APP_LIST', help=\
'run scheduled tasks for the specified apps: expects a list of ' \
'app names as -K app1,app2,app3 ' \
'or a list of app:groups as -K app1:group1:group2,app2:group1 ' \
'to override specific group_names. (only strings, no spaces ' \
'allowed. Requires a scheduler defined in the models')
'app names as app1,app2,app3 ' \
'or a list of app:groups as app1:group1:group2,app2:group1 ' \
'(only strings, no spaces allowed). NOTE: ' \
'Requires a scheduler defined in the models')
parser.add_option('-X', '--with-scheduler', dest='with_scheduler', # not needed
default=False,
action='store_true',
help=\
'run schedulers alongside webserver, needs -K app1 and -a too')
'run schedulers alongside webserver, needs -K')
parser.add_option('-T', '--test',
default=None,
@@ -756,12 +758,16 @@ web2py will attempt to run a GUI to ask for it
default=False,
action='store_true',
help=\
'trigger a cron run manually; usually invoked from a system crontab')
'trigger a cron run and exit; usually used when invoked ' \
'from a system crontab')
parser.add_option('--softcron',
default=False,
action='store_true',
help='triggers the use of softcron')
help=\
'use software cron emulation instead of separate cron process, '\
'needs -Y; NOTE: use of software cron emulation is strongly '
'discouraged')
parser.add_option('-Y', '--run-cron', dest='runcron',
default=False,
@@ -795,7 +801,8 @@ web2py will attempt to run a GUI to ask for it
default=None,
help=\
'should be followed by a list of arguments to be passed to script, ' \
'to be used with -S, -A must be the last option')
'to be used with -S; NOTE: must be the last option because eat all ' \
'remaining arguments')
parser.add_option('--no-banner', dest='nobanner',
default=False,
@@ -819,8 +826,8 @@ web2py will attempt to run a GUI to ask for it
default=False,
action='store_true',
help=\
'adds coverage reporting (needs --run_system_tests), ' \
'python 2.7 and the coverage module installed. ' \
'adds coverage reporting (should be used with --run_system_tests), ' \
'needs Python 2.7+ and the coverage module installed. ' \
'You can alter the default path setting the environment ' \
'variable "COVERAGE_PROCESS_START" ' \
'(by default it takes gluon/tests/coverage.ini)')
@@ -849,6 +856,7 @@ web2py will attempt to run a GUI to ask for it
if hasattr(options, key):
setattr(options, key, getattr(options2, key))
# store in options.ips the list of server IP addresses
try:
options.ips = list(set( # no duplicates
[addrinfo[4][0] for addrinfo in getipaddrinfo(socket.getfqdn())
@@ -862,10 +870,11 @@ web2py will attempt to run a GUI to ask for it
options.nobanner = True
options.nogui = True
# accept --interfaces in the form
# "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3"
# (no spaces; optional key:cert:ca_cert indicate SSL)
if isinstance(options.interfaces, str):
# transform options.interfaces, in the form
# "ip1:port1:key1:cert1:ca_cert1;[ip2]:port2;ip3:port3:key3:cert3"
# (no spaces; optional key:cert:ca_cert indicate SSL), into
# a list of tuples
if options.interfaces:
interfaces = options.interfaces.split(';')
options.interfaces = []
for interface in interfaces:
@@ -881,16 +890,16 @@ web2py will attempt to run a GUI to ask for it
interface[1] = int(interface[1]) # numeric port
options.interfaces.append(tuple(interface))
# accepts --scheduler in the form
# "app:group1:group2,app2:group1"
scheduler = []
options.scheduler_groups = None
if isinstance(options.scheduler, str):
if ':' in options.scheduler:
for opt in options.scheduler.split(','):
scheduler.append(opt.split(':'))
options.scheduler = ','.join([app[0] for app in scheduler])
options.scheduler_groups = scheduler
# strip group infos from options.scheduler, in the form
# "app:group1:group2,app2:group1", and put into a list of lists
# in options.scheduler_groups
if options.scheduler and ':' in options.scheduler:
sg = options.scheduler_groups = []
for awg in options.scheduler.split(','):
sg.append(awg.split(':'))
options.scheduler = ','.join([app[0] for app in sg])
else:
options.scheduler_groups = None
if options.numthreads is not None and options.minthreads is None:
options.minthreads = options.numthreads # legacy
@@ -898,8 +907,6 @@ web2py will attempt to run a GUI to ask for it
copy_options = copy.deepcopy(options)
copy_options.password = '******'
global_settings.cmd_options = copy_options
# FIXME: do we still really need this?
global_settings.cmd_args = args
return options, args
@@ -930,9 +937,8 @@ def start_schedulers(options):
sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
return
processes = []
apps = [(app.strip(), None) for app in options.scheduler.split(',')]
if options.scheduler_groups:
apps = options.scheduler_groups
apps = options.scheduler_groups or \
[(app.strip(), None) for app in options.scheduler.split(',')]
code = "from gluon.globals import current;current._scheduler.loop()"
logging.getLogger().setLevel(options.debuglevel)
if options.folder:
@@ -1007,11 +1013,26 @@ def start(cron=True):
run_system_tests(options)
if options.quiet:
capture = StringIO()
sys.stdout = capture
logger.setLevel(logging.CRITICAL + 1)
else:
logger.setLevel(options.debuglevel)
# to prevent writes on stdout set a null stream
class NullFile(object):
def write(self, x):
pass
sys.stdout = NullFile()
# but still has to mute existing loggers, to do that iterate
# over all existing loggers (root logger included) and remove
# all attached logging.StreamHandler instances currently
# streaming on sys.stdout or sys.stderr
loggers = [logging.getLogger()]
loggers.extend(logging.Logger.manager.loggerDict.values())
for logger in loggers:
if isinstance(logger, logging.PlaceHolder): continue
for h in logger.handlers[:]:
if isinstance(h, logging.StreamHandler) and \
h.stream in (sys.stdout, sys.stderr):
logger.removeHandler(h)
# NOTE: stderr.write() is still working
logger.setLevel(options.debuglevel)
if not options.nobanner:
# banner
@@ -1144,7 +1165,7 @@ end tell
ip = first_if[0]
port = first_if[1]
if options.ssl_certificate or options.ssl_private_key:
if options.ssl_certificate and options.ssl_private_key:
proto = 'https'
else:
proto = 'http'