Merge branch 'master' into dev-ah

gluon/main.py

@@ -554,10 +554,12 @@ def wsgibase(environ, responder):
         http_response, request, environ, ticket)
     if not http_response:
         return wsgibase(new_environ, responder)
+
     if global_settings.web2py_crontype == 'soft':
         cmd_opts = global_settings.cmd_options
         newcron.softcron(global_settings.applications_parent,
-                         apps=cmd_opts and cmd_opts.crontabs).start()
+                         apps=cmd_opts and cmd_opts.crontabs)
+
     return http_response.to(responder, env=env)
 
 
@@ -783,7 +785,11 @@ class HttpServer(object):
         """
         stop cron and the web server
         """
-        newcron.stopcron()
+        if global_settings.web2py_crontype == 'soft':
+            try:
+                newcron.stopcron()
+            except:
+                pass
         self.server.stop(stoplogging)
         try:
             os.unlink(self.pid_filename)

gluon/newcron.py

@@ -5,6 +5,7 @@
 | This file is part of the web2py Web Framework
 | Created by Attila Csipa <web2py@csipa.in.rs>
 | Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu>
+| Worker, SoftWorker and SimplePool added by Paolo Pastori
 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
 
 Cron-style interface
@@ -12,16 +13,15 @@ Cron-style interface
 
 import threading
 import os
-from logging import getLogger, DEBUG
+from logging import getLogger
 import time
 import sched
 import sys
 import re
+from functools import reduce
+import datetime
 import shlex
 
-import datetime
-from functools import reduce
-from gluon.settings import global_settings
 from gluon import fileutils
 from gluon._compat import to_bytes, pickle
 from pydal.contrib import portalocker
@@ -29,14 +29,19 @@ from pydal.contrib import portalocker
 logger_name = 'web2py.cron'
 
 
-_cron_stopping = False
+_stopping = False
 
-_cron_subprocs_lock = threading.RLock()
-_cron_subprocs = []
+def reset():
+    global _stopping
+    _stopping = False
+
+
+_subprocs_lock = threading.RLock()
+_subprocs = []
 
 def subprocess_count():
-    with _cron_subprocs_lock:
-        return len(_cron_subprocs)
+    with _subprocs_lock:
+        return len(_subprocs)
 
 
 def absolute_path_link(path):
@@ -54,11 +59,11 @@ def absolute_path_link(path):
 
 def stopcron():
     """Graceful shutdown of cron"""
-    global _cron_stopping
-    _cron_stopping = True
+    global _stopping
+    _stopping = True
     while subprocess_count():
-        with _cron_subprocs_lock:
-            proc = _cron_subprocs.pop()
+        with _subprocs_lock:
+            proc = _subprocs.pop()
         if proc.poll() is None:
             try:
                 proc.terminate()
@@ -66,18 +71,9 @@ def stopcron():
                 getLogger(logger_name).exception('error in stopcron')
 
 
-class extcron(threading.Thread):
-
-    def __init__(self, applications_parent, apps=None):
-        threading.Thread.__init__(self)
-        self.setDaemon(False)
-        self.path = applications_parent
-        self.apps = apps
-
-    def run(self):
-        if not _cron_stopping:
-            getLogger(logger_name).debug('external cron invocation')
-            crondance(self.path, 'external', startup=False, apps=self.apps)
+def extcron(applications_parent, apps=None):
+    getLogger(logger_name).debug('external cron invocation')
+    crondance(applications_parent, 'external', startup=False, apps=apps)
 
 
 class hardcron(threading.Thread):
@@ -92,7 +88,7 @@ class hardcron(threading.Thread):
         crondance(self.path, 'hard', startup=True, apps=self.apps)
 
     def launch(self):
-        if not _cron_stopping:
+        if not _stopping:
             self.logger.debug('hard cron invocation')
             crondance(self.path, 'hard', startup=False, apps=self.apps)
 
@@ -100,26 +96,21 @@ class hardcron(threading.Thread):
         self.logger = getLogger(logger_name)
         self.logger.info('hard cron daemon started')
         s = sched.scheduler(time.time, time.sleep)
-        while not _cron_stopping:
+        while not _stopping:
             now = time.time()
             s.enter(60 - now % 60, 1, self.launch, ())
             s.run()
 
 
-class softcron(threading.Thread):
-
-    def __init__(self, applications_parent, apps=None):
-        threading.Thread.__init__(self)
-        self.path = applications_parent
-        self.apps = apps
-
-    def run(self):
-        if not _cron_stopping:
-            getLogger(logger_name).debug('soft cron invocation')
-            crondance(self.path, 'soft', startup=False, apps=self.apps)
+def softcron(applications_parent, apps=None):
+    logger = getLogger(logger_name)
+    try:
+        if not _dancer((applications_parent, apps)):
+            logger.warning('no thread available for soft crondance')
+    except Exception:
+        logger.exception('error executing soft crondance')
 
 
-# TODO: context manager
 class Token(object):
 
     def __init__(self, path):
@@ -140,7 +131,7 @@ class Token(object):
         stop == 0 if job started but did not yet complete
         if a cron job started within less than 60 seconds, acquire returns None
         if a cron job started before 60 seconds and did not stop,
-        a warning is issue "Stale cron.master detected"
+        a warning is issued ("Stale cron.master detected")
         """
         if sys.platform == 'win32':
             locktime = 59.5
@@ -150,8 +141,8 @@ class Token(object):
             self.logger.warning('cron disabled because no file locking')
             return None
         self.master = fileutils.open_file(self.path, 'rb+')
+        ret = None
         try:
-            ret = None
             portalocker.lock(self.master, portalocker.LOCK_EX)
             try:
                 (start, stop) = pickle.load(self.master)
@@ -254,36 +245,144 @@ def parsecronline(line):
     return task
 
 
-class cronlauncher(threading.Thread):
+class Worker(threading.Thread):
 
-    def __init__(self, cmd):
+    def __init__(self, pool):
         threading.Thread.__init__(self)
-        self.cmd = cmd
+        self.setDaemon(True)
+        self.pool = pool
+        self.run_lock = threading.Lock()
+        self.run_lock.acquire()
+        self.payload = None
 
     def run(self):
-        import subprocess
         logger = getLogger(logger_name)
-        proc = subprocess.Popen(self.cmd,
-                                stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-        with _cron_subprocs_lock:
-            _cron_subprocs.append(proc)
-        (stdoutdata, stderrdata) = proc.communicate()
-        try:
-            with _cron_subprocs_lock:
-                _cron_subprocs.remove(proc)
-        except ValueError:
-            pass
-        if proc.returncode != 0:
-            logger.warning('%r call returned code %s:\n%s\n%s',
-                ' '.join(self.cmd), proc.returncode, stdoutdata, stderrdata)
-        elif logger.isEnabledFor(DEBUG):
-            logger.debug('%r call returned success:\n%s',
-                ' '.join(self.cmd), stdoutdata)
+        logger.info('Worker %s: started', self.name)
+        while True:
+            try:
+                with self.run_lock:  # waiting for run_lock.release()
+                    cmd = ' '.join(self.payload)
+                    logger.debug('Worker %s: now calling %r', self.name, cmd)
+                    import subprocess
+                    proc = subprocess.Popen(self.payload,
+                                    stdin=subprocess.PIPE,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+                    with _subprocs_lock:
+                        _subprocs.append(proc)
+                    stdoutdata, stderrdata = proc.communicate()
+                    try:
+                        with _subprocs_lock:
+                            _subprocs.remove(proc)
+                    except ValueError:
+                        pass
+                    if proc.returncode != 0:
+                        logger.warning('Worker %s: %r call returned code %s:\n%s\n%s',
+                            self.name, cmd, proc.returncode, stdoutdata, stderrdata)
+                    else:
+                        logger.debug('Worker %s: %r call returned success:\n%s',
+                            self.name, cmd, stdoutdata)
+            finally:
+                self.run_lock.acquire()
+                self.pool.stop(self)
 
 
-def crondance(applications_parent, ctype='soft', startup=False, apps=None):
+class SoftWorker(threading.Thread):
+
+    def __init__(self, pool):
+        threading.Thread.__init__(self)
+        self.setDaemon(True)
+        self.pool = pool
+        self.run_lock = threading.Lock()
+        self.run_lock.acquire()
+        self.payload = None
+
+    def run(self):
+        logger = getLogger(logger_name)
+        logger.info('SoftWorker %s: started', self.name)
+        while True:
+            try:
+                with self.run_lock:  # waiting for run_lock.release()
+                    getLogger(logger_name).debug('soft cron invocation')
+                    applications_parent, apps = self.payload
+                    crondance(applications_parent, 'soft', startup=False, apps=apps)
+            finally:
+                self.run_lock.acquire()
+                self.pool.stop(self)
+
+
+class SimplePool(object):
+    """
+    Very simple thread pool,
+    (re)uses a maximum number of threads to launch cron tasks.
+
+    Pool size can be incremented after initialization,
+    this allows delayed configuration of a global instance
+    for the case you do not want to use lazy initialization.
+    """
+
+    def __init__(self, size, worker_cls=Worker):
+        """
+        Create the pool setting initial size.
+
+        Notice that no thread is created until the instance is called.
+        """
+        self.size = size
+        self.worker_cls = worker_cls
+        self.lock = threading.RLock()
+        self.idle = list()
+        self.running = set()
+
+    def grow(self, size):
+        if size > self.size:
+            self.size = size
+
+    def start(self, t):
+        with self.lock:
+            try:
+                self.idle.remove(t)
+            except ValueError:
+                pass
+            self.running.add(t)
+
+    def stop(self, t):
+        with self.lock:
+            self.idle.append(t)
+            try:
+                self.running.remove(t)
+            except KeyError:
+                pass
+
+    def __call__(self, payload):
+        """
+        Pass payload to a thread for immediate execution.
+
+        Returns a boolean indicating if a thread is available.
+        """
+        with self.lock:
+            if len(self.running) == self.size:
+                # no worker available
+                return False
+            idle_num = len(self.idle)
+            if idle_num:
+                # use an existing (idle) thread
+                t = self.idle.pop(0)
+            else:
+                # create a new thread
+                t = self.worker_cls(self)
+            self.start(t)
+        t.payload = payload
+        t.run_lock.release()
+        if not idle_num:
+            t.start()
+        return True
+
+
+_dancer = SimplePool(5, worker_cls=SoftWorker)
+
+_launcher = SimplePool(5)
+
+def crondance(applications_parent, ctype='hard', startup=False, apps=None):
     """
     Does the periodic job of cron service: read the crontab(s) and launch
     the various commands.
@@ -293,96 +392,94 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
     cronmaster = token.acquire(startup=startup)
     if not cronmaster:
         return
-    now_s = time.localtime()
-    checks = (('min', now_s.tm_min),
-              ('hr', now_s.tm_hour),
-              ('mon', now_s.tm_mon),
-              ('dom', now_s.tm_mday),
-              ('dow', (now_s.tm_wday + 1) % 7))
+    try:
+        now_s = time.localtime()
+        checks = (('min', now_s.tm_min),
+                  ('hr', now_s.tm_hour),
+                  ('mon', now_s.tm_mon),
+                  ('dom', now_s.tm_mday),
+                  ('dow', (now_s.tm_wday + 1) % 7))
+        logger = getLogger(logger_name)
 
-    logger = getLogger(logger_name)
+        if not apps:
+            apps = [x for x in os.listdir(apppath)
+                    if os.path.isdir(os.path.join(apppath, x))]
 
-    if not apps:
-        apps = [x for x in os.listdir(apppath)
-                if os.path.isdir(os.path.join(apppath, x))]
+        full_apath_links = set()
 
-    full_apath_links = set()
-
-    if sys.executable.lower().endswith('pythonservice.exe'):
-        _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
-    else:
-        _python_exe = sys.executable
-    base_commands = [_python_exe]
-    w2p_path = fileutils.abspath('web2py.py', gluon=True)
-    if os.path.exists(w2p_path):
-        base_commands.append(w2p_path)
-    if applications_parent != global_settings.gluon_parent:
-        base_commands.extend(('-f', applications_parent))
-    base_commands.extend(('--cron_job', '--no_banner', '--no_gui', '--plain'))
-
-    for app in apps:
-        if _cron_stopping:
-            break
-        apath = os.path.join(apppath, app)
-
-        # if app is a symbolic link to other app, skip it
-        full_apath_link = absolute_path_link(apath)
-        if full_apath_link in full_apath_links:
-            continue
+        if sys.executable.lower().endswith('pythonservice.exe'):
+            _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
         else:
-            full_apath_links.add(full_apath_link)
+            _python_exe = sys.executable
+        base_commands = [_python_exe]
+        w2p_path = fileutils.abspath('web2py.py', gluon=True)
+        if os.path.exists(w2p_path):
+            base_commands.append(w2p_path)
+        base_commands.extend(('--cron_job', '--no_banner', '--no_gui', '--plain'))
 
-        cronpath = os.path.join(apath, 'cron')
-        crontab = os.path.join(cronpath, 'crontab')
-        if not os.path.exists(crontab):
-            continue
-        try:
-            cronlines = [line.strip() for line in fileutils.readlines_file(crontab, 'rt')]
-            lines = [line for line in cronlines if line and not line.startswith('#')]
-            tasks = [parsecronline(cline) for cline in lines]
-        except Exception as e:
-            logger.error('crontab read error %s', e)
-            continue
-
-        for task in tasks:
-            if _cron_stopping:
+        for app in apps:
+            if _stopping:
                 break
-            if not task:
-                continue
-            task_min = task.get('min', [])
-            if not startup and task_min == [-1]:
-                continue
-            citems = [(k in task and not v in task[k]) for k, v in checks]
-            if task_min != [-1] and reduce(lambda a, b: a or b, citems):
-                continue
+            apath = os.path.join(apppath, app)
 
-            logger.info('%s cron: %s executing %r in %s at %s',
-                ctype, app, task.get('cmd'),
-                os.getcwd(), datetime.datetime.now())
-            action = models = False
-            command = task['cmd']
-            if command.startswith('**'):
-                action = True
-                command = command[2:]
-            elif command.startswith('*'):
-                action = models = True
-                command = command[1:]
-
-            if action:
-                commands = base_commands[:]
-                if command.endswith('.py'):
-                    commands.extend(('-S', app, '-R', command))
-                else:
-                    commands.extend(('-S', app + '/' + command))
-                if models:
-                    commands.append('-M')
+            # if app is a symbolic link to other app, skip it
+            full_apath_link = absolute_path_link(apath)
+            if full_apath_link in full_apath_links:
+                continue
             else:
-                commands = shlex.split(command)
+                full_apath_links.add(full_apath_link)
 
+            cronpath = os.path.join(apath, 'cron')
+            crontab = os.path.join(cronpath, 'crontab')
+            if not os.path.exists(crontab):
+                continue
             try:
-                # FIXME: using a new thread every time there is a task to
-                #        launch is not a good idea in a long running process
-                cronlauncher(commands).start()
-            except Exception:
-                logger.exception('error starting %r', task['cmd'])
-    token.release()
+                cronlines = [line.strip() for line in fileutils.readlines_file(crontab, 'rt')]
+                lines = [line for line in cronlines if line and not line.startswith('#')]
+                tasks = [parsecronline(cline) for cline in lines]
+            except Exception as e:
+                logger.error('crontab read error %s', e)
+                continue
+
+            for task in tasks:
+                if _stopping:
+                    break
+                if not task:
+                    continue
+                task_min = task.get('min', [])
+                if not startup and task_min == [-1]:
+                    continue
+                citems = [(k in task and not v in task[k]) for k, v in checks]
+                if task_min != [-1] and reduce(lambda a, b: a or b, citems):
+                    continue
+
+                logger.info('%s cron: %s executing %r in %s at %s',
+                    ctype, app, task.get('cmd'),
+                    os.getcwd(), datetime.datetime.now())
+                action = models = False
+                command = task['cmd']
+                if command.startswith('**'):
+                    action = True
+                    command = command[2:]
+                elif command.startswith('*'):
+                    action = models = True
+                    command = command[1:]
+
+                if action:
+                    commands = base_commands[:]
+                    if command.endswith('.py'):
+                        commands.extend(('-S', app, '-R', command))
+                    else:
+                        commands.extend(('-S', app + '/' + command))
+                    if models:
+                        commands.append('-M')
+                else:
+                    commands = shlex.split(command)
+
+                try:
+                    if not _launcher(commands):
+                        logger.warning('no thread available, cannot execute %r', task['cmd'])
+                except Exception:
+                    logger.exception('error executing %r', task['cmd'])
+    finally:
+        token.release()

gluon/tests/test_cron.py

@@ -8,8 +8,10 @@ import unittest
 import os
 import shutil
 import time
+import sys
 
-from gluon.newcron import Token, crondance, subprocess_count
+from gluon.newcron import (Token, crondance, subprocess_count,
+    SimplePool, stopcron, reset)
 from gluon.fileutils import create_app, write_file
 
 
@@ -28,13 +30,18 @@ def tearDownModule():
 
 TEST_CRONTAB = """@reboot peppe **applications/%s/cron/test.py
 """ %  test_app_name
-TEST_SCRIPT = """
+TEST_SCRIPT1 = """
 from os.path import join as pjoin
 
 with open(pjoin(request.folder, 'private', 'cron_req'), 'w') as f:
     f.write(str(request))
 """
 TARGET = os.path.join(appdir, 'private', 'cron_req')
+TEST_SCRIPT2 = """
+import time
+
+time.sleep(13)
+"""
 
 class TestCron(unittest.TestCase):
 
@@ -44,7 +51,7 @@ class TestCron(unittest.TestCase):
             if os.path.exists(master):
                 os.unlink(master)
 
-    def test_Token(self):
+    def test_1_Token(self):
         app_path = os.path.join(os.getcwd(), 'applications', test_app_name)
         t = Token(path=app_path)
         self.assertEqual(t.acquire(), t.now)
@@ -52,10 +59,10 @@ class TestCron(unittest.TestCase):
         self.assertIsNone(t.acquire())
         self.assertTrue(t.release())
 
-    def test_crondance(self):
+    def test_2_crondance(self):
         base = os.path.join(appdir, 'cron')
         write_file(os.path.join(base, 'crontab'), TEST_CRONTAB)
-        write_file(os.path.join(base, 'test.py'), TEST_SCRIPT)
+        write_file(os.path.join(base, 'test.py'), TEST_SCRIPT1)
         if os.path.exists(TARGET):
             os.unlink(TARGET)
         crondance(os.getcwd(), 'hard', startup=True, apps=[test_app_name])
@@ -65,3 +72,19 @@ class TestCron(unittest.TestCase):
             time.sleep(1)
         time.sleep(1)
         self.assertTrue(os.path.exists(TARGET))
+
+    def test_3_SimplePool(self):
+        base = os.path.join(appdir, 'cron')
+        write_file(os.path.join(base, 'test.py'), TEST_SCRIPT2)
+        w2p_path = os.path.join(os.getcwd(), 'web2py.py')
+        self.assertTrue(os.path.exists(w2p_path))
+        launcher = SimplePool(1)
+        cmd1 = [sys.executable, w2p_path,
+            '--cron_job', '--no_banner', '--no_gui', '--plain',
+            '-S', test_app_name, '-R', "applications/%s/cron/test.py" % test_app_name]
+        self.assertTrue(launcher(cmd1))
+        self.assertFalse(launcher(None))
+        time.sleep(1)
+        stopcron()
+        time.sleep(1)
+        reset()

gluon/widget.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
 
 """
 | This file is part of the web2py Web Framework
@@ -11,23 +12,26 @@ GUI widget and services start function
 
 from __future__ import print_function
 
-import sys
-from gluon._compat import thread, xrange, PY2
 import time
-import threading
+import sys
 import os
+from collections import OrderedDict
 import socket
-import signal
+import threading
 import math
 import logging
+import signal
 import getpass
 
-from gluon import main, newcron
 from gluon.fileutils import read_file, create_welcome_w2p
-from gluon.console import console
-from gluon.settings import global_settings
 from gluon.shell import die, run, test
-from gluon.utils import is_valid_ip_address, is_loopback_ip_address, getipaddrinfo
+from gluon._compat import PY2, xrange
+from gluon.utils import (getipaddrinfo, is_loopback_ip_address,
+    is_valid_ip_address)
+from gluon.console import is_appdir, console
+from gluon import newcron
+from gluon import main
+from gluon.settings import global_settings
 
 
 ProgramName = 'web2py Web Framework'
@@ -119,24 +123,27 @@ class web2pyDialog(object):
             import tkinter
             from tkinter import messagebox
 
-
-        bg_color = 'white'
         root.withdraw()
 
+        bg_color = 'white'
         self.root = tkinter.Toplevel(root, bg=bg_color)
         self.root.resizable(0, 0)
         self.root.title(ProgramName)
 
         self.options = options
-        self.scheduler_processes = {}
-        self.menu = tkinter.Menu(self.root)
-        servermenu = tkinter.Menu(self.menu, tearoff=0)
-        httplog = os.path.join(self.options.folder, self.options.log_filename)
+        self.scheduler_processes_lock = threading.RLock()
+        self.scheduler_processes = OrderedDict()
+
         iconphoto = os.path.join('extras', 'icons', 'web2py.gif')
         if os.path.exists(iconphoto):
             img = tkinter.PhotoImage(file=iconphoto)
             self.root.tk.call('wm', 'iconphoto', self.root._w, img)
+
         # Building the Menu
+        self.menu = tkinter.Menu(self.root)
+        servermenu = tkinter.Menu(self.menu, tearoff=0)
+
+        httplog = os.path.join(options.folder, options.log_filename)
         item = lambda: start_browser(httplog)
         servermenu.add_command(label='View httpserver.log',
                                command=item)
@@ -149,10 +156,9 @@ class web2pyDialog(object):
         self.pagesmenu = tkinter.Menu(self.menu, tearoff=0)
         self.menu.add_cascade(label='Pages', menu=self.pagesmenu)
 
-        #scheduler menu
         self.schedmenu = tkinter.Menu(self.menu, tearoff=0)
         self.menu.add_cascade(label='Scheduler', menu=self.schedmenu)
-        #start and register schedulers from options
+        # register and start schedulers
         self.update_schedulers(start=True)
 
         helpmenu = tkinter.Menu(self.menu, tearoff=0)
@@ -257,7 +263,7 @@ class web2pyDialog(object):
                                                   sticky=sticky)
 
         self.port_number = tkinter.Entry(self.root)
-        self.port_number.insert(tkinter.END, self.options.port)
+        self.port_number.insert(tkinter.END, options.port)
         self.port_number.grid(row=shift, column=2, sticky=sticky, pady=10)
 
         # Password
@@ -314,6 +320,26 @@ class web2pyDialog(object):
             self.tb = None
 
     def update_schedulers(self, start=False):
+        applications_folder = os.path.join(self.options.folder, 'applications')
+        available_apps = [
+            arq for arq in os.listdir(applications_folder)
+            if os.path.isdir(os.path.join(applications_folder, arq))
+        ]
+        with self.scheduler_processes_lock:
+            # reset the menu
+            # since applications can disappear (be disinstalled) must
+            # clear the menu (should use tkinter.END or tkinter.LAST)
+            self.schedmenu.delete(0, 'end')
+            for arq in available_apps:
+                if arq not in self.scheduler_processes:
+                    item = lambda a = arq: self.try_start_scheduler(a)
+                    self.schedmenu.add_command(label="start %s" % arq,
+                                               command=item)
+                if arq in self.scheduler_processes:
+                    item = lambda a = arq: self.try_stop_scheduler(a)
+                    self.schedmenu.add_command(label="stop %s" % arq,
+                                               command=item)
+
         if start and self.options.with_scheduler and self.options.schedulers:
             # the widget takes care of starting the schedulers
             apps = [ag.split(':', 1)[0] for ag in self.options.schedulers]
@@ -322,100 +348,90 @@ class web2pyDialog(object):
         for app in apps:
             self.try_start_scheduler(app)
 
-        # reset the menu
-        applications_folder = os.path.join(self.options.folder, 'applications')
-        available_apps = [
-            arq for arq in os.listdir(applications_folder)
-            if os.path.isdir(os.path.join(applications_folder, arq))
-        ]
-        self.schedmenu.delete(0, len(available_apps))
-
-        for arq in available_apps:
-            if arq not in self.scheduler_processes:
-                item = lambda u = arq: self.try_start_scheduler(u)
-                self.schedmenu.add_command(label="start %s" % arq,
-                                           command=item)
-            if arq in self.scheduler_processes:
-                item = lambda u = arq: self.try_stop_scheduler(u)
-                self.schedmenu.add_command(label="stop %s" % arq,
-                                           command=item)
-
     def start_schedulers(self, app):
-        try:
-            from multiprocessing import Process
-        except:
-            sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
-            return
+        from multiprocessing import Process
         code = "from gluon.globals import current;current._scheduler.loop()"
         print('starting scheduler from widget for "%s"...' % app)
         args = (app, True, True, None, False, code, False, True)
-        logging.getLogger().setLevel(self.options.log_level)
         p = Process(target=run, args=args)
-        self.scheduler_processes[app] = p
-        self.update_schedulers()
-        print("Currently running %s scheduler processes" % (
-            len(self.scheduler_processes)))
+        with self.scheduler_processes_lock:
+            self.scheduler_processes[app] = p
+            self.update_schedulers()
+            print("Currently running %s scheduler processes" % (
+                len(self.scheduler_processes)))
         p.start()
         print("Processes started")
 
-    def try_stop_scheduler(self, app):
-        if app in self.scheduler_processes:
-            p = self.scheduler_processes[app]
-            del self.scheduler_processes[app]
+    def try_stop_scheduler(self, app, skip_update=False):
+        p = None
+        with self.scheduler_processes_lock:
+            if app in self.scheduler_processes:
+                p = self.scheduler_processes[app]
+                del self.scheduler_processes[app]
+        if p is not None:
             p.terminate()
             p.join()
-        self.update_schedulers()
+        if not skip_update:
+            self.update_schedulers()
 
     def try_start_scheduler(self, app):
-        if app not in self.scheduler_processes:
-            t = threading.Thread(target=self.start_schedulers, args=(app,))
+        t = None
+        with self.scheduler_processes_lock:
+            if not is_appdir(self.options.folder, app):
+                self.schedmenu.delete("start %s" % app)
+                return
+            if app not in self.scheduler_processes:
+                t = threading.Thread(target=self.start_schedulers, args=(app,))
+        if t is not None:
             t.start()
 
     def checkTaskBar(self):
         """ Checks taskbar status """
-
-        if self.tb.status:
-            if self.tb.status[0] == self.tb.EnumStatus.QUIT:
+        tb = self.tb
+        if tb.status:
+            st0 = tb.status[0]
+            EnumStatus = tb.EnumStatus
+            if st0 == EnumStatus.QUIT:
                 self.quit()
-            elif self.tb.status[0] == self.tb.EnumStatus.TOGGLE:
+            elif st0 == EnumStatus.TOGGLE:
                 if self.root.state() == 'withdrawn':
                     self.root.deiconify()
                 else:
                     self.root.withdraw()
-            elif self.tb.status[0] == self.tb.EnumStatus.STOP:
+            elif st0 == EnumStatus.STOP:
                 self.stop()
-            elif self.tb.status[0] == self.tb.EnumStatus.START:
+            elif st0 == EnumStatus.START:
                 self.start()
-            elif self.tb.status[0] == self.tb.EnumStatus.RESTART:
+            elif st0 == EnumStatus.RESTART:
                 self.stop()
                 self.start()
-            del self.tb.status[0]
+            del tb.status[0]
 
         self.root.after(1000, self.checkTaskBar)
 
-    def update(self, text):
-        """ Updates app text """
-
-        try:
-            self.text.configure(state='normal')
-            self.text.insert('end', text)
-            self.text.configure(state='disabled')
-        except:
-            pass  # ## this should only happen in case app is destroyed
-
     def connect_pages(self):
         """ Connects pages """
-        # reset the menu
+        # reset the menu,
+        # since applications can disappear (be disinstalled) must
+        # clear the menu (should use tkinter.END or tkinter.LAST)
+        self.pagesmenu.delete(0, 'end')
         applications_folder = os.path.join(self.options.folder, 'applications')
         available_apps = [
             arq for arq in os.listdir(applications_folder)
             if os.path.exists(os.path.join(applications_folder, arq, '__init__.py'))
         ]
-        self.pagesmenu.delete(0, len(available_apps))
         for arq in available_apps:
             url = self.url + arq
+            item = lambda a = arq: self.try_start_browser(a)
             self.pagesmenu.add_command(
-                label=url, command=lambda u=url: start_browser(u))
+                label=url, command=item)
+
+    def try_start_browser(self, app):
+        url = self.url + app
+        if not is_appdir(self.options.folder, app):
+            self.pagesmenu.delete(url)
+            return
+        start_browser(url)
 
     def quit(self, justHide=False):
         """ Finishes the program execution """
@@ -423,16 +439,20 @@ class web2pyDialog(object):
             self.root.withdraw()
         else:
             try:
-                scheds = self.scheduler_processes.keys()
+                with self.scheduler_processes_lock:
+                    scheds = list(self.scheduler_processes.keys())
                 for t in scheds:
-                    self.try_stop_scheduler(t)
-            except:
-                pass
-            try:
-                newcron.stopcron()
+                    self.try_stop_scheduler(t, skip_update=True)
             except:
                 pass
+            if self.options.with_cron and not self.options.soft_cron:
+                # shutting down hardcron
+                try:
+                    newcron.stopcron()
+                except:
+                    pass
             try:
+                # HttpServer.stop takes care of stopping softcron
                 self.server.stop()
             except:
                 pass
@@ -450,34 +470,40 @@ class web2pyDialog(object):
             import tkMessageBox as messagebox
         else:
             from tkinter import messagebox
-
         messagebox.showerror('web2py start server', message)
 
     def start(self):
         """ Starts web2py server """
-
         password = self.password.get()
-
         if not password:
             self.error('no password, no web admin interface')
 
         ip = self.selected_ip.get()
-
         if not is_valid_ip_address(ip):
             return self.error('invalid host ip address')
-
         try:
             port = int(self.port_number.get())
-        except:
+        except ValueError:
             return self.error('invalid port number')
 
         if self.options.server_key and self.options.server_cert:
             proto = 'https'
         else:
             proto = 'http'
-
         self.url = get_url(ip, proto=proto, port=port)
+
         self.connect_pages()
+        self.update_schedulers()
+
+        # softcron is stopped with HttpServer, thus if starting again
+        # need to reset newcron._stopping to re-enable cron
+        if self.options.soft_cron:
+            newcron.reset()
+
+        # FIXME: if the HttpServer is stopped, then started again,
+        #        does not start because of following error:
+        # WARNING:Rocket.Errors.Port8000:Listener started when not ready.
+
         self.button_start.configure(state='disabled')
 
         try:
@@ -502,7 +528,7 @@ class web2pyDialog(object):
                 path=options.folder,
                 interfaces=options.interfaces)
 
-            thread.start_new_thread(self.server.start, ())
+            threading.Thread(target=self.server.start).start()
         except Exception as e:
             self.button_start.configure(state='normal')
             return self.error(str(e))
@@ -514,11 +540,14 @@ class web2pyDialog(object):
         self.button_stop.configure(state='normal')
 
         if not options.taskbar:
-            thread.start_new_thread(
-                start_browser, (get_url(ip, proto=proto, port=port), True))
+            cpt = threading.Thread(target=start_browser,
+                args=(get_url(ip, proto=proto, port=port), True))
+            cpt.setDaemon(True)
+            cpt.start()
 
         self.password.configure(state='readonly')
-        [ip.configure(state='disabled') for ip in self.ips.values()]
+        for ip in self.ips.values():
+            ip.configure(state='disabled')
         self.port_number.configure(state='readonly')
 
         if self.tb:
@@ -528,16 +557,15 @@ class web2pyDialog(object):
         for listener in self.server.server.listeners:
             if listener.ready:
                 return True
-
         return False
 
     def stop(self):
         """ Stops web2py server """
-
         self.button_start.configure(state='normal')
         self.button_stop.configure(state='disabled')
         self.password.configure(state='normal')
-        [ip.configure(state='normal') for ip in self.ips.values()]
+        for ip in self.ips.values():
+            ip.configure(state='normal')
         self.port_number.configure(state='normal')
         self.server.stop()
 
@@ -546,48 +574,41 @@ class web2pyDialog(object):
 
     def update_canvas(self):
         """ Updates canvas """
-
         httplog = os.path.join(self.options.folder, self.options.log_filename)
+        canvas = self.canvas
         try:
             t1 = os.path.getsize(httplog)
-        except:
-            self.canvas.after(1000, self.update_canvas)
+        except OSError:
+            canvas.after(1000, self.update_canvas)
             return
 
+        points = 400
         try:
-            fp = open(httplog, 'r')
-            fp.seek(self.t0)
-            data = fp.read(t1 - self.t0)
-            fp.close()
-            value = self.p0[1:] + [10 + 90.0 / math.sqrt(1 + data.count('\n'))]
-            self.p0 = value
+            pvalues = self.p0[1:]
+            with open(httplog, 'r') as fp:
+                fp.seek(self.t0)
+                data = fp.read(t1 - self.t0)
+            self.p0 = pvalues + [10 + 90.0 / math.sqrt(1 + data.count('\n'))]
 
-            for i in xrange(len(self.p0) - 1):
-                c = self.canvas.coords(self.q0[i])
-                self.canvas.coords(self.q0[i],
-                                   (c[0],
-                                    self.p0[i],
-                                    c[2],
-                                    self.p0[i + 1]))
+            for i in xrange(points - 1):
+                c = canvas.coords(self.q0[i])
+                canvas.coords(self.q0[i],
+                                   (c[0], self.p0[i],
+                                    c[2], self.p0[i + 1]))
             self.t0 = t1
-        except BaseException:
+        except AttributeError:
             self.t0 = time.time()
             self.t0 = t1
-            self.p0 = [100] * 400
-            self.q0 = [self.canvas.create_line(i, 100, i + 1, 100,
-                       fill='green') for i in xrange(len(self.p0) - 1)]
+            self.p0 = [100] * points
+            self.q0 = [canvas.create_line(i, 100, i + 1, 100,
+                       fill='green') for i in xrange(points - 1)]
 
-        self.canvas.after(1000, self.update_canvas)
+        canvas.after(1000, self.update_canvas)
 
 
-def check_existent_app(options, appname):
-    if os.path.isdir(os.path.join(options.folder, 'applications', appname)):
-        return True
-
-
-def get_code_for_scheduler(app_groups, options):
+def get_code_for_scheduler(applications_parent, app_groups):
     app = app_groups[0]
-    if not check_existent_app(options, app):
+    if not is_appdir(applications_parent, app):
         print("Application '%s' doesn't exist, skipping" % app)
         return None, None
     code = 'from gluon.globals import current;'
@@ -599,16 +620,10 @@ def get_code_for_scheduler(app_groups, options):
 
 
 def start_schedulers(options):
-    try:
-        from multiprocessing import Process
-    except:
-        sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
-        return
-    logging.getLogger().setLevel(options.log_level)
-
+    from multiprocessing import Process
     apps = [ag.split(':') for ag in options.schedulers]
     if not options.with_scheduler and len(apps) == 1:
-        app, code = get_code_for_scheduler(apps[0], options)
+        app, code = get_code_for_scheduler(options.folder, apps[0])
         if not app:
             return
         print('starting single-scheduler for "%s"...' % app)
@@ -624,7 +639,7 @@ def start_schedulers(options):
 
     processes = []
     for app_groups in apps:
-        app, code = get_code_for_scheduler(app_groups, options)
+        app, code = get_code_for_scheduler(options.folder, app_groups)
         if not app:
             continue
         print('starting scheduler for "%s"...' % app)
@@ -652,6 +667,12 @@ def start():
     # get command line arguments
     options = console(version=ProgramVersion)
 
+    if options.with_scheduler or len(options.schedulers) > 1:
+        try:
+            from multiprocessing import Process
+        except:
+            die('Sorry, -K/--scheduler only supported for Python 2.6+')
+
     if options.gae:
         # write app.yaml, gaehandler.py, and exit
         if not os.path.exists('app.yaml'):
@@ -673,6 +694,7 @@ def start():
 
     logger = logging.getLogger("web2py")
     logger.setLevel(options.log_level)
+    logging.getLogger().setLevel(options.log_level) # root logger
 
     # on new installation build the scaffolding app
     create_welcome_w2p()
@@ -724,11 +746,9 @@ def start():
 
     if options.cron_run:
         # run cron (extcron) and exit
-        logger.debug('Starting extcron...')
+        logger.debug('Running extcron...')
         global_settings.web2py_crontype = 'external'
-        extcron = newcron.extcron(options.folder, apps=options.crontabs)
-        extcron.start()
-        extcron.join()
+        newcron.extcron(options.folder, apps=options.crontabs)
         return
 
     if not options.with_scheduler and options.schedulers:
@@ -807,7 +827,7 @@ end tell
         options.password = getpass.getpass('choose a password:')
 
     if not options.password and not options.no_banner:
-        print('no password, disable admin interface')
+        print('no password, no web admin interface')
 
     # Use first interface IP and port if interfaces specified, since the
     # interfaces option overrides the IP (and related) options.
@@ -883,5 +903,4 @@ end tell
                 spt.join()
             except:
                 logger.exception('error terminating schedulers')
-                pass
     logging.shutdown()