Merge pull request #565 from niphlod/scheduler/enhancement

better assignment (tasks go to not running workers only). Thanks @PengfeiYu

gluon/scheduler.py

@@ -87,9 +87,15 @@ if 'WEB2PY_PATH' not in os.environ:
     os.environ['WEB2PY_PATH'] = path
 
 try:
-    from gluon.contrib.simplejson import loads, dumps
+    # try external module
-except:
     from simplejson import loads, dumps
+except ImportError:
+    try:
+        # try stdlib (Python >= 2.6)
+        from json import loads, dumps
+    except:
+        # fallback to pure-Python module
+        from gluon.contrib.simplejson import loads, dumps
 
 IDENTIFIER = "%s#%s" % (socket.gethostname(), os.getpid())
 
@@ -160,6 +166,7 @@ class TaskReport(object):
     def __str__(self):
         return '<TaskReport: %s>' % self.status
 
+
 class JobGraph(object):
     """Experimental: with JobGraph you can specify
     dependencies amongs tasks"""
@@ -170,7 +177,9 @@ class JobGraph(object):
 
     def add_deps(self, task_parent, task_child):
         """Creates a dependency between task_parent and task_child"""
-        self.db.scheduler_task_deps.insert(task_parent=task_parent, task_child=task_child, job_name=self.job_name)
+        self.db.scheduler_task_deps.insert(task_parent=task_parent,
+                                           task_child=task_child,
+                                           job_name=self.job_name)
 
     def validate(self, job_name):
         """Validates if all tasks job_name can be completed, i.e. there
@@ -203,8 +212,10 @@ class JobGraph(object):
                 if not ordered:
                     break
                 rtn.append(ordered)
-                nested_dict = dict((item, (dep - ordered)) for item, dep in nested_dict.items()
+                nested_dict = dict(
-                    if item not in ordered)
+                    (item, (dep - ordered)) for item, dep in nested_dict.items()
+                    if item not in ordered
+                    )
             assert not nested_dict, "A cyclic dependency exists amongst %r" % nested_dict
             db.commit()
             return rtn
@@ -212,6 +223,7 @@ class JobGraph(object):
             db.rollback()
             return None
 
+
 def demo_function(*argv, **kwargs):
     """ test function """
     for i in range(argv[0]):
@@ -357,8 +369,7 @@ class MetaScheduler(threading.Thread):
 
             start = time.time()
 
-            while p.is_alive() and (
+            while p.is_alive() and (not task.timeout or time.time() - start < task.timeout):
-                    not task.timeout or time.time() - start < task.timeout):
                 if tout:
                     try:
                         logger.debug(' partial output saved')
@@ -1077,6 +1088,8 @@ class Scheduler(MetaScheduler):
         #build workers as dict of groups
         wkgroups = {}
         for w in all_workers:
+            if w.worker_stats['status'] == 'RUNNING':
+                continue
             group_names = w.group_names
             for gname in group_names:
                 if gname not in wkgroups:
@@ -1092,6 +1105,7 @@ class Scheduler(MetaScheduler):
             (st.stop_time < now)
         ).update(status=EXPIRED)
 
+        #calculate dependencies
         deps_with_no_deps = db(
             (sd.can_visit == False) &
             (~sd.task_child.belongs(