dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes #1191

Browse Source 
fix is in lines 828-835 . needed to backport total_seconds for py2.6
(694-701).
everything else is just pep8.
This commit is contained in:
niphlod
2016-02-25 02:46:36 +01:00
parent 5a59149514
commit 642ec2b934
1 changed files with 46 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gluon/scheduler.py
+46 -35
View File
@@ -215,7 +215,7 @@ class JobGraph(object):
nested_dict = dict( nested_dict = dict(
(item, (dep - ordered)) for item, dep in nested_dict.items() (item, (dep - ordered)) for item, dep in nested_dict.items()
if item not in ordered if item not in ordered
) )
assert not nested_dict, "A cyclic dependency exists amongst %r" % nested_dict assert not nested_dict, "A cyclic dependency exists amongst %r" % nested_dict
db.commit() db.commit()
return rtn return rtn
@@ -297,7 +297,7 @@ def executor(queue, task, out):
f = task.function f = task.function
functions = current._scheduler.tasks functions = current._scheduler.tasks
if not functions: if not functions:
#look into env # look into env
_function = _env.get(f) _function = _env.get(f)
else: else:
_function = functions.get(f) _function = functions.get(f)
@@ -314,7 +314,7 @@ def executor(queue, task, out):
vars = loads(task.vars, object_hook=_decode_dict) vars = loads(task.vars, object_hook=_decode_dict)
result = dumps(_function(*args, **vars)) result = dumps(_function(*args, **vars))
else: else:
### for testing purpose only # for testing purpose only
result = eval(task.function)( result = eval(task.function)(
*loads(task.args, object_hook=_decode_dict), *loads(task.args, object_hook=_decode_dict),
**loads(task.vars, object_hook=_decode_dict)) **loads(task.vars, object_hook=_decode_dict))
@@ -663,7 +663,7 @@ class Scheduler(MetaScheduler):
Field('traceback', 'text'), Field('traceback', 'text'),
Field('worker_name', default=self.worker_name), Field('worker_name', default=self.worker_name),
migrate=self.__get_migrate('scheduler_run', migrate) migrate=self.__get_migrate('scheduler_run', migrate)
) )
db.define_table( db.define_table(
'scheduler_worker', 'scheduler_worker',
@@ -675,23 +675,30 @@ class Scheduler(MetaScheduler):
Field('group_names', 'list:string', default=self.group_names), Field('group_names', 'list:string', default=self.group_names),
Field('worker_stats', 'json'), Field('worker_stats', 'json'),
migrate=self.__get_migrate('scheduler_worker', migrate) migrate=self.__get_migrate('scheduler_worker', migrate)
) )
db.define_table( db.define_table(
'scheduler_task_deps', 'scheduler_task_deps',
Field('job_name', default='job_0'), Field('job_name', default='job_0'),
Field('task_parent', 'integer', Field('task_parent', 'integer',
requires=IS_IN_DB(db, 'scheduler_task.id', requires=IS_IN_DB(db, 'scheduler_task.id', '%(task_name)s')
'%(task_name)s') ),
),
Field('task_child', 'reference scheduler_task'), Field('task_child', 'reference scheduler_task'),
Field('can_visit', 'boolean', default=False), Field('can_visit', 'boolean', default=False),
migrate=self.__get_migrate('scheduler_task_deps', migrate) migrate=self.__get_migrate('scheduler_task_deps', migrate)
) )
if migrate is not False: if migrate is not False:
db.commit() db.commit()
@staticmethod
def total_seconds(td):
# backport for py2.6
if hasattr(td, 'total_seconds'):
return td.total_seconds()
else:
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6
def loop(self, worker_name=None): def loop(self, worker_name=None):
"""Main loop """Main loop
@@ -716,7 +723,7 @@ class Scheduler(MetaScheduler):
while True and self.have_heartbeat: while True and self.have_heartbeat:
if self.w_stats.status == DISABLED: if self.w_stats.status == DISABLED:
logger.debug('Someone stopped me, sleeping until better' logger.debug('Someone stopped me, sleeping until better'
' times come (%s)', self.w_stats.sleep) ' times come (%s)', self.w_stats.sleep)
self.sleep() self.sleep()
continue continue
logger.debug('looping...') logger.debug('looping...')
@@ -733,7 +740,8 @@ class Scheduler(MetaScheduler):
logger.debug('sleeping...') logger.debug('sleeping...')
if self.max_empty_runs != 0: if self.max_empty_runs != 0:
logger.debug('empty runs %s/%s', logger.debug('empty runs %s/%s',
self.w_stats.empty_runs, self.max_empty_runs) self.w_stats.empty_runs,
self.max_empty_runs)
if self.w_stats.empty_runs >= self.max_empty_runs: if self.w_stats.empty_runs >= self.max_empty_runs:
logger.info( logger.info(
'empty runs limit reached, killing myself') 'empty runs limit reached, killing myself')
@@ -789,15 +797,15 @@ class Scheduler(MetaScheduler):
now = self.now() now = self.now()
st = self.db.scheduler_task st = self.db.scheduler_task
if self.is_a_ticker and self.do_assign_tasks: if self.is_a_ticker and self.do_assign_tasks:
#I'm a ticker, and 5 loops passed without reassigning tasks, # I'm a ticker, and 5 loops passed without reassigning tasks,
#let's do that and loop again # let's do that and loop again
self.wrapped_assign_tasks(db) self.wrapped_assign_tasks(db)
return None return None
# ready to process something # ready to process something
grabbed = db( grabbed = db(
(st.assigned_worker_name == self.worker_name) & (st.assigned_worker_name == self.worker_name) &
(st.status == ASSIGNED) (st.status == ASSIGNED)
) )
task = grabbed.select(limitby=(0, 1), orderby=st.next_run_time).first() task = grabbed.select(limitby=(0, 1), orderby=st.next_run_time).first()
if task: if task:
@@ -817,11 +825,15 @@ class Scheduler(MetaScheduler):
if not task.prevent_drift: if not task.prevent_drift:
next_run_time = task.last_run_time + datetime.timedelta( next_run_time = task.last_run_time + datetime.timedelta(
seconds=task.period seconds=task.period
)
else:
next_run_time = task.start_time + datetime.timedelta(
seconds=task.period * times_run
) )
else:
# calc next_run_time based on available slots
# see #1191
next_run_time = task.start_time
secondspassed = self.total_seconds(now - next_run_time)
steps = secondspassed // task.period + 1
next_run_time += datetime.timedelta(seconds=task.period * steps)
if times_run < task.repeats or task.repeats == 0: if times_run < task.repeats or task.repeats == 0:
# need to run (repeating task) # need to run (repeating task)
run_again = True run_again = True
@@ -843,7 +855,7 @@ class Scheduler(MetaScheduler):
time.sleep(0.5) time.sleep(0.5)
db.rollback() db.rollback()
logger.info('new task %(id)s "%(task_name)s"' logger.info('new task %(id)s "%(task_name)s"'
' %(application_name)s.%(function_name)s' % task) ' %(application_name)s.%(function_name)s' % task)
return Task( return Task(
app=task.application_name, app=task.application_name,
function=task.function_name, function=task.function_name,
@@ -926,7 +938,7 @@ class Scheduler(MetaScheduler):
and QUEUED or task.retry_failed == -1 and QUEUED or task.retry_failed == -1
and QUEUED or st_mapping) and QUEUED or st_mapping)
db(st.id == task.task_id).update( db(st.id == task.task_id).update(
times_failed=db.scheduler_task.times_failed + 1, times_failed=st.times_failed + 1,
next_run_time=task.next_run_time, next_run_time=task.next_run_time,
status=status status=status
) )
@@ -980,7 +992,7 @@ class Scheduler(MetaScheduler):
# keep sleeping # keep sleeping
self.w_stats.status = DISABLED self.w_stats.status = DISABLED
logger.debug('........recording heartbeat (%s)', logger.debug('........recording heartbeat (%s)',
self.w_stats.status) self.w_stats.status)
db(sw.worker_name == self.worker_name).update( db(sw.worker_name == self.worker_name).update(
last_heartbeat=now, last_heartbeat=now,
worker_stats=self.w_stats) worker_stats=self.w_stats)
@@ -997,7 +1009,7 @@ class Scheduler(MetaScheduler):
logger.info('Asked to kill the current task') logger.info('Asked to kill the current task')
self.terminate_process() self.terminate_process()
logger.debug('........recording heartbeat (%s)', logger.debug('........recording heartbeat (%s)',
self.w_stats.status) self.w_stats.status)
db(sw.worker_name == self.worker_name).update( db(sw.worker_name == self.worker_name).update(
last_heartbeat=now, status=ACTIVE, last_heartbeat=now, status=ACTIVE,
worker_stats=self.w_stats) worker_stats=self.w_stats)
@@ -1023,7 +1035,7 @@ class Scheduler(MetaScheduler):
db( db(
(st.assigned_worker_name.belongs(dead_workers_name)) & (st.assigned_worker_name.belongs(dead_workers_name)) &
(st.status == RUNNING) (st.status == RUNNING)
).update(assigned_worker_name='', status=QUEUED) ).update(assigned_worker_name='', status=QUEUED)
dead_workers.delete() dead_workers.delete()
try: try:
self.is_a_ticker = self.being_a_ticker() self.is_a_ticker = self.being_a_ticker()
@@ -1108,20 +1120,20 @@ class Scheduler(MetaScheduler):
(sd.can_visit == False) & (sd.can_visit == False) &
(~sd.task_child.belongs( (~sd.task_child.belongs(
db(sd.can_visit == False)._select(sd.task_parent) db(sd.can_visit == False)._select(sd.task_parent)
)
) )
)._select(sd.task_child) )
)._select(sd.task_child)
no_deps = db( no_deps = db(
(st.status.belongs((QUEUED, ASSIGNED))) & (st.status.belongs((QUEUED, ASSIGNED))) &
( (
(sd.id == None) | (st.id.belongs(deps_with_no_deps)) (sd.id == None) | (st.id.belongs(deps_with_no_deps))
) )
)._select(st.id, distinct=True, left=sd.on( )._select(st.id, distinct=True, left=sd.on(
(st.id == sd.task_parent) & (st.id == sd.task_parent) &
(sd.can_visit == False) (sd.can_visit == False)
) )
) )
all_available = db( all_available = db(
(st.status.belongs((QUEUED, ASSIGNED))) & (st.status.belongs((QUEUED, ASSIGNED))) &
@@ -1133,7 +1145,6 @@ class Scheduler(MetaScheduler):
(st.id.belongs(no_deps)) (st.id.belongs(no_deps))
) )
limit = len(all_workers) * (50 / (len(wkgroups) or 1)) limit = len(all_workers) * (50 / (len(wkgroups) or 1))
# if there are a moltitude of tasks, let's figure out a maximum of # if there are a moltitude of tasks, let's figure out a maximum of
# tasks per worker. This can be further tuned with some added # tasks per worker. This can be further tuned with some added
@@ -1177,7 +1188,7 @@ class Scheduler(MetaScheduler):
db( db(
(st.id == task.id) & (st.id == task.id) &
(st.status.belongs((QUEUED, ASSIGNED))) (st.status.belongs((QUEUED, ASSIGNED)))
).update(**d) ).update(**d)
wkgroups[gname]['workers'][myw]['c'] += 1 wkgroups[gname]['workers'][myw]['c'] += 1
db.commit() db.commit()
# I didn't report tasks but I'm working nonetheless!!!! # I didn't report tasks but I'm working nonetheless!!!!
@@ -1215,7 +1226,7 @@ class Scheduler(MetaScheduler):
self.db( self.db(
(ws.group_names.contains(group)) & (ws.group_names.contains(group)) &
(~ws.status.belongs(exclusion)) (~ws.status.belongs(exclusion))
).update(status=action) ).update(status=action)
else: else:
for group in group_names: for group in group_names:
workers = self.db((ws.group_names.contains(group)) & workers = self.db((ws.group_names.contains(group)) &
@@ -1300,7 +1311,7 @@ class Scheduler(MetaScheduler):
if immediate: if immediate:
self.db( self.db(
(self.db.scheduler_worker.is_ticker == True) (self.db.scheduler_worker.is_ticker == True)
).update(status=PICK) ).update(status=PICK)
else: else:
rtn.uuid = None rtn.uuid = None
return rtn return rtn
@@ -1350,7 +1361,7 @@ class Scheduler(MetaScheduler):
**dict(orderby=orderby, **dict(orderby=orderby,
left=left, left=left,
limitby=(0, 1)) limitby=(0, 1))
).first() ).first()
if row and output: if row and output:
row.result = row.scheduler_run.run_result and \ row.result = row.scheduler_run.run_result and \
loads(row.scheduler_run.run_result, loads(row.scheduler_run.run_result,