Dropbox.web2py.gluon.newcron

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 Created by Attila Csipa <web2py@csipa.in.rs> 6 Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 """ 8 9 import sys 10 import os 11 import threading 12 import logging 13 import time 14 import sched 15 import re 16 import datetime 17 import platform 18 import portalocker 19 import fileutils 20 import cPickle 21 from settings import global_settings 22 23 logger = logging.getLogger("web2py.cron") 24 _cron_stopping = False 25 26

27 -def absolute_path_link(path):

28 """ 29 Return an absolute path for the destination of a symlink 30 31 """ 32 if os.path.islink(path): 33 link = os.readlink(path) 34 if not os.path.isabs(link): 35 link = os.path.join(os.path.dirname(path), link) 36 else: 37 link = os.path.abspath(path) 38 return link

39 40

41 -def stopcron():

42 "graceful shutdown of cron" 43 global _cron_stopping 44 _cron_stopping = True

45 46

47 -class extcron(threading.Thread):

48

49 - def __init__(self, applications_parent, apps=None):

50 threading.Thread.__init__(self) 51 self.setDaemon(False) 52 self.path = applications_parent 53 self.apps = apps

54 # crondance(self.path, 'external', startup=True, apps=self.apps) 55

56 - def run(self):

57 if not _cron_stopping: 58 logger.debug('external cron invocation') 59 crondance(self.path, 'external', startup=False, apps=self.apps)

60 61

62 -class hardcron(threading.Thread):

63

64 - def __init__(self, applications_parent):

65 threading.Thread.__init__(self) 66 self.setDaemon(True) 67 self.path = applications_parent 68 crondance(self.path, 'hard', startup=True)

69

70 - def launch(self):

71 if not _cron_stopping: 72 logger.debug('hard cron invocation') 73 crondance(self.path, 'hard', startup=False)

74

75 - def run(self):

76 s = sched.scheduler(time.time, time.sleep) 77 logger.info('Hard cron daemon started') 78 while not _cron_stopping: 79 now = time.time() 80 s.enter(60 - now % 60, 1, self.launch, ()) 81 s.run()

82 83

84 -class softcron(threading.Thread):

85

86 - def __init__(self, applications_parent):

87 threading.Thread.__init__(self) 88 self.path = applications_parent

89 # crondance(self.path, 'soft', startup=True) 90

91 - def run(self):

92 if not _cron_stopping: 93 logger.debug('soft cron invocation') 94 crondance(self.path, 'soft', startup=False)

95 96

97 -class Token(object):

98

99 - def __init__(self, path):

100 self.path = os.path.join(path, 'cron.master') 101 if not os.path.exists(self.path): 102 fileutils.write_file(self.path, '', 'wb') 103 self.master = None 104 self.now = time.time()

105

106 - def acquire(self, startup=False):

107 """ 108 returns the time when the lock is acquired or 109 None if cron already running 110 111 lock is implemented by writing a pickle (start, stop) in cron.master 112 start is time when cron job starts and stop is time when cron completed 113 stop == 0 if job started but did not yet complete 114 if a cron job started within less than 60 seconds, acquire returns None 115 if a cron job started before 60 seconds and did not stop, 116 a warning is issue "Stale cron.master detected" 117 """ 118 if portalocker.LOCK_EX is None: 119 logger.warning('WEB2PY CRON: Disabled because no file locking') 120 return None 121 self.master = open(self.path, 'rb+') 122 try: 123 ret = None 124 portalocker.lock(self.master, portalocker.LOCK_EX) 125 try: 126 (start, stop) = cPickle.load(self.master) 127 except: 128 (start, stop) = (0, 1) 129 if startup or self.now - start > 59.99: 130 ret = self.now 131 if not stop: 132 # this happens if previous cron job longer than 1 minute 133 logger.warning('WEB2PY CRON: Stale cron.master detected') 134 logger.debug('WEB2PY CRON: Acquiring lock') 135 self.master.seek(0) 136 cPickle.dump((self.now, 0), self.master) 137 finally: 138 portalocker.unlock(self.master) 139 if not ret: 140 # do this so no need to release 141 self.master.close() 142 return ret

143

144 - def release(self):

145 """ 146 this function writes into cron.master the time when cron job 147 was completed 148 """ 149 if not self.master.closed: 150 portalocker.lock(self.master, portalocker.LOCK_EX) 151 logger.debug('WEB2PY CRON: Releasing cron lock') 152 self.master.seek(0) 153 (start, stop) = cPickle.load(self.master) 154 if start == self.now: # if this is my lock 155 self.master.seek(0) 156 cPickle.dump((self.now, time.time()), self.master) 157 portalocker.unlock(self.master) 158 self.master.close()

159 160

161 -def rangetolist(s, period='min'):

162 retval = [] 163 if s.startswith('*'): 164 if period == 'min': 165 s = s.replace('*', '0-59', 1) 166 elif period == 'hr': 167 s = s.replace('*', '0-23', 1) 168 elif period == 'dom': 169 s = s.replace('*', '1-31', 1) 170 elif period == 'mon': 171 s = s.replace('*', '1-12', 1) 172 elif period == 'dow': 173 s = s.replace('*', '0-6', 1) 174 m = re.compile(r'(\d+)-(\d+)/(\d+)') 175 match = m.match(s) 176 if match: 177 for i in range(int(match.group(1)), int(match.group(2)) + 1): 178 if i % int(match.group(3)) == 0: 179 retval.append(i) 180 return retval

181 182

183 -def parsecronline(line):

184 task = {} 185 if line.startswith('@reboot'): 186 line = line.replace('@reboot', '-1 * * * *') 187 elif line.startswith('@yearly'): 188 line = line.replace('@yearly', '0 0 1 1 *') 189 elif line.startswith('@annually'): 190 line = line.replace('@annually', '0 0 1 1 *') 191 elif line.startswith('@monthly'): 192 line = line.replace('@monthly', '0 0 1 * *') 193 elif line.startswith('@weekly'): 194 line = line.replace('@weekly', '0 0 * * 0') 195 elif line.startswith('@daily'): 196 line = line.replace('@daily', '0 0 * * *') 197 elif line.startswith('@midnight'): 198 line = line.replace('@midnight', '0 0 * * *') 199 elif line.startswith('@hourly'): 200 line = line.replace('@hourly', '0 * * * *') 201 params = line.strip().split(None, 6) 202 if len(params) < 7: 203 return None 204 daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 205 'fri': 5, 'sat': 6} 206 for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): 207 if not s in [None, '*']: 208 task[id] = [] 209 vals = s.split(',') 210 for val in vals: 211 if val != '-1' and '-' in val and '/' not in val: 212 val = '%s/1' % val 213 if '/' in val: 214 task[id] += rangetolist(val, id) 215 elif val.isdigit() or val == '-1': 216 task[id].append(int(val)) 217 elif id == 'dow' and val[:3].lower() in daysofweek: 218 task[id].append(daysofweek(val[:3].lower())) 219 task['user'] = params[5] 220 task['cmd'] = params[6] 221 return task

222 223

224 -class cronlauncher(threading.Thread):

225

226 - def __init__(self, cmd, shell=True):

227 threading.Thread.__init__(self) 228 if platform.system() == 'Windows': 229 shell = False 230 self.cmd = cmd 231 self.shell = shell

232

233 - def run(self):

234 import subprocess 235 if isinstance(self.cmd, (list, tuple)): 236 cmd = self.cmd 237 else: 238 cmd = self.cmd.split() 239 proc = subprocess.Popen(cmd, 240 stdin=subprocess.PIPE, 241 stdout=subprocess.PIPE, 242 stderr=subprocess.PIPE, 243 shell=self.shell) 244 (stdoutdata, stderrdata) = proc.communicate() 245 if proc.returncode != 0: 246 logger.warning( 247 'WEB2PY CRON Call returned code %s:\n%s' % 248 (proc.returncode, stdoutdata + stderrdata)) 249 else: 250 logger.debug('WEB2PY CRON Call returned success:\n%s' 251 % stdoutdata)

252 253

254 -def crondance(applications_parent, ctype='soft', startup=False, apps=None):

255 apppath = os.path.join(applications_parent, 'applications') 256 cron_path = os.path.join(applications_parent) 257 token = Token(cron_path) 258 cronmaster = token.acquire(startup=startup) 259 if not cronmaster: 260 return 261 now_s = time.localtime() 262 checks = (('min', now_s.tm_min), 263 ('hr', now_s.tm_hour), 264 ('mon', now_s.tm_mon), 265 ('dom', now_s.tm_mday), 266 ('dow', (now_s.tm_wday + 1) % 7)) 267 268 if apps is None: 269 apps = [x for x in os.listdir(apppath) 270 if os.path.isdir(os.path.join(apppath, x))] 271 272 full_apath_links = set() 273 274 for app in apps: 275 if _cron_stopping: 276 break 277 apath = os.path.join(apppath, app) 278 279 # if app is a symbolic link to other app, skip it 280 full_apath_link = absolute_path_link(apath) 281 if full_apath_link in full_apath_links: 282 continue 283 else: 284 full_apath_links.add(full_apath_link) 285 286 cronpath = os.path.join(apath, 'cron') 287 crontab = os.path.join(cronpath, 'crontab') 288 if not os.path.exists(crontab): 289 continue 290 try: 291 cronlines = fileutils.readlines_file(crontab, 'rt') 292 lines = [x.strip() for x in cronlines if x.strip( 293 ) and not x.strip().startswith('#')] 294 tasks = [parsecronline(cline) for cline in lines] 295 except Exception, e: 296 logger.error('WEB2PY CRON: crontab read error %s' % e) 297 continue 298 299 for task in tasks: 300 if _cron_stopping: 301 break 302 commands = [sys.executable] 303 w2p_path = fileutils.abspath('web2py.py', gluon=True) 304 if os.path.exists(w2p_path): 305 commands.append(w2p_path) 306 if global_settings.applications_parent != global_settings.gluon_parent: 307 commands.extend(('-f', global_settings.applications_parent)) 308 citems = [(k in task and not v in task[k]) for k, v in checks] 309 task_min = task.get('min', []) 310 if not task: 311 continue 312 elif not startup and task_min == [-1]: 313 continue 314 elif task_min != [-1] and reduce(lambda a, b: a or b, citems): 315 continue 316 logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' 317 % (ctype, app, task.get('cmd'), 318 os.getcwd(), datetime.datetime.now())) 319 action, command, models = False, task['cmd'], '' 320 if command.startswith('**'): 321 (action, models, command) = (True, '', command[2:]) 322 elif command.startswith('*'): 323 (action, models, command) = (True, '-M', command[1:]) 324 else: 325 action = False 326 327 if action and command.endswith('.py'): 328 commands.extend(('-J', # cron job 329 models, # import models? 330 '-S', app, # app name 331 '-a', '"<recycle>"', # password 332 '-R', command)) # command 333 elif action: 334 commands.extend(('-J', # cron job 335 models, # import models? 336 '-S', app + '/' + command, # app name 337 '-a', '"<recycle>"')) # password 338 else: 339 commands = command 340 341 # from python docs: 342 # You do not need shell=True to run a batch file or 343 # console-based executable. 344 shell = False 345 346 try: 347 cronlauncher(commands, shell=shell).start() 348 except Exception, e: 349 logger.warning( 350 'WEB2PY CRON: Execution error for %s: %s' 351 % (task.get('cmd'), e)) 352 token.release()

353

Source Code for Module Dropbox.web2py.gluon.newcron