import os os.environ['COLUMNS'] = "1024" import time import subprocess import sys python3binpath = "/########/python3" scriptpath = "/########.py" outpath = "out_monitor.txt" time_sleep_creation = 10 #sec base_time_sleep_active_def = 60 #sec time_sleep_inactive = 15 #sec max_time_sleep_active = 600 #sec time_sleep_exception = 300 #sec def split_and_remove_null_strings (inputstr): to_ret =[] for s in inputstr.split(" "): if s != "": to_ret.append(s) return to_ret def get_cpu_util(pattern) : out = subprocess.check_output(["top", "-b","-n", "1"]) out = out.decode("utf-8") #print(out) out = out.split("\n") row= "" for s in out: if pattern in s: row =s if row == "": return [0, 0] row = split_and_remove_null_strings (row) print (row) pid = row[0] cpu = row[-6] return [int(pid), float(cpu)] poll=0 p=None base_time_sleep_active = base_time_sleep_active_def try: while True: pid,cpu = get_cpu_util (scriptpath) if pid == 0: print ("Starting " + scriptpath) p = subprocess.Popen(['nohup', python3binpath, "-u", scriptpath,">" + outpath, '&']) time.sleep(time_sleep_creation) counter = 0 while counter < 10: pid,cpu = get_cpu_util (scriptpath) if pid == 0 or (p and p.poll()): print(scriptpath + " completed. Exiting") poll=1 sys.exit(0) if cpu > 0: print ("Process active" ) time.sleep(base_time_sleep_active) base_time_sleep_active = min(base_time_sleep_active + 30, max_time_sleep_active ) counter = 0 else: print ("Process inactive, attempt " + str(counter) ) time.sleep(time_sleep_inactive) counter = counter + 1 base_time_sleep_active = base_time_sleep_active_def print ("Killing") out = subprocess.call(["kill", "-9", str(pid)]) except: while True : print("Monitor Exception") if p and not p.poll() and poll==0: print("Waiting " + scriptpath + " to finish") time.sleep(time_sleep_exception) else: if not p: print(scriptpath + " not launched by monitor. Exiting") else: print(scriptpath + " completed. Exiting") sys.exit(-1)