88 lines
2.4 KiB
Python
88 lines
2.4 KiB
Python
|
|
import os
|
|
os.environ['COLUMNS'] = "1024"
|
|
import time
|
|
import subprocess
|
|
import sys
|
|
|
|
python3binpath = "/########/python3"
|
|
scriptpath = "/########.py"
|
|
outpath = "out_monitor.txt"
|
|
time_sleep_creation = 10 #sec
|
|
base_time_sleep_active_def = 60 #sec
|
|
time_sleep_inactive = 15 #sec
|
|
max_time_sleep_active = 600 #sec
|
|
time_sleep_exception = 300 #sec
|
|
|
|
|
|
def split_and_remove_null_strings (inputstr):
|
|
to_ret =[]
|
|
for s in inputstr.split(" "):
|
|
if s != "":
|
|
to_ret.append(s)
|
|
return to_ret
|
|
|
|
def get_cpu_util(pattern) :
|
|
out = subprocess.check_output(["top", "-b","-n", "1"])
|
|
out = out.decode("utf-8")
|
|
#print(out)
|
|
|
|
out = out.split("\n")
|
|
row= ""
|
|
for s in out:
|
|
if pattern in s:
|
|
row =s
|
|
if row == "":
|
|
return [0, 0]
|
|
row = split_and_remove_null_strings (row)
|
|
print (row)
|
|
pid = row[0]
|
|
cpu = row[-6]
|
|
return [int(pid), float(cpu)]
|
|
poll=0
|
|
p=None
|
|
base_time_sleep_active = base_time_sleep_active_def
|
|
try:
|
|
while True:
|
|
pid,cpu = get_cpu_util (scriptpath)
|
|
if pid == 0:
|
|
print ("Starting " + scriptpath)
|
|
p = subprocess.Popen(['nohup', python3binpath, "-u", scriptpath,">" + outpath, '&'])
|
|
time.sleep(time_sleep_creation)
|
|
counter = 0
|
|
while counter < 10:
|
|
pid,cpu = get_cpu_util (scriptpath)
|
|
if pid == 0 or (p and p.poll()):
|
|
print(scriptpath + " completed. Exiting")
|
|
poll=1
|
|
sys.exit(0)
|
|
if cpu > 0:
|
|
print ("Process active" )
|
|
time.sleep(base_time_sleep_active)
|
|
base_time_sleep_active = min(base_time_sleep_active + 30, max_time_sleep_active )
|
|
counter = 0
|
|
else:
|
|
print ("Process inactive, attempt " + str(counter) )
|
|
time.sleep(time_sleep_inactive)
|
|
counter = counter + 1
|
|
base_time_sleep_active = base_time_sleep_active_def
|
|
print ("Killing")
|
|
out = subprocess.call(["kill", "-9", str(pid)])
|
|
except:
|
|
while True :
|
|
print("Monitor Exception")
|
|
if p and not p.poll() and poll==0:
|
|
print("Waiting " + scriptpath + " to finish")
|
|
time.sleep(time_sleep_exception)
|
|
else:
|
|
if not p:
|
|
print(scriptpath + " not launched by monitor. Exiting")
|
|
else:
|
|
print(scriptpath + " completed. Exiting")
|
|
sys.exit(-1)
|
|
|
|
|
|
|
|
|
|
|