[Tool] Automatic restart of multi-process program interruption

Preface 

By the way, I deployed the program on Ubuntu. Since I was particularly worried about program interruption, I added a lot of try excepts, which caused the program to fail but still run. Instead of doing this, let it go and restart to make up for it. ". So I wrote a multi-process restart program.

code

This restart program needs to cooperate with the shell script to monitor the pid of each process. Once the pid is found to disappear, the entire program will be restarted immediately.

shell script

for i in $(seq 1 10)
do
      old_ids=$(ps -ef | grep python | awk '{print $2}');
      #echo "$old_ids"
      kill -9 $old_ids;
done

cd /home/。。。。/pythonwork
nohup python3 main.py > /dev/null 2>&1& #./log.log

python multi-process code

import multiprocessing
import time
import psutil
import us
from loguru import logger

ROOTPATH = os.path.dirname(os.path.abspath(__file__))

logger.add(sink='log/log_{time}_main.log', level='DEBUG', enqueue=True,rotation='00:00', retention="7 days")
def _pull_img(share_dic):
    print("_pull_img")
    logger.info("pull")
    _pid = os.getpid()
    print(share_dic["algo_pids"])
    share_dic["pull_pids"]=_pid

    time.sleep(50)
    a=0/0

    pass

def _algo(share_dic):
    print("_algo")

    logger.info("algo")
    _pid = os.getpid()
    share_dic["algo_pids"]=_pid
    time.sleep(100)
    pass

def _supervisor(share_dic):
    logger.info("_supervisor")
    while True:


        if share_dic["algo_pids"]==-1 or share_dic["pull_pids"]==-1 :
            time.sleep(10)
        else:
            pl = psutil.pids()
            if share_dic["algo_pids"] not in pl:
                logger.info("algo_pids process terminated unexpectedly, start restarting process reset")
                cmd = f"{ROOTPATH}/algo_kill_and_run.sh"
                os.system(cmd)
            elif share_dic["pull_pids"] not in pl:
                logger.info("algo_pids process terminated unexpectedly, start restarting process reset")
                cmd = f"{ROOTPATH}/algo_kill_and_run.sh"
                os.system(cmd)

            time.sleep(30)


def main():
    # share_parame = multiprocessing.Array('i', [False])

    share_dic = multiprocessing.Manager().dict()
    share_dic["box_changed"] = False # Whether the drawing area has changed, True means change
    share_dic["last_angle"] = 0 # Previous angle
    share_dic["algo_pids"] = -1
    share_dic["pul_pids"] = -1




    pull_q = multiprocessing.Queue(30)
    alarm_q = multiprocessing.Queue(60)

    # pull stream
    p_pull_img = multiprocessing.Process(target=_pull_img, args=(share_dic,))

    # algorithm
    p_something = multiprocessing.Process(target=_something, args=(share_dic,))



    #Monitor process
    p_supervisor=multiprocessing.Process(target=_supervisor, args=(share_dic,))

    p_pull_img.start()
    p_something.start()
    p_supervisor.start()

    p_pull_img.join()
    p_something.join()
    p_supervisor.join()

if __name__ == '__main__':
    main()