# -*- coding: utf-8 -*- """主程序进程守护(看门狗)。 周期探测主程序健康: - 连续 3 次失败 → 认为主程序宕机 → 强杀残留 → 调启动命令拉起 - 拉起后等待健康恢复;连续 3 次拉起失败 → 上报告警(ALERT) """ import logging import subprocess import threading log = logging.getLogger(__name__) class ProcessGuard: """看门狗:周期健康探测 + 宕机拉起 + 告警上报。""" def __init__(self, config, main_app_client, client): self.config = config self.main_app_client = main_app_client self.client = client self._fail_count = 0 # 连续健康探测失败次数 self._restart_count = 0 # 连续拉起次数 self._stop = threading.Event() self._thread: threading.Thread | None = None def start(self): """启动守护(首检延迟 30 秒,之后每 15 秒探测一次)。""" if not self.config.guard_enabled: log.info("进程守护未启用(guard.enable=false)") return self._thread = threading.Thread(target=self._loop, name="process-guard", daemon=True) self._thread.start() log.info("进程守护已启动:每 15 秒探测 %s", self.config.main_health_url) def stop(self): """停止守护。""" self._stop.set() # ================= 探测循环 ================= def _loop(self): """探测主循环:先等 30 秒(给主程序启动时间),再周期探测。""" if self._stop.wait(30): return while not self._stop.wait(15): try: self._check() except Exception as e: log.warning("守护探测异常:%s", e) def _check(self): """单次探测:健康则清零计数;连续 3 次失败则拉起。""" health = self.main_app_client.health() if health.get("ok") is True: self._fail_count = 0 return self._fail_count += 1 log.warning("主程序健康探测失败(%d/3):%s", self._fail_count, health.get("error")) if self._fail_count < 3: return # 连续 3 次失败 → 拉起 self._fail_count = 0 self._restart_count += 1 attempts = self._restart_count if attempts > 3: self.client.send("ALERT", { "level": "CRITICAL", "msg": f"主程序连续 {attempts} 次拉起失败,请人工介入", }) # 重置计数,之后继续尝试(降频可后续优化) self._restart_count = 3 return log.error("主程序宕机,执行第 %d 次拉起:%s", attempts, self.config.main_start_cmd) self.client.send("ALERT", { "level": "WARN", "msg": f"主程序宕机,Agent 正在执行第 {attempts} 次拉起", }) try: proc = start_main_app(self.config.main_start_cmd) log.info("启动命令已执行,pid=%s", proc.pid if proc else None) except Exception as e: log.error("启动命令执行失败:%s", e) # ================= 公共工具(upgrade.py 复用) ================= def start_main_app(start_cmd: str) -> subprocess.Popen | None: """ 拉起主程序。 Windows 下 .bat 走 cmd /c(路径转反斜杠);其余走 sh。 返回 Popen 对象(不等待进程结束)。 """ cmd = start_cmd.replace("/", "\\") # cmd/c 要求反斜杠路径 if cmd.lower().endswith(".bat"): args = ["cmd", "/c", cmd, "start"] else: args = ["sh", start_cmd, "start"] return subprocess.Popen( args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, # 不设 cwd,继承 Agent 工作目录(与主程序同根目录部署) )