feat: Python 版前置机 Agent 完整实现
- WebSocket 连接 B 端(注册/心跳/指数退避重连) - 命令分发(10 种远程运维命令) - 看门狗守护、日志实时流、在线升级含自动回滚 - 修复 main.py 日志格式 style 与占位符不匹配的问题 - 补充 README、配置示例、依赖清单 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""主程序进程守护(看门狗)。
|
||||
|
||||
周期探测主程序健康:
|
||||
- 连续 3 次失败 → 认为主程序宕机 → 强杀残留 → 调启动命令拉起
|
||||
- 拉起后等待健康恢复;连续 3 次拉起失败 → 上报告警(ALERT)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ProcessGuard:
|
||||
"""看门狗:周期健康探测 + 宕机拉起 + 告警上报。"""
|
||||
|
||||
def __init__(self, config, main_app_client, client):
|
||||
self.config = config
|
||||
self.main_app_client = main_app_client
|
||||
self.client = client
|
||||
|
||||
self._fail_count = 0 # 连续健康探测失败次数
|
||||
self._restart_count = 0 # 连续拉起次数
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self):
|
||||
"""启动守护(首检延迟 30 秒,之后每 15 秒探测一次)。"""
|
||||
if not self.config.guard_enabled:
|
||||
log.info("进程守护未启用(guard.enable=false)")
|
||||
return
|
||||
self._thread = threading.Thread(target=self._loop, name="process-guard", daemon=True)
|
||||
self._thread.start()
|
||||
log.info("进程守护已启动:每 15 秒探测 %s", self.config.main_health_url)
|
||||
|
||||
def stop(self):
|
||||
"""停止守护。"""
|
||||
self._stop.set()
|
||||
|
||||
# ================= 探测循环 =================
|
||||
|
||||
def _loop(self):
|
||||
"""探测主循环:先等 30 秒(给主程序启动时间),再周期探测。"""
|
||||
if self._stop.wait(30):
|
||||
return
|
||||
while not self._stop.wait(15):
|
||||
try:
|
||||
self._check()
|
||||
except Exception as e:
|
||||
log.warning("守护探测异常:%s", e)
|
||||
|
||||
def _check(self):
|
||||
"""单次探测:健康则清零计数;连续 3 次失败则拉起。"""
|
||||
health = self.main_app_client.health()
|
||||
if health.get("ok") is True:
|
||||
self._fail_count = 0
|
||||
return
|
||||
|
||||
self._fail_count += 1
|
||||
log.warning("主程序健康探测失败(%d/3):%s", self._fail_count, health.get("error"))
|
||||
if self._fail_count < 3:
|
||||
return
|
||||
|
||||
# 连续 3 次失败 → 拉起
|
||||
self._fail_count = 0
|
||||
self._restart_count += 1
|
||||
attempts = self._restart_count
|
||||
if attempts > 3:
|
||||
self.client.send("ALERT", {
|
||||
"level": "CRITICAL",
|
||||
"msg": f"主程序连续 {attempts} 次拉起失败,请人工介入",
|
||||
})
|
||||
# 重置计数,之后继续尝试(降频可后续优化)
|
||||
self._restart_count = 3
|
||||
return
|
||||
|
||||
log.error("主程序宕机,执行第 %d 次拉起:%s", attempts, self.config.main_start_cmd)
|
||||
self.client.send("ALERT", {
|
||||
"level": "WARN",
|
||||
"msg": f"主程序宕机,Agent 正在执行第 {attempts} 次拉起",
|
||||
})
|
||||
try:
|
||||
proc = start_main_app(self.config.main_start_cmd)
|
||||
log.info("启动命令已执行,pid=%s", proc.pid if proc else None)
|
||||
except Exception as e:
|
||||
log.error("启动命令执行失败:%s", e)
|
||||
|
||||
|
||||
# ================= 公共工具(upgrade.py 复用) =================
|
||||
|
||||
def start_main_app(start_cmd: str) -> subprocess.Popen | None:
|
||||
"""
|
||||
拉起主程序。
|
||||
|
||||
Windows 下 .bat 走 cmd /c(路径转反斜杠);其余走 sh。
|
||||
返回 Popen 对象(不等待进程结束)。
|
||||
"""
|
||||
cmd = start_cmd.replace("/", "\\") # cmd/c 要求反斜杠路径
|
||||
if cmd.lower().endswith(".bat"):
|
||||
args = ["cmd", "/c", cmd, "start"]
|
||||
else:
|
||||
args = ["sh", start_cmd, "start"]
|
||||
return subprocess.Popen(
|
||||
args,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.STDOUT,
|
||||
# 不设 cwd,继承 Agent 工作目录(与主程序同根目录部署)
|
||||
)
|
||||
Reference in New Issue
Block a user