lianlonggangandClaude
f07bbbc49d
- RESTART 命令优先通过 nssm restart 重启主程序服务（假死场景可用），未配置服务名回退环回 HTTP - nssm 输出按 UTF-16 解码并记录日志，权限不足时提示需以服务方式运行 - 环回管理接口 code=3001 时明确报鉴权失败，不再误报不可达 - 新增 nssm.path / main.service.name / log.dir 配置项 - 增加文件日志（RotatingFileHandler 5MB×5），服务方式运行可查日志 - HTTP 调用失败与状态码异常统一 WARNING 记录（原 DEBUG 不可见） - 断线日志区分「网络不通」与「被服务器拒绝（未注册/token 错误）」 - 连接 URL 自动拼接 device.id，消除双处配置不一致 Co-Authored-By: Claude <noreply@anthropic.com>
150 lines
5.7 KiB
Python
150 lines
5.7 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""主程序进程守护（看门狗）。
|
||
|
||
周期探测主程序健康：
|
||
- 连续 3 次失败 → 认为主程序宕机 → 强杀残留 → 调启动命令拉起
|
||
- 拉起后等待健康恢复；连续 3 次拉起失败 → 上报告警（ALERT）
|
||
"""
|
||
|
||
import logging
|
||
import subprocess
|
||
import threading
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
|
||
class ProcessGuard:
|
||
"""看门狗：周期健康探测 + 宕机拉起 + 告警上报。"""
|
||
|
||
def __init__(self, config, main_app_client, client):
|
||
self.config = config
|
||
self.main_app_client = main_app_client
|
||
self.client = client
|
||
|
||
self._fail_count = 0 # 连续健康探测失败次数
|
||
self._restart_count = 0 # 连续拉起次数
|
||
self._stop = threading.Event()
|
||
self._thread: threading.Thread | None = None
|
||
|
||
def start(self):
|
||
"""启动守护（首检延迟 30 秒，之后每 15 秒探测一次）。"""
|
||
if not self.config.guard_enabled:
|
||
log.info("进程守护未启用（guard.enable=false）")
|
||
return
|
||
self._thread = threading.Thread(target=self._loop, name="process-guard", daemon=True)
|
||
self._thread.start()
|
||
log.info("进程守护已启动：每 15 秒探测 %s", self.config.main_health_url)
|
||
|
||
def stop(self):
|
||
"""停止守护。"""
|
||
self._stop.set()
|
||
|
||
# ================= 探测循环 =================
|
||
|
||
def _loop(self):
|
||
"""探测主循环：先等 30 秒（给主程序启动时间），再周期探测。"""
|
||
if self._stop.wait(30):
|
||
return
|
||
while not self._stop.wait(15):
|
||
try:
|
||
self._check()
|
||
except Exception as e:
|
||
log.warning("守护探测异常：%s", e)
|
||
|
||
def _check(self):
|
||
"""单次探测：健康则清零计数；连续 3 次失败则拉起。"""
|
||
health = self.main_app_client.health()
|
||
if health.get("ok") is True:
|
||
self._fail_count = 0
|
||
return
|
||
|
||
self._fail_count += 1
|
||
log.warning("主程序健康探测失败（%d/3）：%s", self._fail_count, health.get("error"))
|
||
if self._fail_count < 3:
|
||
return
|
||
|
||
# 连续 3 次失败 → 拉起
|
||
self._fail_count = 0
|
||
self._restart_count += 1
|
||
attempts = self._restart_count
|
||
if attempts > 3:
|
||
self.client.send("ALERT", {
|
||
"level": "CRITICAL",
|
||
"msg": f"主程序连续 {attempts} 次拉起失败，请人工介入",
|
||
})
|
||
# 重置计数，之后继续尝试（降频可后续优化）
|
||
self._restart_count = 3
|
||
return
|
||
|
||
# 拉起方式：配置了服务名则走 nssm restart（幂等），否则回退启动命令
|
||
if self.config.main_service_name:
|
||
start_desc = f"nssm restart {self.config.main_service_name}"
|
||
else:
|
||
start_desc = self.config.main_start_cmd
|
||
log.error("主程序宕机，执行第 %d 次拉起：%s", attempts, start_desc)
|
||
self.client.send("ALERT", {
|
||
"level": "WARN",
|
||
"msg": f"主程序宕机，Agent 正在执行第 {attempts} 次拉起",
|
||
})
|
||
try:
|
||
if self.config.main_service_name:
|
||
ok, reason = restart_main_service(self.config.nssm_path, self.config.main_service_name)
|
||
if not ok:
|
||
log.error("nssm 拉起失败：%s", reason)
|
||
else:
|
||
proc = start_main_app(self.config.main_start_cmd)
|
||
log.info("启动命令已执行，pid=%s", proc.pid if proc else None)
|
||
except Exception as e:
|
||
log.error("拉起失败：%s", e)
|
||
|
||
|
||
# ================= 公共工具（upgrade.py 复用） =================
|
||
|
||
def restart_main_service(nssm_path: str, service_name: str) -> tuple[bool, str]:
|
||
"""
|
||
通过 NSSM 重启主程序服务（幂等：无论服务处于停止/运行态均到运行态）。
|
||
|
||
nssm_path 为 nssm.exe 完整路径；service_name 为注册的 Windows 服务名。
|
||
返回 (是否成功, 输出/原因)。nssm 输出打印到日志，权限不足（拒绝访问）等
|
||
失败原因一眼可见。
|
||
"""
|
||
try:
|
||
proc = subprocess.run(
|
||
[nssm_path, "restart", service_name],
|
||
capture_output=True,
|
||
timeout=60,
|
||
)
|
||
# nssm 输出为 UTF-16 编码（宽字符），需按 utf-16 解码
|
||
out = ((proc.stdout or b"") + (proc.stderr or b"")).decode("utf-16", errors="replace").strip()
|
||
ok = proc.returncode == 0
|
||
if ok:
|
||
log.info("nssm restart %s 成功：%s", service_name, out or "(无输出)")
|
||
else:
|
||
log.warning("nssm restart %s 失败（exit=%s）：%s", service_name, proc.returncode, out)
|
||
if "拒绝访问" in out or "Access" in out:
|
||
out += "（提示：重启服务需要管理员权限，请将 Agent 以 Windows 服务方式运行）"
|
||
return ok, out
|
||
except Exception as e:
|
||
log.warning("nssm restart %s 执行异常：%s", service_name, e)
|
||
return False, str(e)
|
||
|
||
|
||
def start_main_app(start_cmd: str) -> subprocess.Popen | None:
|
||
"""
|
||
拉起主程序。
|
||
|
||
Windows 下 .bat 走 cmd /c（路径转反斜杠）；其余走 sh。
|
||
返回 Popen 对象（不等待进程结束）。
|
||
"""
|
||
cmd = start_cmd.replace("/", "\\") # cmd/c 要求反斜杠路径
|
||
if cmd.lower().endswith(".bat"):
|
||
args = ["cmd", "/c", cmd, "start"]
|
||
else:
|
||
args = ["sh", start_cmd, "start"]
|
||
return subprocess.Popen(
|
||
args,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.STDOUT,
|
||
# 不设 cwd，继承 Agent 工作目录（与主程序同根目录部署）
|
||
)
|