From 38a3a3ec3ec3cbcb81b0074c21f8b956ae4d2ae8 Mon Sep 17 00:00:00 2001 From: lianlonggang Date: Wed, 9 Sep 2026 14:32:30 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=9C=8B=E9=97=A8=E7=8B=97=E6=8B=89?= =?UTF-8?q?=E8=B5=B7=E5=90=8E=E5=A2=9E=E5=8A=A0=E5=AE=BD=E9=99=90=E6=9C=9F?= =?UTF-8?q?=E9=98=B2=E9=87=8D=E5=90=AF=E5=BE=AA=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 主程序重启后 JVM 启动 + DB 建连可能耗时数分钟，300 秒宽限期内探测失败只记录不累计，避免重启循环。 Co-Authored-By: Claude Code --- agent/process_guard.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/agent/process_guard.py b/agent/process_guard.py index a8b7bf2..3e35d68 100644 --- a/agent/process_guard.py +++ b/agent/process_guard.py @@ -9,6 +9,7 @@ import logging import subprocess import threading +import time log = logging.getLogger(__name__) @@ -17,6 +18,11 @@ log = logging.getLogger(__name__) # 避免升级替换 jar 时被看门狗拉起的新进程抢占文件锁。 _upgrading = threading.Event() +# 拉起后的宽限期（秒）：主程序重启后 JVM 启动 + DB 建连可能耗时数分钟 +# （如机器休眠唤醒后网络未就绪，实测 4 分钟）。宽限期内探测失败只记录不累计， +# 避免重启循环（重启→DB 慢→又判宕机→再重启）。 +GRACE_PERIOD_SEC = 300 + class ProcessGuard: """看门狗：周期健康探测 + 宕机拉起 + 告警上报。""" @@ -28,6 +34,7 @@ class ProcessGuard: self._fail_count = 0 # 连续健康探测失败次数 self._restart_count = 0 # 连续拉起次数 + self._grace_until = 0.0 # 拉起后的宽限期截止时刻（monotonic 秒） self._stop = threading.Event() self._thread: threading.Thread | None = None @@ -57,10 +64,19 @@ class ProcessGuard: log.warning("守护探测异常：%s", e) def _check(self): - """单次探测：健康则清零计数；连续 3 次失败则拉起（升级期间跳过）。""" + """单次探测：健康则清零计数；连续 3 次失败则拉起（升级期间/宽限期内跳过）。""" health = self.main_app_client.health() if health.get("ok") is True: self._fail_count = 0 + if self._grace_until and time.monotonic() < self._grace_until: + log.info("宽限期内主程序恢复健康，结束宽限") + self._grace_until = 0.0 + return + + # 拉起后宽限期内：主程序可能在启动/DB 建连中，只记录不累计失败 + if self._grace_until and time.monotonic() < self._grace_until: + log.info("主程序拉起后宽限期内（剩 %d 秒），本次探测失败不累计：%s", + int(self._grace_until - time.monotonic()), health.get("error")) return # 升级流程主动停的主程序，看门狗不接管，避免抢占 jar 锁 @@ -105,6 +121,8 @@ class ProcessGuard: else: proc = start_main_app(self.config.main_start_cmd) log.info("启动命令已执行，pid=%s", proc.pid if proc else None) + # 拉起后进入宽限期：JVM 启动 + DB 建连期间不累计探测失败，防重启循环 + self._grace_until = time.monotonic() + GRACE_PERIOD_SEC except Exception as e: log.error("拉起失败：%s", e)