fix: 看门狗拉起后增加宽限期防重启循环

主程序重启后 JVM 启动 + DB 建连可能耗时数分钟,300 秒宽限期内探测失败只记录不累计,避免重启循环。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
2026-09-09 14:32:30 +08:00
co-authored by Claude Code
parent a6a801220c
commit 38a3a3ec3e
+19 -1
View File
@@ -9,6 +9,7 @@
import logging import logging
import subprocess import subprocess
import threading import threading
import time
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -17,6 +18,11 @@ log = logging.getLogger(__name__)
# 避免升级替换 jar 时被看门狗拉起的新进程抢占文件锁。 # 避免升级替换 jar 时被看门狗拉起的新进程抢占文件锁。
_upgrading = threading.Event() _upgrading = threading.Event()
# 拉起后的宽限期(秒):主程序重启后 JVM 启动 + DB 建连可能耗时数分钟
# (如机器休眠唤醒后网络未就绪,实测 4 分钟)。宽限期内探测失败只记录不累计,
# 避免重启循环(重启→DB 慢→又判宕机→再重启)。
GRACE_PERIOD_SEC = 300
class ProcessGuard: class ProcessGuard:
"""看门狗:周期健康探测 + 宕机拉起 + 告警上报。""" """看门狗:周期健康探测 + 宕机拉起 + 告警上报。"""
@@ -28,6 +34,7 @@ class ProcessGuard:
self._fail_count = 0 # 连续健康探测失败次数 self._fail_count = 0 # 连续健康探测失败次数
self._restart_count = 0 # 连续拉起次数 self._restart_count = 0 # 连续拉起次数
self._grace_until = 0.0 # 拉起后的宽限期截止时刻(monotonic 秒)
self._stop = threading.Event() self._stop = threading.Event()
self._thread: threading.Thread | None = None self._thread: threading.Thread | None = None
@@ -57,10 +64,19 @@ class ProcessGuard:
log.warning("守护探测异常:%s", e) log.warning("守护探测异常:%s", e)
def _check(self): def _check(self):
"""单次探测:健康则清零计数;连续 3 次失败则拉起(升级期间跳过)。""" """单次探测:健康则清零计数;连续 3 次失败则拉起(升级期间/宽限期内跳过)。"""
health = self.main_app_client.health() health = self.main_app_client.health()
if health.get("ok") is True: if health.get("ok") is True:
self._fail_count = 0 self._fail_count = 0
if self._grace_until and time.monotonic() < self._grace_until:
log.info("宽限期内主程序恢复健康,结束宽限")
self._grace_until = 0.0
return
# 拉起后宽限期内:主程序可能在启动/DB 建连中,只记录不累计失败
if self._grace_until and time.monotonic() < self._grace_until:
log.info("主程序拉起后宽限期内(剩 %d 秒),本次探测失败不累计:%s",
int(self._grace_until - time.monotonic()), health.get("error"))
return return
# 升级流程主动停的主程序,看门狗不接管,避免抢占 jar 锁 # 升级流程主动停的主程序,看门狗不接管,避免抢占 jar 锁
@@ -105,6 +121,8 @@ class ProcessGuard:
else: else:
proc = start_main_app(self.config.main_start_cmd) proc = start_main_app(self.config.main_start_cmd)
log.info("启动命令已执行,pid=%s", proc.pid if proc else None) log.info("启动命令已执行,pid=%s", proc.pid if proc else None)
# 拉起后进入宽限期:JVM 启动 + DB 建连期间不累计探测失败,防重启循环
self._grace_until = time.monotonic() + GRACE_PERIOD_SEC
except Exception as e: except Exception as e:
log.error("拉起失败:%s", e) log.error("拉起失败:%s", e)