fix: 看门狗拉起后增加宽限期防重启循环
主程序重启后 JVM 启动 + DB 建连可能耗时数分钟,300 秒宽限期内探测失败只记录不累计,避免重启循环。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
+19
-1
@@ -9,6 +9,7 @@
|
||||
import logging
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -17,6 +18,11 @@ log = logging.getLogger(__name__)
|
||||
# 避免升级替换 jar 时被看门狗拉起的新进程抢占文件锁。
|
||||
_upgrading = threading.Event()
|
||||
|
||||
# 拉起后的宽限期(秒):主程序重启后 JVM 启动 + DB 建连可能耗时数分钟
|
||||
# (如机器休眠唤醒后网络未就绪,实测 4 分钟)。宽限期内探测失败只记录不累计,
|
||||
# 避免重启循环(重启→DB 慢→又判宕机→再重启)。
|
||||
GRACE_PERIOD_SEC = 300
|
||||
|
||||
|
||||
class ProcessGuard:
|
||||
"""看门狗:周期健康探测 + 宕机拉起 + 告警上报。"""
|
||||
@@ -28,6 +34,7 @@ class ProcessGuard:
|
||||
|
||||
self._fail_count = 0 # 连续健康探测失败次数
|
||||
self._restart_count = 0 # 连续拉起次数
|
||||
self._grace_until = 0.0 # 拉起后的宽限期截止时刻(monotonic 秒)
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
@@ -57,10 +64,19 @@ class ProcessGuard:
|
||||
log.warning("守护探测异常:%s", e)
|
||||
|
||||
def _check(self):
|
||||
"""单次探测:健康则清零计数;连续 3 次失败则拉起(升级期间跳过)。"""
|
||||
"""单次探测:健康则清零计数;连续 3 次失败则拉起(升级期间/宽限期内跳过)。"""
|
||||
health = self.main_app_client.health()
|
||||
if health.get("ok") is True:
|
||||
self._fail_count = 0
|
||||
if self._grace_until and time.monotonic() < self._grace_until:
|
||||
log.info("宽限期内主程序恢复健康,结束宽限")
|
||||
self._grace_until = 0.0
|
||||
return
|
||||
|
||||
# 拉起后宽限期内:主程序可能在启动/DB 建连中,只记录不累计失败
|
||||
if self._grace_until and time.monotonic() < self._grace_until:
|
||||
log.info("主程序拉起后宽限期内(剩 %d 秒),本次探测失败不累计:%s",
|
||||
int(self._grace_until - time.monotonic()), health.get("error"))
|
||||
return
|
||||
|
||||
# 升级流程主动停的主程序,看门狗不接管,避免抢占 jar 锁
|
||||
@@ -105,6 +121,8 @@ class ProcessGuard:
|
||||
else:
|
||||
proc = start_main_app(self.config.main_start_cmd)
|
||||
log.info("启动命令已执行,pid=%s", proc.pid if proc else None)
|
||||
# 拉起后进入宽限期:JVM 启动 + DB 建连期间不累计探测失败,防重启循环
|
||||
self._grace_until = time.monotonic() + GRACE_PERIOD_SEC
|
||||
except Exception as e:
|
||||
log.error("拉起失败:%s", e)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user