fix: 看门狗拉起后增加宽限期防重启循环
主程序重启后 JVM 启动 + DB 建连可能耗时数分钟,300 秒宽限期内探测失败只记录不累计,避免重启循环。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
+19
-1
@@ -9,6 +9,7 @@
|
|||||||
import logging
|
import logging
|
||||||
import subprocess
|
import subprocess
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -17,6 +18,11 @@ log = logging.getLogger(__name__)
|
|||||||
# 避免升级替换 jar 时被看门狗拉起的新进程抢占文件锁。
|
# 避免升级替换 jar 时被看门狗拉起的新进程抢占文件锁。
|
||||||
_upgrading = threading.Event()
|
_upgrading = threading.Event()
|
||||||
|
|
||||||
|
# 拉起后的宽限期(秒):主程序重启后 JVM 启动 + DB 建连可能耗时数分钟
|
||||||
|
# (如机器休眠唤醒后网络未就绪,实测 4 分钟)。宽限期内探测失败只记录不累计,
|
||||||
|
# 避免重启循环(重启→DB 慢→又判宕机→再重启)。
|
||||||
|
GRACE_PERIOD_SEC = 300
|
||||||
|
|
||||||
|
|
||||||
class ProcessGuard:
|
class ProcessGuard:
|
||||||
"""看门狗:周期健康探测 + 宕机拉起 + 告警上报。"""
|
"""看门狗:周期健康探测 + 宕机拉起 + 告警上报。"""
|
||||||
@@ -28,6 +34,7 @@ class ProcessGuard:
|
|||||||
|
|
||||||
self._fail_count = 0 # 连续健康探测失败次数
|
self._fail_count = 0 # 连续健康探测失败次数
|
||||||
self._restart_count = 0 # 连续拉起次数
|
self._restart_count = 0 # 连续拉起次数
|
||||||
|
self._grace_until = 0.0 # 拉起后的宽限期截止时刻(monotonic 秒)
|
||||||
self._stop = threading.Event()
|
self._stop = threading.Event()
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
|
|
||||||
@@ -57,10 +64,19 @@ class ProcessGuard:
|
|||||||
log.warning("守护探测异常:%s", e)
|
log.warning("守护探测异常:%s", e)
|
||||||
|
|
||||||
def _check(self):
|
def _check(self):
|
||||||
"""单次探测:健康则清零计数;连续 3 次失败则拉起(升级期间跳过)。"""
|
"""单次探测:健康则清零计数;连续 3 次失败则拉起(升级期间/宽限期内跳过)。"""
|
||||||
health = self.main_app_client.health()
|
health = self.main_app_client.health()
|
||||||
if health.get("ok") is True:
|
if health.get("ok") is True:
|
||||||
self._fail_count = 0
|
self._fail_count = 0
|
||||||
|
if self._grace_until and time.monotonic() < self._grace_until:
|
||||||
|
log.info("宽限期内主程序恢复健康,结束宽限")
|
||||||
|
self._grace_until = 0.0
|
||||||
|
return
|
||||||
|
|
||||||
|
# 拉起后宽限期内:主程序可能在启动/DB 建连中,只记录不累计失败
|
||||||
|
if self._grace_until and time.monotonic() < self._grace_until:
|
||||||
|
log.info("主程序拉起后宽限期内(剩 %d 秒),本次探测失败不累计:%s",
|
||||||
|
int(self._grace_until - time.monotonic()), health.get("error"))
|
||||||
return
|
return
|
||||||
|
|
||||||
# 升级流程主动停的主程序,看门狗不接管,避免抢占 jar 锁
|
# 升级流程主动停的主程序,看门狗不接管,避免抢占 jar 锁
|
||||||
@@ -105,6 +121,8 @@ class ProcessGuard:
|
|||||||
else:
|
else:
|
||||||
proc = start_main_app(self.config.main_start_cmd)
|
proc = start_main_app(self.config.main_start_cmd)
|
||||||
log.info("启动命令已执行,pid=%s", proc.pid if proc else None)
|
log.info("启动命令已执行,pid=%s", proc.pid if proc else None)
|
||||||
|
# 拉起后进入宽限期:JVM 启动 + DB 建连期间不累计探测失败,防重启循环
|
||||||
|
self._grace_until = time.monotonic() + GRACE_PERIOD_SEC
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error("拉起失败:%s", e)
|
log.error("拉起失败:%s", e)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user