feat: RESTART 走 NSSM 服务重启 + 失败原因可观测化
- RESTART 命令优先通过 nssm restart 重启主程序服务（假死场景可用），未配置服务名回退环回 HTTP - nssm 输出按 UTF-16 解码并记录日志，权限不足时提示需以服务方式运行 - 环回管理接口 code=3001 时明确报鉴权失败，不再误报不可达 - 新增 nssm.path / main.service.name / log.dir 配置项 - 增加文件日志（RotatingFileHandler 5MB×5），服务方式运行可查日志 - HTTP 调用失败与状态码异常统一 WARNING 记录（原 DEBUG 不可见） - 断线日志区分「网络不通」与「被服务器拒绝（未注册/token 错误）」 - 连接 URL 自动拼接 device.id，消除双处配置不一致 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -10,8 +10,11 @@
|
||||
- main.health.url 主程序健康检查地址（http://127.0.0.1:8082/api/admin/health）
|
||||
- main.admin.token 主程序管理接口 Token（与主程序 admin.token 一致）
|
||||
- main.start.cmd 主程序启动命令（Windows: bin\\start.bat；Linux: bin/start.sh）
|
||||
- nssm.path NSSM 可执行文件路径（配置 main.service.name 时用于拉起服务）
|
||||
- main.service.name 主程序注册的 Windows 服务名（配置后看门狗用 nssm restart 拉起，优先于 main.start.cmd）
|
||||
- heartbeat.sec 心跳间隔（默认 15 秒，须小于网关空闲超时）
|
||||
- guard.enable 是否守护主程序（默认 true）
|
||||
- log.dir 日志目录（默认 logs，相对部署根目录）
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -28,8 +31,11 @@ class AgentConfig:
|
||||
main_health_url: str
|
||||
main_admin_token: str
|
||||
main_start_cmd: str
|
||||
nssm_path: str
|
||||
main_service_name: str
|
||||
heartbeat_sec: int
|
||||
guard_enabled: bool
|
||||
log_dir: str
|
||||
|
||||
@classmethod
|
||||
def load(cls, conf_path: str = "conf/agent.properties") -> "AgentConfig":
|
||||
@@ -57,8 +63,11 @@ class AgentConfig:
|
||||
main_health_url=_get(props, "main.health.url", "http://127.0.0.1:8082/api/admin/health"),
|
||||
main_admin_token=_get(props, "main.admin.token", ""),
|
||||
main_start_cmd=_get(props, "main.start.cmd", "bin/start.bat"),
|
||||
nssm_path=_get(props, "nssm.path", ""),
|
||||
main_service_name=_get(props, "main.service.name", ""),
|
||||
heartbeat_sec=int(_get(props, "heartbeat.sec", "15")),
|
||||
guard_enabled=_get(props, "guard.enable", "true").lower() == "true",
|
||||
log_dir=_get(props, "log.dir", "logs"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
+12
-2
@@ -93,10 +93,20 @@ class CommandDispatcher:
|
||||
result = self._get_log_stream().control(action, file)
|
||||
|
||||
elif msg_type == "RESTART":
|
||||
ok = self.main_app_client.restart(1000)
|
||||
# 优先走 NSSM 服务重启（主程序假死时 HTTP 不可达也能重启成功）；
|
||||
# 未配置服务名时回退环回 HTTP 触发主程序优雅重启
|
||||
if self.config.main_service_name:
|
||||
from agent.process_guard import restart_main_service
|
||||
ok, reason = restart_main_service(self.config.nssm_path, self.config.main_service_name)
|
||||
result = {
|
||||
"ok": ok,
|
||||
"msg": "已触发优雅重启，等待 ProcessGuard 拉起" if ok else "重启请求失败",
|
||||
"msg": "已通过 NSSM 重启主程序服务" if ok else f"NSSM 重启失败：{reason}",
|
||||
}
|
||||
else:
|
||||
ok, reason = self.main_app_client.restart(1000)
|
||||
result = {
|
||||
"ok": ok,
|
||||
"msg": "已触发优雅重启，等待 ProcessGuard 拉起" if ok else f"重启请求失败：{reason}",
|
||||
}
|
||||
|
||||
elif msg_type == "GET_CONFIG":
|
||||
|
||||
+32
-8
@@ -13,6 +13,8 @@ hospital-agent.exe （PyInstaller 打包后）
|
||||
"""
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
@@ -29,18 +31,13 @@ _client = None
|
||||
|
||||
|
||||
def run():
|
||||
"""主流程：加载配置 → 写 PID → 组装组件 → 启动 → 主线程保活。"""
|
||||
"""主流程：加载配置 → 组装组件 → 启动 → 主线程保活。"""
|
||||
global _dispatcher, _guard, _client
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="{asctime} {levelname} [{name}] {message}",
|
||||
style="{",
|
||||
)
|
||||
# 源码运行时根目录即工作目录，PyInstaller 打包后同理（服务管理器负责设置工作目录）
|
||||
config = AgentConfig.load()
|
||||
_setup_logging(config.log_dir)
|
||||
log.info("Agent 启动：deviceId=%s, server=%s", config.device_id, config.server_url)
|
||||
|
||||
# 源码运行时根目录即工作目录，PyInstaller 打包后同理（服务管理器负责设置工作目录）
|
||||
# 组装组件（延迟导入避免环）
|
||||
from agent.dispatcher import CommandDispatcher
|
||||
from agent.process_guard import ProcessGuard
|
||||
@@ -64,6 +61,33 @@ def run():
|
||||
threading.Event().wait()
|
||||
|
||||
|
||||
def _setup_logging(log_dir: str):
|
||||
"""日志输出到控制台 + 文件（服务方式运行时靠文件留存，按大小轮转）。"""
|
||||
fmt = logging.Formatter(
|
||||
"{asctime} {levelname} [{name}] {message}",
|
||||
style="{",
|
||||
)
|
||||
root = logging.getLogger()
|
||||
root.setLevel(logging.INFO)
|
||||
|
||||
console = logging.StreamHandler()
|
||||
console.setFormatter(fmt)
|
||||
root.addHandler(console)
|
||||
|
||||
try:
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
os.path.join(log_dir, "agent.log"),
|
||||
maxBytes=5 * 1024 * 1024, # 单文件 5MB
|
||||
backupCount=5, # 保留 5 个轮转备份
|
||||
encoding="utf-8",
|
||||
)
|
||||
file_handler.setFormatter(fmt)
|
||||
root.addHandler(file_handler)
|
||||
except Exception as e:
|
||||
root.warning("日志文件初始化失败（仅控制台输出）：%s", e)
|
||||
|
||||
|
||||
def _shutdown(signum, frame):
|
||||
"""优雅停机：释放日志流 → 停守护 → 断连接 → 退出。"""
|
||||
log.info("Agent 停机中...（signal=%s）", signum)
|
||||
|
||||
@@ -57,9 +57,13 @@ class MainAppClient:
|
||||
"""日志文件列表（原始 JSON 文本）。"""
|
||||
return self._get_text(self._url("/api/admin/log/files"))
|
||||
|
||||
def restart(self, delay_ms: int) -> bool:
|
||||
"""触发主程序优雅重启。"""
|
||||
return self._post_text(self._url(f"/api/admin/restart?delayMs={delay_ms}"), "") is not None
|
||||
def restart(self, delay_ms: int) -> tuple[bool, str]:
|
||||
"""触发主程序优雅重启。返回 (是否成功, 失败原因)。"""
|
||||
body, reason = self._post_text_with_reason(self._url(f"/api/admin/restart?delayMs={delay_ms}"), "")
|
||||
# 主程序管理接口鉴权失败时返回 code=3001 的 JSON（HTTP 层可能为 200）
|
||||
if body is not None and "3001" in body:
|
||||
return False, "管理接口鉴权失败（main.admin.token 与主程序不一致）"
|
||||
return (body is not None), reason
|
||||
|
||||
def config_file(self, hospital: str) -> str | None:
|
||||
"""读取医院配置文件（原始 JSON 文本）。"""
|
||||
@@ -89,9 +93,12 @@ class MainAppClient:
|
||||
headers={"X-Admin-Token": self.config.main_admin_token},
|
||||
timeout=(5, 30),
|
||||
)
|
||||
return resp if resp.ok else None
|
||||
if not resp.ok:
|
||||
log.warning("GET %s 失败：HTTP %s", url, resp.status_code)
|
||||
return None
|
||||
return resp
|
||||
except Exception as e:
|
||||
log.debug("GET %s 失败：%s", url, e)
|
||||
log.warning("GET %s 失败：%s", url, e)
|
||||
return None
|
||||
|
||||
def _get_text(self, url: str) -> str | None:
|
||||
@@ -101,6 +108,11 @@ class MainAppClient:
|
||||
|
||||
def _post_text(self, url: str, form_body: str) -> str | None:
|
||||
"""POST 表单并返回文本；失败返回 None。"""
|
||||
body, _ = self._post_text_with_reason(url, form_body)
|
||||
return body
|
||||
|
||||
def _post_text_with_reason(self, url: str, form_body: str) -> tuple[str | None, str]:
|
||||
"""POST 表单并返回 (文本, 失败原因)；成功时原因为空串。"""
|
||||
try:
|
||||
resp = self._session.post(
|
||||
url,
|
||||
@@ -111,7 +123,11 @@ class MainAppClient:
|
||||
data=form_body.encode("utf-8"),
|
||||
timeout=(5, 30),
|
||||
)
|
||||
return resp.text if resp.ok else None
|
||||
if not resp.ok:
|
||||
reason = f"HTTP {resp.status_code}"
|
||||
log.warning("POST %s 失败：%s", url, reason)
|
||||
return None, reason
|
||||
return resp.text, ""
|
||||
except Exception as e:
|
||||
log.debug("POST %s 失败：%s", url, e)
|
||||
return None
|
||||
log.warning("POST %s 失败：%s", url, e)
|
||||
return None, str(e)
|
||||
|
||||
+41
-2
@@ -76,20 +76,59 @@ class ProcessGuard:
|
||||
self._restart_count = 3
|
||||
return
|
||||
|
||||
log.error("主程序宕机，执行第 %d 次拉起：%s", attempts, self.config.main_start_cmd)
|
||||
# 拉起方式：配置了服务名则走 nssm restart（幂等），否则回退启动命令
|
||||
if self.config.main_service_name:
|
||||
start_desc = f"nssm restart {self.config.main_service_name}"
|
||||
else:
|
||||
start_desc = self.config.main_start_cmd
|
||||
log.error("主程序宕机，执行第 %d 次拉起：%s", attempts, start_desc)
|
||||
self.client.send("ALERT", {
|
||||
"level": "WARN",
|
||||
"msg": f"主程序宕机，Agent 正在执行第 {attempts} 次拉起",
|
||||
})
|
||||
try:
|
||||
if self.config.main_service_name:
|
||||
ok, reason = restart_main_service(self.config.nssm_path, self.config.main_service_name)
|
||||
if not ok:
|
||||
log.error("nssm 拉起失败：%s", reason)
|
||||
else:
|
||||
proc = start_main_app(self.config.main_start_cmd)
|
||||
log.info("启动命令已执行，pid=%s", proc.pid if proc else None)
|
||||
except Exception as e:
|
||||
log.error("启动命令执行失败：%s", e)
|
||||
log.error("拉起失败：%s", e)
|
||||
|
||||
|
||||
# ================= 公共工具（upgrade.py 复用） =================
|
||||
|
||||
def restart_main_service(nssm_path: str, service_name: str) -> tuple[bool, str]:
|
||||
"""
|
||||
通过 NSSM 重启主程序服务（幂等：无论服务处于停止/运行态均到运行态）。
|
||||
|
||||
nssm_path 为 nssm.exe 完整路径；service_name 为注册的 Windows 服务名。
|
||||
返回 (是否成功, 输出/原因)。nssm 输出打印到日志，权限不足（拒绝访问）等
|
||||
失败原因一眼可见。
|
||||
"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[nssm_path, "restart", service_name],
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
# nssm 输出为 UTF-16 编码（宽字符），需按 utf-16 解码
|
||||
out = ((proc.stdout or b"") + (proc.stderr or b"")).decode("utf-16", errors="replace").strip()
|
||||
ok = proc.returncode == 0
|
||||
if ok:
|
||||
log.info("nssm restart %s 成功：%s", service_name, out or "(无输出)")
|
||||
else:
|
||||
log.warning("nssm restart %s 失败（exit=%s）：%s", service_name, proc.returncode, out)
|
||||
if "拒绝访问" in out or "Access" in out:
|
||||
out += "（提示：重启服务需要管理员权限，请将 Agent 以 Windows 服务方式运行）"
|
||||
return ok, out
|
||||
except Exception as e:
|
||||
log.warning("nssm restart %s 执行异常：%s", service_name, e)
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def start_main_app(start_cmd: str) -> subprocess.Popen | None:
|
||||
"""
|
||||
拉起主程序。
|
||||
|
||||
@@ -98,6 +98,11 @@ class AgentClient:
|
||||
|
||||
def on_error(ws, error):
|
||||
self._connected = False
|
||||
# websocket-client 收到服务器关闭帧时会先回调本方法（特征：fin=1 opcode=8），
|
||||
# 这不是网络问题，而是被服务器拒绝（设备未注册 / token 错误）
|
||||
if "opcode=8" in str(error):
|
||||
log.warning("连接被服务器拒绝（设备未注册或 token 错误）：%s", error)
|
||||
else:
|
||||
log.warning("网络不通（无法连接管理服务器）：%s，将重连", error)
|
||||
# websocket-client 出错后必定回调 on_close，重连由 on_close 统一调度
|
||||
# 此处不重复调度，避免双重重连
|
||||
|
||||
@@ -27,9 +27,19 @@ main.health.url=http://127.0.0.1:8082/api/admin/health
|
||||
main.admin.token=
|
||||
|
||||
# \u4E3B\u7A0B\u5E8F\u542F\u52A8\u547D\u4EE4\uFF08\u770B\u95E8\u72D7\u68C0\u6D4B\u5230\u4E3B\u7A0B\u5E8F\u5B95\u673A\u540E\u6267\u884C\u6B64\u547D\u4EE4\u62C9\u8D77\uFF09
|
||||
# \u6CE8\u610F\uFF1A\u82E5\u914D\u7F6E\u4E86 main.service.name\uFF0C\u5219\u4F18\u5148\u7528 NSSM \u91CD\u542F\u670D\u52A1\uFF0C\u672C\u9879\u4E0D\u751F\u6548
|
||||
# \u5BF9\u5E94\u73AF\u5883\u53D8\u91CF\uFF1AAGENT_MAIN_START_CMD
|
||||
main.start.cmd=bin/service.bat main start
|
||||
|
||||
# NSSM \u53EF\u6267\u884C\u6587\u4EF6\u5B8C\u6574\u8DEF\u5F84\uFF08\u914D\u7F6E main.service.name \u65F6\u5FC5\u586B\uFF09
|
||||
# \u5BF9\u5E94\u73AF\u5883\u53D8\u91CF\uFF1AAGENT_NSSM_PATH
|
||||
nssm.path=D:\tools\nssm-2.24\win64\nssm.exe
|
||||
|
||||
# \u4E3B\u7A0B\u5E8F\u6CE8\u518C\u7684 Windows \u670D\u52A1\u540D\uFF08\u914D\u7F6E\u540E\u770B\u95E8\u72D7\u901A\u8FC7 nssm restart <\u670D\u52A1\u540D> \u62C9\u8D77\u4E3B\u7A0B\u5E8F\uFF09
|
||||
# \u7559\u7A7A\u5219\u56DE\u9000\u4F7F\u7528 main.start.cmd
|
||||
# \u5BF9\u5E94\u73AF\u5883\u53D8\u91CF\uFF1AAGENT_MAIN_SERVICE_NAME
|
||||
main.service.name=HospitalFront
|
||||
|
||||
# \u5FC3\u8DF3\u95F4\u9694\uFF0C\u5355\u4F4D\u79D2\uFF08\u9ED8\u8BA4 15\uFF0C\u987B\u5C0F\u4E8E\u7F51\u5173\u7A7A\u95F2\u8D85\u65F6\uFF0C\u5426\u5219\u8FDE\u63A5\u4F1A\u88AB\u7F51\u5173\u65AD\u5F00\uFF09
|
||||
# \u5BF9\u5E94\u73AF\u5883\u53D8\u91CF\uFF1AAGENT_HEARTBEAT_SEC
|
||||
heartbeat.sec=15
|
||||
@@ -37,3 +47,8 @@ heartbeat.sec=15
|
||||
# \u662F\u5426\u542F\u7528\u770B\u95E8\u72D7\u5B88\u62A4\u4E3B\u7A0B\u5E8F\uFF08true/false\uFF0C\u9ED8\u8BA4 true\uFF1Bfalse \u65F6\u4EC5\u505A\u8FD0\u7EF4\u547D\u4EE4\u901A\u9053\uFF09
|
||||
# \u5BF9\u5E94\u73AF\u5883\u53D8\u91CF\uFF1AAGENT_GUARD_ENABLE
|
||||
guard.enable=true
|
||||
|
||||
# \u65E5\u5FD7\u76EE\u5F55\uFF08\u76F8\u5BF9\u90E8\u7F72\u6839\u76EE\u5F55\uFF0C\u9ED8\u8BA4 logs\uFF1B\u6587\u4EF6 agent.log\uFF0C\u5355\u6587\u4EF6 5MB \u8F6E\u8F6C\u3001\u4FDD\u7559 5 \u4E2A\u5907\u4EFD\uFF09
|
||||
# \u670D\u52A1\u65B9\u5F0F\u8FD0\u884C\u65F6\u63A7\u5236\u53F0\u65E5\u5FD7\u4E0D\u53EF\u89C1\uFF0C\u8BF7\u4F9D\u8D56\u6B64\u6587\u4EF6\u65E5\u5FD7
|
||||
# \u5BF9\u5E94\u73AF\u5883\u53D8\u91CF\uFF1AAGENT_LOG_DIR
|
||||
log.dir=logs
|
||||
|
||||
Reference in New Issue
Block a user