feat: 升级流程增强 + 网关化配置 + 重连循环化

- 配置化：gateway.url/socket.path/download.prefix（网关基础地址统一推导 ws 与下载地址）、 main.home 主目录（jar/pid 相对它推导）、order.basic；properties 中文 \uXXXX 转义自动还原 - 升级流程：下载 PK 魔数检查与诊断日志；SHA 校验；替换改「改名腾位」绕开 Windows 文件锁延迟； nssm stop/start 驱动停止与拉起（pid 强杀回退）；升级中标志暂停看门狗拉起避免抢占 jar 锁 - ws_client：重连改循环实现修复递归栈溢出（长时间断线后 maximum recursion depth）；退避封顶延长至 5 分钟；心跳线程独立修复「暂无心跳数据」 - 新增命令：GET_APIS/HTTP_PROXY（接口测试代理，纯透传鉴权）、GET_LOG_DOWNLOAD（日志下载备用） - 诊断增强：taskkill/进程退出/替换重试逐次日志 Co-Authored-By: Claude <noreply@anthropic.com>
2026-09-03 16:52:34 +08:00
parent b572697e17
commit 0b13de14eb
7 changed files with 366 additions and 152 deletions
@@ -24,7 +24,7 @@ import time

 import requests

-from agent.process_guard import start_main_app
+from agent.process_guard import start_main_app, stop_main_service, start_main_service, _upgrading

 log = logging.getLogger(__name__)

@@ -48,101 +48,142 @@ class UpgradeCommand:
        执行升级。

        :param version:     新版本号（仅用于日志与上报）
-        :param download_url: B 端提供的 jar 下载地址（A 主动出站拉取）
+        :param download_url: B 端提供的下载地址：相对路径（以 / 开头，经网关下载）或完整 URL
        :param sha256:      可选校验值
        """
        with self._upgrade_lock:
-            log.info("开始升级到版本 %s：%s", version, download_url)
-            self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "download"})
-
-            # ===== 1. 下载 =====
-            tmp_jar = os.path.join("tmp", f"upgrade-{version}.jar")
-            os.makedirs(os.path.dirname(tmp_jar), exist_ok=True)
+            # 置「升级中」标志：升级主动停主程序期间，看门狗跳过拉起，
+            # 避免新进程抢占 jar 文件锁导致替换失败
+            _upgrading.set()
            try:
-                with requests.get(download_url, stream=True, timeout=(10, 120)) as resp:
-                    if resp.status_code != 200:
-                        return self._fail(version, f"下载失败：HTTP {resp.status_code}")
-                    with open(tmp_jar, "wb") as f:
-                        for chunk in resp.iter_content(8192):
-                            f.write(chunk)
+                return self._do_upgrade(version, download_url, sha256)
+            finally:
+                _upgrading.clear()
+
+    def _do_upgrade(self, version: str, download_url: str, sha256: str | None) -> dict:
+        """升级实际执行（由 upgrade 加锁 + 置标志后调用）。"""
+        # 相对路径（B 端下发的文件路径）：拼 网关基础地址 + 下载路由前缀 + 文件路径
+        # 例：http://网关:19901 + /file/down + /hf-upgrade/20260902/xxx.jar
+        if download_url.startswith("/"):
+            if not self.config.gateway_url:
+                return self._fail(version, "B 端下发了相对下载路径，但 Agent 未配置 gateway.url")
+            prefix = "/" + self.config.download_prefix.strip("/")
+            download_url = self.config.gateway_url.rstrip("/") + prefix + "/" + download_url.strip("/")
+        log.info("开始升级到版本 %s：%s", version, download_url)
+        self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "download"})
+
+        # ===== 1. 下载 =====
+        tmp_jar = os.path.join("tmp", f"upgrade-{version}.jar")
+        os.makedirs(os.path.dirname(tmp_jar), exist_ok=True)
+        try:
+            with requests.get(download_url, stream=True, timeout=(10, 120)) as resp:
+                if resp.status_code != 200:
+                    return self._fail(version, f"下载失败：HTTP {resp.status_code}")
+                content_type = resp.headers.get("Content-Type", "")
+                with open(tmp_jar, "wb") as f:
+                    for chunk in resp.iter_content(8192):
+                        f.write(chunk)
+        except Exception as e:
+            return self._fail(version, f"下载异常：{e}")
+
+        size = os.path.getsize(tmp_jar)
+        # 诊断：打印响应类型与文件头（下载到错误页面/JSON 而非 jar 时一眼可见）
+        with open(tmp_jar, "rb") as f:
+            head = f.read(64)
+        head_preview = head[:32].decode("utf-8", errors="replace")
+        log.info("下载完成：%s（%d 字节，Content-Type=%s，文件头=%r）",
+                 os.path.basename(tmp_jar), size, content_type, head_preview)
+
+        # jar 为 zip 格式（PK 魔数）。文件服务器下载失败常返回 200+错误 JSON/HTML，
+        # 此处提前拦截并给出明确原因，避免误报 SHA-256 校验失败
+        if not head.startswith(b"PK"):
+            # 疑似鉴权/路径错误，读出响应体前 200 字符帮助定位
+            with open(tmp_jar, "rb") as f:
+                body_preview = f.read(200).decode("utf-8", errors="replace")
+            os.remove(tmp_jar)
+            return self._fail(
+                version,
+                f"下载内容不是 jar（PK 魔数缺失）：大小={size}，Content-Type={content_type}，"
+                f"响应开头={body_preview!r}（疑似下载地址需鉴权或路径错误）",
+            )
+
+        # ===== 2. 校验 =====
+        if sha256:
+            try:
+                actual = sha256_file(tmp_jar)
            except Exception as e:
-                return self._fail(version, f"下载异常：{e}")
-            log.info("下载完成：%s（%d 字节）", os.path.basename(tmp_jar), os.path.getsize(tmp_jar))
-
-            # ===== 2. 校验 =====
-            if sha256:
-                try:
-                    actual = sha256_file(tmp_jar)
-                except Exception as e:
-                    return self._fail(version, f"SHA-256 计算异常：{e}")
-                if actual.lower() != sha256.lower():
-                    os.remove(tmp_jar)
-                    return self._fail(version, f"SHA-256 校验失败：expected={sha256}, actual={actual}")
-                log.info("SHA-256 校验通过")
-
-            # ===== 3. 备份 =====
-            jar_file = os.path.join("bin", "hospital-front.jar")
-            backup_file = os.path.join("backup", f"hospital-front-{int(time.time() * 1000)}.jar")
-            os.makedirs(os.path.dirname(backup_file), exist_ok=True)
-            if os.path.exists(jar_file):
-                try:
-                    shutil.copyfile(jar_file, backup_file)
-                except Exception as e:
-                    return self._fail(version, f"备份失败：{e}")
-                log.info("已备份当前版本 → %s", os.path.basename(backup_file))
-                self._clean_old_backups()
-
-            # ===== 4. 停止主程序 =====
-            self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "stop"})
-            self._stop_main_app()
-            log.info("主程序已停止")
-
-            # ===== 5. 替换 =====
-            # 注意：Agent 为独立 Python 进程，不占用 bin/hospital-front.jar，
-            # 可安全替换；无需再维护 Agent jar 副本（Java 版遗留逻辑）。
-            try:
-                copy_with_retry(tmp_jar, jar_file, 10)
+                return self._fail(version, f"SHA-256 计算异常：{e}")
+            if actual.lower() != sha256.lower():
                os.remove(tmp_jar)
-            except Exception as e:
-                self._rollback(backup_file)
-                return self._fail(version, f"替换 jar 失败，已回滚：{e}")
-            log.info("jar 已替换")
+                return self._fail(version, f"SHA-256 校验失败：expected={sha256}, actual={actual}")
+            log.info("SHA-256 校验通过")

-            # ===== 6. 拉起 + 健康检查 =====
-            self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "restart"})
+        # ===== 3. 备份 =====
+        # jar 路径来自配置（main.jar.path），适配不同部署结构
+        #（如 Agent 与主程序分目录、jar 文件名带版本号等）
+        jar_file = self.config.main_jar
+        backup_file = os.path.join("backup", f"hospital-front-{int(time.time() * 1000)}.jar")
+        os.makedirs(os.path.dirname(backup_file), exist_ok=True)
+        if os.path.exists(jar_file):
            try:
-                start_main_app(self.config.main_start_cmd)
+                shutil.copyfile(jar_file, backup_file)
            except Exception as e:
-                log.error("主程序拉起失败：%s", e)
+                return self._fail(version, f"备份失败：{e}")
+            log.info("已备份当前版本 → %s", os.path.basename(backup_file))
+            self._clean_old_backups()

-            self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "healthcheck"})
-            if self._wait_healthy(90):
-                log.info("升级成功：%s", version)
-                return {"ok": True, "version": version, "msg": "升级成功"}
+        # ===== 4. 停止主程序 =====
+        self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "stop"})
+        self._stop_main_app()
+        log.info("主程序已停止")

-            # ===== 7. 健康检查失败 → 回滚 =====
-            log.error("新版本健康检查失败，自动回滚到 %s", os.path.basename(backup_file))
-            self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "rollback"})
-            self._stop_main_app()
+        # ===== 5. 替换 =====
+        # 注意：Agent 为独立 Python 进程，不占用 bin/hospital-front.jar，
+        # 可安全替换；无需再维护 Agent jar 副本（Java 版遗留逻辑）。
+        try:
+            copy_with_retry(tmp_jar, jar_file, 10)
+            os.remove(tmp_jar)
+        except Exception as e:
            self._rollback(backup_file)
-            try:
-                start_main_app(self.config.main_start_cmd)
-            except Exception as e:
-                log.error("回滚后拉起失败：%s", e)
-            if self._wait_healthy(90):
-                return self._fail(version, f"新版本启动失败，已自动回滚到 {os.path.basename(backup_file)} 并恢复服务")
+            return self._fail(version, f"替换 jar 失败，已回滚：{e}")
+        log.info("jar 已替换")

-            self.client.send("ALERT", {
-                "level": "CRITICAL",
-                "msg": f"升级失败且回滚后仍不健康，请人工介入！版本={version}",
-            })
-            return self._fail(version, "升级失败且回滚后仍不健康，请人工介入")
+        # ===== 6. 拉起 + 健康检查 =====
+        self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "restart"})
+        self._start_main_app()
+
+        self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "healthcheck"})
+        if self._wait_healthy(90):
+            log.info("升级成功：%s", version)
+            return {"ok": True, "version": version, "msg": "升级成功"}
+
+        # ===== 7. 健康检查失败 → 回滚 =====
+        log.error("新版本健康检查失败，自动回滚到 %s", os.path.basename(backup_file))
+        self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "rollback"})
+        self._stop_main_app()
+        self._rollback(backup_file)
+        self._start_main_app()
+        if self._wait_healthy(90):
+            return self._fail(version, f"新版本启动失败，已自动回滚到 {os.path.basename(backup_file)} 并恢复服务")
+
+        self.client.send("ALERT", {
+            "level": "CRITICAL",
+            "msg": f"升级失败且回滚后仍不健康，请人工介入！版本={version}",
+        })
+        return self._fail(version, "升级失败且回滚后仍不健康，请人工介入")

    # ================= 内部步骤 =================

    def _stop_main_app(self):
-        """停止主程序：优先优雅（admin API），超时后按 PID 强杀；确保进程完全退出（jar 锁释放）。"""
-        pid = read_pid_file(os.path.join("pid", "hospital-front.pid"))
+        """停止主程序：优先 nssm stop（等进程退出再返回，文件锁释放可靠）；
+        未配置服务名时回退 admin API + PID 强杀。"""
+        if self.config.main_service_name:
+            # nssm 管理的服务：nssm stop 等待服务进程退出后返回，文件锁释放可靠
+            stop_main_service(self.config.nssm_path, self.config.main_service_name)
+            time.sleep(1)   # 保险等锁释放
+            return
+        # 回退：admin API 触发优雅退出 + PID 强杀
+        pid = read_pid_file(self.config.main_pid)
        if self.main_app_client.restart(100):
            # 等待健康探测失败（最长 30 秒）
            deadline = time.time() + 30
@@ -151,12 +192,23 @@ class UpgradeCommand:
                    break
                time.sleep(1)
        # 兜底：按 PID 文件强杀
-        kill_by_pid_file(os.path.join("pid", "hospital-front.pid"))
+        kill_by_pid_file(self.config.main_pid)
        # 等待进程真正退出（Windows 下进程退出后才释放 jar 文件锁）
        if pid is not None:
            wait_process_exit(pid, 15)
        time.sleep(1)

+    def _start_main_app(self):
+        """拉起主程序：优先 nssm start（服务模式），否则回退启动命令。"""
+        if self.config.main_service_name:
+            start_main_service(self.config.nssm_path, self.config.main_service_name)
+        else:
+            try:
+                proc = start_main_app(self.config.main_start_cmd)
+                log.info("启动命令已执行，pid=%s", proc.pid if proc else None)
+            except Exception as e:
+                log.error("主程序拉起失败：%s", e)
+
    def _wait_healthy(self, timeout_sec: int) -> bool:
        """轮询等待主程序健康。"""
        deadline = time.time() + timeout_sec
@@ -167,11 +219,10 @@ class UpgradeCommand:
        return False

    def _rollback(self, backup_file: str):
-        """回滚：备份 jar 覆盖回 bin。"""
+        """回滚：备份 jar 覆盖回主程序 jar（路径来自配置）。"""
        try:
            if os.path.exists(backup_file):
-                jar_file = os.path.join("bin", "hospital-front.jar")
-                copy_with_retry(backup_file, jar_file, 10)
+                copy_with_retry(backup_file, self.config.main_jar, 10)
                log.info("已回滚到 %s", os.path.basename(backup_file))
        except Exception as e:
            log.error("回滚失败：%s", e)
@@ -209,14 +260,20 @@ def kill_by_pid_file(pid_file: str):
    """按 PID 文件强杀进程（Windows taskkill / Linux kill -9）。"""
    pid = read_pid_file(pid_file)
    if not pid:
+        log.warning("强杀跳过：PID 文件 %s 无内容或不存在", pid_file)
        return
+    log.info("按 PID 强杀：%s（来自 %s）", pid, pid_file)
    try:
        if os.name == "nt":
-            subprocess.run(["taskkill", "/PID", pid, "/T", "/F"],
-                           capture_output=True, timeout=15)
+            r = subprocess.run(["taskkill", "/PID", pid, "/T", "/F"],
+                               capture_output=True, timeout=15)
+            out = (r.stdout or b"") + (r.stderr or b"")
+            log.info("taskkill 退出码=%s，输出=%s", r.returncode,
+                     out.decode("gbk", errors="replace").strip())
        else:
-            subprocess.run(["kill", "-9", pid],
-                           capture_output=True, timeout=15)
+            r = subprocess.run(["kill", "-9", pid],
+                               capture_output=True, timeout=15)
+            log.info("kill -9 退出码=%s", r.returncode)
    except Exception as e:
        log.warning("PID 强杀失败：%s", e)

@@ -231,31 +288,49 @@ def wait_process_exit(pid: str, timeout_sec: int):
                out = subprocess.run(
                    ["tasklist", "/FI", f"PID eq {pid}"],
                    capture_output=True, timeout=10,
-                ).stdout.decode("utf-8", errors="ignore")
+                ).stdout.decode("gbk", errors="ignore")
                alive = pid in out
            else:
                alive = subprocess.run(["kill", "-0", pid], capture_output=True).returncode == 0
        except Exception:
            alive = False
        if not alive:
+            log.info("进程 %s 已退出", pid)
            return
        time.sleep(0.5)
+    log.warning("等待进程 %s 退出超时（%ds），可能仍占用 jar 锁", pid, timeout_sec)


 def copy_with_retry(src: str, dst: str, retries: int):
-    """带重试的文件覆盖（Windows 下进程退出有延迟，锁释放需等待）。"""
+    """带重试的文件覆盖（Windows 下用「改名腾位」策略绕开文件锁延迟）。
+
+    进程退出后 Windows 对 jar 的内存映射句柄释放有滞后，直接 os.remove 会持续
+    WinError 32。改为：先把旧 jar 改名为 .old（Windows 允许重命名正在使用的文件），
+    腾出原文件名写入新 jar；.old 留待进程完全退出后清理。
+    """
    last_error = None
-    for _ in range(retries):
+    old_name = dst + ".old"
+    for i in range(retries):
        try:
+            # 清理上一次残留的 .old（若进程已退出可删；删不动也无妨，继续）
+            if os.path.exists(old_name):
+                try:
+                    os.remove(old_name)
+                except Exception:
+                    pass
+            # 把占用中的旧 jar 改名腾位（关键：即使文件被锁，重命名通常也能成功）
            if os.path.exists(dst):
-                os.remove(dst)
-            if not os.path.exists(dst):
-                shutil.copyfile(src, dst)
-                return
+                os.replace(dst, old_name)
+            # 原文件名已腾空，写入新 jar
+            shutil.copyfile(src, dst)
+            return
        except Exception as e:
            last_error = e
+            log.warning("替换重试 %d/%d 失败：%s（dst=%s）", i + 1, retries, e, dst)
        time.sleep(1)
-    raise RuntimeError(f"文件替换失败（重试 {retries} 次）：{os.path.basename(dst)}") from last_error
+    raise RuntimeError(
+        f"文件替换失败（重试 {retries} 次）：{os.path.basename(dst)}，最后错误：{last_error}"
+    ) from last_error


 def sha256_file(path: str) -> str: