lianlonggangandClaude
68751c2354
- WebSocket 连接 B 端（注册/心跳/指数退避重连） - 命令分发（10 种远程运维命令） - 看门狗守护、日志实时流、在线升级含自动回滚 - 修复 main.py 日志格式 style 与占位符不匹配的问题 - 补充 README、配置示例、依赖清单 Co-Authored-By: Claude <noreply@anthropic.com>
268 lines
10 KiB
Python
268 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""主程序升级命令。
|
||
|
||
流程（见 05-部署运维.md SOP）：
|
||
1. 从 B 端下载新 jar 到 tmp（SHA-256 校验，防传输损坏）
|
||
2. 备份当前 bin/hospital-front.jar → backup/hospital-front-{时间戳}.jar（保留最近 5 个）
|
||
3. 优雅停止主程序（调 /api/admin/restart；超时强杀）
|
||
4. 替换 jar
|
||
5. 调启动命令拉起主程序
|
||
6. 轮询健康检查（最长 90 秒）
|
||
7. 成功 → 上报；失败 → 用备份 jar 自动回滚并重启 → 上报
|
||
|
||
期间整个流程串行（upgrade_lock 防并发升级）。
|
||
"""
|
||
|
||
import glob
|
||
import hashlib
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
|
||
import requests
|
||
|
||
from agent.process_guard import start_main_app
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
# 备份保留数量
|
||
KEEP_BACKUPS = 5
|
||
|
||
|
||
class UpgradeCommand:
|
||
"""升级 SOP 执行器。"""
|
||
|
||
def __init__(self, config, main_app_client, client):
|
||
self.config = config
|
||
self.main_app_client = main_app_client
|
||
self.client = client
|
||
|
||
# 升级互斥锁（同一时刻只允许一个升级在跑）
|
||
self._upgrade_lock = threading.Lock()
|
||
|
||
def upgrade(self, version: str, download_url: str, sha256: str | None) -> dict:
|
||
"""
|
||
执行升级。
|
||
|
||
:param version: 新版本号（仅用于日志与上报）
|
||
:param download_url: B 端提供的 jar 下载地址（A 主动出站拉取）
|
||
:param sha256: 可选校验值
|
||
"""
|
||
with self._upgrade_lock:
|
||
log.info("开始升级到版本 %s：%s", version, download_url)
|
||
self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "download"})
|
||
|
||
# ===== 1. 下载 =====
|
||
tmp_jar = os.path.join("tmp", f"upgrade-{version}.jar")
|
||
os.makedirs(os.path.dirname(tmp_jar), exist_ok=True)
|
||
try:
|
||
with requests.get(download_url, stream=True, timeout=(10, 120)) as resp:
|
||
if resp.status_code != 200:
|
||
return self._fail(version, f"下载失败：HTTP {resp.status_code}")
|
||
with open(tmp_jar, "wb") as f:
|
||
for chunk in resp.iter_content(8192):
|
||
f.write(chunk)
|
||
except Exception as e:
|
||
return self._fail(version, f"下载异常：{e}")
|
||
log.info("下载完成：%s（%d 字节）", os.path.basename(tmp_jar), os.path.getsize(tmp_jar))
|
||
|
||
# ===== 2. 校验 =====
|
||
if sha256:
|
||
try:
|
||
actual = sha256_file(tmp_jar)
|
||
except Exception as e:
|
||
return self._fail(version, f"SHA-256 计算异常：{e}")
|
||
if actual.lower() != sha256.lower():
|
||
os.remove(tmp_jar)
|
||
return self._fail(version, f"SHA-256 校验失败：expected={sha256}, actual={actual}")
|
||
log.info("SHA-256 校验通过")
|
||
|
||
# ===== 3. 备份 =====
|
||
jar_file = os.path.join("bin", "hospital-front.jar")
|
||
backup_file = os.path.join("backup", f"hospital-front-{int(time.time() * 1000)}.jar")
|
||
os.makedirs(os.path.dirname(backup_file), exist_ok=True)
|
||
if os.path.exists(jar_file):
|
||
try:
|
||
shutil.copyfile(jar_file, backup_file)
|
||
except Exception as e:
|
||
return self._fail(version, f"备份失败：{e}")
|
||
log.info("已备份当前版本 → %s", os.path.basename(backup_file))
|
||
self._clean_old_backups()
|
||
|
||
# ===== 4. 停止主程序 =====
|
||
self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "stop"})
|
||
self._stop_main_app()
|
||
log.info("主程序已停止")
|
||
|
||
# ===== 5. 替换 =====
|
||
# 注意：Agent 为独立 Python 进程，不占用 bin/hospital-front.jar，
|
||
# 可安全替换；无需再维护 Agent jar 副本（Java 版遗留逻辑）。
|
||
try:
|
||
copy_with_retry(tmp_jar, jar_file, 10)
|
||
os.remove(tmp_jar)
|
||
except Exception as e:
|
||
self._rollback(backup_file)
|
||
return self._fail(version, f"替换 jar 失败，已回滚：{e}")
|
||
log.info("jar 已替换")
|
||
|
||
# ===== 6. 拉起 + 健康检查 =====
|
||
self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "restart"})
|
||
try:
|
||
start_main_app(self.config.main_start_cmd)
|
||
except Exception as e:
|
||
log.error("主程序拉起失败：%s", e)
|
||
|
||
self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "healthcheck"})
|
||
if self._wait_healthy(90):
|
||
log.info("升级成功：%s", version)
|
||
return {"ok": True, "version": version, "msg": "升级成功"}
|
||
|
||
# ===== 7. 健康检查失败 → 回滚 =====
|
||
log.error("新版本健康检查失败，自动回滚到 %s", os.path.basename(backup_file))
|
||
self.client.send("UPGRADE_PROGRESS", {"version": version, "stage": "rollback"})
|
||
self._stop_main_app()
|
||
self._rollback(backup_file)
|
||
try:
|
||
start_main_app(self.config.main_start_cmd)
|
||
except Exception as e:
|
||
log.error("回滚后拉起失败：%s", e)
|
||
if self._wait_healthy(90):
|
||
return self._fail(version, f"新版本启动失败，已自动回滚到 {os.path.basename(backup_file)} 并恢复服务")
|
||
|
||
self.client.send("ALERT", {
|
||
"level": "CRITICAL",
|
||
"msg": f"升级失败且回滚后仍不健康，请人工介入！版本={version}",
|
||
})
|
||
return self._fail(version, "升级失败且回滚后仍不健康，请人工介入")
|
||
|
||
# ================= 内部步骤 =================
|
||
|
||
def _stop_main_app(self):
|
||
"""停止主程序：优先优雅（admin API），超时后按 PID 强杀；确保进程完全退出（jar 锁释放）。"""
|
||
pid = read_pid_file(os.path.join("pid", "hospital-front.pid"))
|
||
if self.main_app_client.restart(100):
|
||
# 等待健康探测失败（最长 30 秒）
|
||
deadline = time.time() + 30
|
||
while time.time() < deadline:
|
||
if self.main_app_client.health().get("ok") is not True:
|
||
break
|
||
time.sleep(1)
|
||
# 兜底：按 PID 文件强杀
|
||
kill_by_pid_file(os.path.join("pid", "hospital-front.pid"))
|
||
# 等待进程真正退出（Windows 下进程退出后才释放 jar 文件锁）
|
||
if pid is not None:
|
||
wait_process_exit(pid, 15)
|
||
time.sleep(1)
|
||
|
||
def _wait_healthy(self, timeout_sec: int) -> bool:
|
||
"""轮询等待主程序健康。"""
|
||
deadline = time.time() + timeout_sec
|
||
while time.time() < deadline:
|
||
if self.main_app_client.health().get("ok") is True:
|
||
return True
|
||
time.sleep(3)
|
||
return False
|
||
|
||
def _rollback(self, backup_file: str):
|
||
"""回滚：备份 jar 覆盖回 bin。"""
|
||
try:
|
||
if os.path.exists(backup_file):
|
||
jar_file = os.path.join("bin", "hospital-front.jar")
|
||
copy_with_retry(backup_file, jar_file, 10)
|
||
log.info("已回滚到 %s", os.path.basename(backup_file))
|
||
except Exception as e:
|
||
log.error("回滚失败：%s", e)
|
||
|
||
def _clean_old_backups(self):
|
||
"""清理旧备份（按文件名倒序保留最近 N 个）。"""
|
||
backups = sorted(glob.glob(os.path.join("backup", "hospital-front-*")), reverse=True)
|
||
for old in backups[KEEP_BACKUPS:]:
|
||
try:
|
||
os.remove(old)
|
||
except Exception:
|
||
pass
|
||
|
||
def _fail(self, version: str, msg: str) -> dict:
|
||
"""统一失败上报。"""
|
||
log.error("升级失败：%s", msg)
|
||
return {"ok": False, "version": version, "msg": msg}
|
||
|
||
|
||
# ================= 模块级工具函数 =================
|
||
|
||
def read_pid_file(pid_file: str) -> str | None:
|
||
"""读 PID 文件，无或为空返回 None。"""
|
||
try:
|
||
if not os.path.exists(pid_file):
|
||
return None
|
||
with open(pid_file, "r", encoding="utf-8") as f:
|
||
s = f.read().strip()
|
||
return s or None
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def kill_by_pid_file(pid_file: str):
|
||
"""按 PID 文件强杀进程（Windows taskkill / Linux kill -9）。"""
|
||
pid = read_pid_file(pid_file)
|
||
if not pid:
|
||
return
|
||
try:
|
||
if os.name == "nt":
|
||
subprocess.run(["taskkill", "/PID", pid, "/T", "/F"],
|
||
capture_output=True, timeout=15)
|
||
else:
|
||
subprocess.run(["kill", "-9", pid],
|
||
capture_output=True, timeout=15)
|
||
except Exception as e:
|
||
log.warning("PID 强杀失败：%s", e)
|
||
|
||
|
||
def wait_process_exit(pid: str, timeout_sec: int):
|
||
"""轮询等待进程退出（tasklist 查不到即退出）。"""
|
||
deadline = time.time() + timeout_sec
|
||
while time.time() < deadline:
|
||
alive = False
|
||
try:
|
||
if os.name == "nt":
|
||
out = subprocess.run(
|
||
["tasklist", "/FI", f"PID eq {pid}"],
|
||
capture_output=True, timeout=10,
|
||
).stdout.decode("utf-8", errors="ignore")
|
||
alive = pid in out
|
||
else:
|
||
alive = subprocess.run(["kill", "-0", pid], capture_output=True).returncode == 0
|
||
except Exception:
|
||
alive = False
|
||
if not alive:
|
||
return
|
||
time.sleep(0.5)
|
||
|
||
|
||
def copy_with_retry(src: str, dst: str, retries: int):
|
||
"""带重试的文件覆盖（Windows 下进程退出有延迟，锁释放需等待）。"""
|
||
last_error = None
|
||
for _ in range(retries):
|
||
try:
|
||
if os.path.exists(dst):
|
||
os.remove(dst)
|
||
if not os.path.exists(dst):
|
||
shutil.copyfile(src, dst)
|
||
return
|
||
except Exception as e:
|
||
last_error = e
|
||
time.sleep(1)
|
||
raise RuntimeError(f"文件替换失败（重试 {retries} 次）：{os.path.basename(dst)}") from last_error
|
||
|
||
|
||
def sha256_file(path: str) -> str:
|
||
"""文件 SHA-256（分块读取，支持大文件）。"""
|
||
digest = hashlib.sha256()
|
||
with open(path, "rb") as f:
|
||
for chunk in iter(lambda: f.read(8192), b""):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|