104 lines
5.3 KiB
Bash
104 lines
5.3 KiB
Bash
#!/bin/bash
|
||
# VPS 常驻进程滚动重启（sanguo-live-supervisor + sanguo-shadow-desk）。
|
||
#
|
||
# 为什么需要：vps-deploy 只推代码+重启 sanguo-api；supervisor/影子柜台是常驻
|
||
# python 进程，不重启就一直跑旧代码（2026-08-15 实录：部署后 16h 旧进程原样跑）。
|
||
#
|
||
# 用法：
|
||
# bash scripts/nas_sync/vps_restart_resident.sh # 直接执行（Mac runner / 本机）
|
||
#
|
||
# 盘中保护：北京时间 工作日 09:15-15:30 跳过重启（盘中杀树丢撮合现场），
|
||
# 打印 SKIP_RESIDENT_RESTART 并 exit 0 —— 代码已就位，下次非盘中部署/手动补跑生效。
|
||
#
|
||
# 引擎核验(2026-08-30 live_17/20 教训)：杀树后立刻重拉,旧引擎的 QMT 会话尚未
|
||
# 在 QMT 端释放,6 引擎并发重连时个别 connect()=-1 → 整引擎退出,supervisor 标
|
||
# stopped 不再自愈——静默缺引擎直到有人看。现在:①杀后等 30s 让会话回收;
|
||
# ②重拉后清点实际引擎进程数,须追平杀前 DB running 期望,追不平 exit 1 让
|
||
# workflow 红灯(引擎侧另有 connect 退避重试兜底,见 runner_live.with_connect_retry)。
|
||
set -u
|
||
VPS=49.232.102.198
|
||
SSH="ssh -o ControlPath=none -o ConnectTimeout=20 $VPS"
|
||
COUNT_ENGINES="C:\\Python310\\python.exe -X utf8 C:\\sanguo_vnpy_v2\\scripts\\ci\\vps_count_engines.py"
|
||
|
||
# ---- 1) 盘中保护 ----
|
||
BJ_HM=$(TZ=Asia/Shanghai date +%H%M)
|
||
BJ_DOW=$(TZ=Asia/Shanghai date +%u)
|
||
if [ "$BJ_DOW" -le 5 ] && [ "$BJ_HM" -ge 0915 ] && [ "$BJ_HM" -le 1530 ]; then
|
||
echo "SKIP_RESIDENT_RESTART 北京时间盘中(周$BJ_DOW $BJ_HM)——supervisor/影子保持运行,下次非盘中部署生效"
|
||
exit 0
|
||
fi
|
||
|
||
# ---- 1.5) 期望引擎数(必须在杀树前取:重拉后 DB status 已被死亡事件改写) ----
|
||
EXPECTED=$($SSH "$COUNT_ENGINES expected" 2>/dev/null || true)
|
||
EXP_LIVE=$(grep -oE 'LIVE=[0-9]+' <<<"${EXPECTED:-}" | head -1 | cut -d= -f2)
|
||
EXP_SHADOW=$(grep -oE 'SHADOW=[0-9]+' <<<"${EXPECTED:-}" | head -1 | cut -d= -f2)
|
||
echo "杀前期望: live=${EXP_LIVE:-?} shadow=${EXP_SHADOW:-?}(DB running)"
|
||
|
||
# ---- 2) 查 supervisor/影子 python 进程树根（动态查 PID：账户号会变）----
|
||
PIDS=$($SSH "wmic process where \"name='python.exe' and (commandline like '%sanguo_live%supervisor%' or commandline like '%sanguo_trader.shadow%')\" get processid /format:csv" 2>/dev/null \
|
||
| tr -d '\r' | grep -oE '[0-9]+$' | sort -u || true)
|
||
echo "重启前进程: $(echo $PIDS | tr '\n' ' ')"
|
||
|
||
# ---- 3) 杀全树（/T 带走 runner_live / shadow --account 子进程）----
|
||
for pid in $PIDS; do
|
||
$SSH "taskkill /F /T /PID $pid" 2>/dev/null || echo " (PID $pid 已自行退出)"
|
||
done
|
||
# schtask 收尸（壳可能残留 running 态，end 掉才好 /run）
|
||
$SSH "schtasks /end /tn sanguo-live-supervisor" 2>/dev/null || true
|
||
$SSH "schtasks /end /tn sanguo-shadow-desk" 2>/dev/null || true
|
||
|
||
# ---- 4) 重拉(先等 QMT 会话回收:旧引擎会话被硬杀,QMT 端释放要秒级~几十秒,
|
||
# 立刻重拉则 6 引擎并发 connect 撞回收窗口——08-30 live_17/20 死因) ----
|
||
sleep 30
|
||
# schtasks /run 带重试(2026-08-26 run756 实锤: 短时海量 ssh 连接可被 sshd 重置
|
||
# `kex_exchange_identification: Connection reset`——单发失败=影子舰队整夜静默,
|
||
# 需手动 schtasks /run 补拉; 3 次重试吃掉瞬时抖动)
|
||
run_task() {
|
||
local tn=$1 i
|
||
for i in 1 2 3; do
|
||
$SSH "schtasks /run /tn $tn" && return 0
|
||
echo " (拉起 $tn 第${i}次失败, 5s 后重试)"
|
||
sleep 5
|
||
done
|
||
return 1
|
||
}
|
||
run_task sanguo-live-supervisor || { echo "❌ supervisor 拉起失败"; exit 1; }
|
||
run_task sanguo-shadow-desk || { echo "❌ shadow-desk 拉起失败"; exit 1; }
|
||
|
||
# ---- 5) 等常驻进程回来（内存实测 40-60s 起齐，给 120s 余量）----
|
||
for i in $(seq 1 12); do
|
||
sleep 10
|
||
GOT=$($SSH "wmic process where \"name='python.exe' and (commandline like '%sanguo_live%supervisor%' or commandline like '%sanguo_trader.shadow%')\" get processid /format:csv" 2>/dev/null \
|
||
| tr -d '\r' | grep -cE '[0-9]+$' || true)
|
||
echo " ${i}0s: 常驻已起 $GOT 个进程"
|
||
if [ "${GOT:-0}" -ge 2 ]; then
|
||
break
|
||
fi
|
||
if [ "$i" = 12 ]; then
|
||
echo "❌ 120s 内常驻进程未起齐(期望≥2: supervisor+shadow auto)"
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
# ---- 6) 引擎核验:实际引擎进程数须追平杀前期望,追不平红灯 ----
|
||
if [ -z "${EXP_LIVE:-}" ] || [ -z "${EXP_SHADOW:-}" ]; then
|
||
echo "⚠️ 期望值读取失败(vps_count_engines.py 未部署?首次部署后自然就位)——跳过引擎核验"
|
||
echo "RESIDENT_RESTART_DONE supervisor+影子已重启并吃上新代码(引擎核验跳过)"
|
||
exit 0
|
||
fi
|
||
ACT_LIVE=0; ACT_SHADOW=0
|
||
for i in $(seq 1 12); do
|
||
sleep 10
|
||
ACTUAL=$($SSH "$COUNT_ENGINES actual" 2>/dev/null || true)
|
||
ACT_LIVE=$(grep -oE 'LIVE=[0-9]+' <<<"${ACTUAL:-}" | head -1 | cut -d= -f2)
|
||
ACT_SHADOW=$(grep -oE 'SHADOW=[0-9]+' <<<"${ACTUAL:-}" | head -1 | cut -d= -f2)
|
||
ACT_LIVE=${ACT_LIVE:-0}; ACT_SHADOW=${ACT_SHADOW:-0}
|
||
echo " ${i}0s: 引擎 live=$ACT_LIVE/$EXP_LIVE shadow=$ACT_SHADOW/$EXP_SHADOW"
|
||
if [ "$ACT_LIVE" -ge "$EXP_LIVE" ] && [ "$ACT_SHADOW" -ge "$EXP_SHADOW" ]; then
|
||
echo "RESIDENT_RESTART_DONE supervisor+影子已重启并吃上新代码,引擎核验通过(live=$ACT_LIVE shadow=$ACT_SHADOW)"
|
||
exit 0
|
||
fi
|
||
done
|
||
echo "❌ 引擎未起齐: live=$ACT_LIVE/$EXP_LIVE shadow=$ACT_SHADOW/$EXP_SHADOW —— 有引擎重拉后退出(连不上 QMT/策略崩),人工排查 logs/live_*.log"
|
||
exit 1
|