diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 2e1e4ce..9a2f5f7 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -84,7 +84,7 @@ jobs: rsync -az --delete frontend/dist/ sanguo-nas:/volume1/homes/admin/.sanguo_projects/sanguo_vnpy_v2/frontend/dist/ - name: restart NAS 容器(加载新后端代码) + 等 web 就绪 run: | - ssh sanguo-nas '/var/packages/Docker/target/usr/bin/docker restart sanguo_vnpy_v2' + ssh -o ConnectTimeout=20 sanguo-nas '/var/packages/Docker/target/usr/bin/docker restart sanguo_vnpy_v2' for i in $(seq 1 30); do curl -sf http://192.168.2.154:8000/ -o /dev/null && { echo "web ready"; exit 0; } sleep 2 @@ -126,7 +126,7 @@ jobs: echo "✅ /health 200, 存活探针通" - name: NAS 副本数据可读(dbbardata) run: | - ssh sanguo-nas "python3 -c 'import sqlite3 + ssh -o ConnectTimeout=20 sanguo-nas "python3 -c 'import sqlite3 c=sqlite3.connect(\"/volume1/stock/sanguo_vnpy_v2/data_backup/quant_trading.db\") n=c.execute(\"SELECT COUNT(*) FROM dbbardata WHERE symbol=?\",(\"600519\",)).fetchone()[0] print(\"dbbardata 600519 rows:\", n) @@ -144,7 +144,9 @@ jobs: RANGE="HEAD~50..HEAD" echo "⚠️ vps-deployed tag 不存在(首次/未推过 VPS), 查最近 50 commit" fi - export VPS_LOG="$(git log --format='%h %s' "$RANGE" --grep='\[vps\]' || true)" + # gap 按 subject 算(与 enforce-label 同定义): 不能用 --grep='\[vps\]'——它匹配整个 + # message, commit body 里引用的命令文本会误命中(曾误开 issue#7)。与 vps_finalize.sh 同款。 + export VPS_LOG="$(git log --format='%h %s' "$RANGE" | grep -E '\[vps\]' || true)" echo "::notice::NAS 验证通过 ✅ 。含 [vps] 则开 [待推VPS] issue(人工/agent 推 vps-deploy.yml)" python3 scripts/ci/vps_pending_check.py diff --git a/.gitea/workflows/vps-deploy.yml b/.gitea/workflows/vps-deploy.yml index 34c42ce..7698536 100644 --- a/.gitea/workflows/vps-deploy.yml +++ b/.gitea/workflows/vps-deploy.yml @@ -51,7 +51,7 @@ jobs: - name: promote 代码到 VPS (只推 VPS, 不动 NAS) run: bash scripts/nas_sync/promote.sh --target vps - name: VPS 冒烟 (LocalUnifiedProvider 真数据) - run: ssh 49.232.102.198 'cd C:\sanguo_vnpy_v2 && C:\Python310\python.exe -X utf8 scripts/data_platform/verify_unified_e2e.py' + run: ssh -o ConnectTimeout=20 49.232.102.198 'cd C:\sanguo_vnpy_v2 && C:\Python310\python.exe -X utf8 scripts/data_platform/verify_unified_e2e.py' - name: 前端构建 + 推 dist 到 VPS (公网前端无管线, 此处补齐) run: | set -e @@ -64,55 +64,52 @@ jobs: - name: VPS 替换 dist + 重启 sanguo-api (等端口释放防 10048 竞态, 再等起服) run: | set -e - ssh 49.232.102.198 'C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\nas_sync\vps_update_dist.py' | tee /tmp/dist_swap.log + ssh -o ConnectTimeout=20 49.232.102.198 'C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\nas_sync\vps_update_dist.py' | tee /tmp/dist_swap.log grep -q DIST_UPDATE_DONE /tmp/dist_swap.log # 停 API → 僵尸端口守卫 → 等端口彻底释放(否则 fail) → 重启 # 2026-08-16 实录: /end 杀不掉 uvicorn reload 孤儿子进程, 旧进程跨两轮 # 部署占 8000, 新实例静默抢不到端口, /docs 200 验的还是旧代码 → 500 旧 bug 重现 - ssh 49.232.102.198 'schtasks /end /tn sanguo-api' || true + ssh -o ConnectTimeout=20 49.232.102.198 'schtasks /end /tn sanguo-api' || true # 注意 || true 兜底: 端口已空时 grep 无匹配, set -e -o pipefail 会把整步炸掉 # (2026-08-16 run674 实录: /end 干净杀掉后 180ms 死在这行, API 停在半路) - ZPID=$(ssh 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000' | grep -i LISTENING | awk '{print $NF}' | sort -u | head -1 || true) + ZPID=$(ssh -o ConnectTimeout=20 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000' | grep -i LISTENING | awk '{print $NF}' | sort -u | head -1 || true) if [ -n "$ZPID" ]; then echo "僵尸 API 进程 PID=$ZPID 仍占 8000, 强杀" - ssh 49.232.102.198 "taskkill /F /T /PID $ZPID" || true + ssh -o ConnectTimeout=20 49.232.102.198 "taskkill /F /T /PID $ZPID" || true fi + # ⚠️ pipefail 雷区(2026-08-22 run724 根因, issue#33 评论1290): 本步 shell 是 -e -o pipefail, + # `大输出 | grep -q` 命中即提前退出 → 上游 tr 吃 SIGPIPE(141) → pipefail 把「匹配成功」判成 + # 整管道失败。本行曾因此被反转: 端口仍在监听却走 || 分支谎报「已释放」→ 下一步 /run 撞 10048。 + # 修法 = `grep 'x' >/dev/null`(读完整流不提前退出); 下面等待循环同雷同修(run724 死因)。 PORT_FREED=0 for i in $(seq 1 30); do - ssh 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep -q ':8000.*LISTEN' || { echo "port freed after $((i*2))s"; PORT_FREED=1; break; } + ssh -o ConnectTimeout=20 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000.*LISTEN' >/dev/null || { echo "port freed after $((i*2))s"; PORT_FREED=1; break; } sleep 2 done if [ "$PORT_FREED" != "1" ]; then echo "❌ 端口 8000 未释放, 强杀后重启会 10048, 终止"; exit 1; fi - ssh 49.232.102.198 'schtasks /run /tn sanguo-api' + ssh -o ConnectTimeout=20 49.232.102.198 'schtasks /run /tn sanguo-api' # 等新实例监听 + /docs 可用(上限 20 分钟:2026-08-16 run681 实录 19 个常驻进程 # 负载下 API 启动超 10 分钟,最终正常起服——10 分钟帽误杀,workflow 放弃时 # API 其实在路上,且跳过了常驻重启/tag/关issue 三步) for i in $(seq 1 120); do - if ssh 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep -q ':8000.*LISTEN'; then + # grep >/dev/null 而非 grep -q: 同上 pipefail+SIGPIPE 雷(run724 死因: 检测恒假→120 轮耗尽 exit 1→收尾三步全跳过) + if ssh -o ConnectTimeout=20 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000.*LISTEN' >/dev/null; then echo "API listening after ~$((i*10))s"; break fi sleep 10 if [ "$i" = "120" ]; then echo "API 20 分钟未监听 8000"; exit 1; fi done for i in $(seq 1 12); do - ssh 49.232.102.198 'powershell -NoProfile -Command "(Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/docs).StatusCode"' 2>/dev/null | grep -q 200 && { echo "API /docs 200"; exit 0; } + ssh -o ConnectTimeout=20 49.232.102.198 'powershell -NoProfile -Command "(Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/docs).StatusCode"' 2>/dev/null | grep 200 >/dev/null && { echo "API /docs 200"; exit 0; } sleep 10 done echo "API 监听但 /docs 2 分钟未返回 200"; exit 1 - - name: 重启 VPS 常驻进程 (supervisor/影子柜台吃新代码; 盘中自动跳过) - run: bash scripts/nas_sync/vps_restart_resident.sh - - name: mark vps-deployed tag (VPS 版本真相, session 用 git log vps-deployed..HEAD 查未部署) - env: - TOKEN: ${{ github.token }} - run: | - set -e - SHA="${{ inputs.sha }}" - git tag -f vps-deployed "$SHA" - git push -f "http://oauth2:${TOKEN}@192.168.2.154:3000/${GITHUB_REPOSITORY}.git" refs/tags/vps-deployed - echo "✅ vps-deployed → ${SHA::7}" - - name: 关闭 [待推VPS] issue (tag=HEAD → gap 清空 = 绿) + # 收尾三步抽成幂等脚本(2026-08-22 基建工作包#2, issue#33): run681/715/724 三次实录 + # 部署成功但收尾被前序步骤失败整段跳过, 只能人工补。现在: 正常走到尾自动执行; + # 被跳过时 Mac 本机手跑即可补: GTOKEN= bash scripts/ci/vps_finalize.sh <部署的sha> + # issue 维护按 vps-deployed..origin/master 真实 gap 开/关(2026-08-21 事故: 部署旧 sha 后 + # 无脑关 issue, gap 非空也关 → 用户误以为推完)。 + - name: 收尾三步 (常驻重启 + vps-deployed tag + [待推VPS] issue, 幂等可重入) env: GTOKEN: ${{ github.token }} - run: | - export VPS_LOG="" - python3 scripts/ci/vps_pending_check.py + run: bash scripts/ci/vps_finalize.sh "${{ inputs.sha }}" diff --git a/scripts/ci/vps_finalize.sh b/scripts/ci/vps_finalize.sh new file mode 100644 index 0000000..bb65581 --- /dev/null +++ b/scripts/ci/vps_finalize.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# vps-deploy 收尾三步: 常驻滚动重启 + 打 vps-deployed tag + [待推VPS] issue 维护。 +# 为什么抽脚本(2026-08-22 基建工作包#2, issue#33): run681/715/724 三次实录——部署本体 +# 成功但前序步骤(端口等待/API探活)失败时, 收尾三步被整段跳过, 只能人工拼三段补。 +# 抽出后幂等可重入: +# - workflow 正常走到尾: vps-deploy.yml 最后一步自动调用 +# - 收尾被跳过时: Mac 本机手跑补, 不再人工拼命令: +# GTOKEN= bash scripts/ci/vps_finalize.sh <本轮部署的sha> +# 三步各自幂等: 常驻重启(盘中 09:15-15:30 自动跳过) / tag -f + push -f / issue 按真实 gap 开或关。 +# ⚠️ gap 按 subject 算(与 enforce-label 同定义, git log --format='%h %s' 再 grep): +# 不能用 --grep='\[vps\]'——它匹配整个 message, commit body 里引用的命令文本会误命中(曾误开 issue#7)。 +set -euo pipefail +VPS_DEPLOYED_TAG=vps-deployed +SHA="${1:?用法: GTOKEN= bash scripts/ci/vps_finalize.sh <本轮部署的sha(40位)>}" +export GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-sanguo/sanguo_vnpy_v2}" + +FAIL=0 + +# ---- 1) 常驻进程滚动重启(幂等; 失败不阻断 tag/issue 记账——代码已就位, 末尾非零退出提醒人工) ---- +if bash scripts/nas_sync/vps_restart_resident.sh; then + echo "✅ 收尾 1/3: 常驻进程已滚动重启(或盘中自动跳过)" +else + echo "❌ 收尾 1/3: 常驻重启失败(部署本体已成功, 代码已就位)——重跑本脚本即可补" + FAIL=1 +fi + +# ---- 2) 打 tag(VPS 版本真相; tag -f + push -f 幂等)。origin 在 workflow checkout 里 +# 已是带 token 的 remote, 手跑时是本仓库日常 remote, 两种上下文都直推 ---- +git tag -f "$VPS_DEPLOYED_TAG" "$SHA" +git push -f origin "refs/tags/$VPS_DEPLOYED_TAG" +echo "✅ 收尾 2/3: $VPS_DEPLOYED_TAG → ${SHA:0:7}" + +# ---- 3) [待推VPS] issue 维护: 按 vps-deployed..origin/master 真实 gap 决定开/关 ---- +# (2026-08-21 事故: 用户 dispatch 填了旧 sha, 收尾硬编码 VPS_LOG="" 无脑关 issue—— +# gap 非空也关, 用户误以为推完。修 = 算真实 gap, 非空则 issue 保持 open 并列出剩余清单) +if [ -z "${GTOKEN:-}" ]; then + echo "⚠️ 收尾 3/3: GTOKEN 未设置, 跳过 issue 维持(workflow 自带; 手跑需 GTOKEN)——issue 保持现状偏保守" +else + git fetch origin master --depth 200 2>/dev/null || git fetch origin master + export VPS_LOG="$(git log --format='%h %s' "$VPS_DEPLOYED_TAG..origin/master" | grep -E '\[vps\]' || true)" + export VPS_DEPLOY_SHA="$SHA" + python3 scripts/ci/vps_pending_check.py + echo "✅ 收尾 3/3: issue 维护完成(按真实 gap 开/关)" +fi + +exit $FAIL diff --git a/scripts/ci/vps_pending_check.py b/scripts/ci/vps_pending_check.py index 99fa186..60e8a74 100644 --- a/scripts/ci/vps_pending_check.py +++ b/scripts/ci/vps_pending_check.py @@ -21,7 +21,7 @@ API_BASE = "http://192.168.2.154:3000/api/v1/repos/" def main(): token = os.environ["GTOKEN"] repo = os.environ["GITHUB_REPOSITORY"] - sha = os.environ["GITHUB_SHA"] + sha = os.environ.get("GITHUB_SHA") or os.environ.get("VPS_DEPLOY_SHA") or "HEAD" vps_log = os.environ.get("VPS_LOG", "").strip() api = API_BASE + repo @@ -53,10 +53,15 @@ def main(): n = len([l for l in vps_log.splitlines() if l.strip()]) print(f"⚠️ 待推 VPS: {n} 个 [vps] commit") print(vps_log) + prev = os.environ.get("VPS_DEPLOY_SHA", "") body = ( f"nas-verify 绿 @{sha[:7]}, 以下 [vps] commit 待推 VPS:\n" f"```\n{vps_log}\n```\n\n" - f"推法: Gitea → Actions → vps-deploy.yml → Run workflow, sha={sha}\n" + + ( + f"⚠️ 上一轮 vps-deploy 只部署到 @{prev[:7]}, 上述 commit 仍未上 VPS —— issue 保持 open(不再无脑关)。\n\n" + if prev else "" + ) + + f"推法: Gitea → Actions → vps-deploy.yml → Run workflow, sha={sha}\n" f"agent: dispatch vps-deploy.yml inputs={{sha:{sha}}}" ) if pending: