From a58d9fd38566aed52d4144e3601e0a15f83aaf19 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Sat, 22 Aug 2026 22:33:24 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20vps-deploy/workflow=E5=9B=9B?= =?UTF-8?q?=E9=9B=B7=E6=A0=B9=E6=B2=BB(pipefail=C3=973+ssh=E8=A3=B8?= =?UTF-8?q?=E8=BF=9E+=E6=94=B6=E5=B0=BE=E9=9D=9E=E5=B9=82=E7=AD=89+?= =?UTF-8?q?=E5=85=B3issue=E4=B8=8D=E6=9F=A5gap)=E2=80=94=E2=80=94issue#33?= =?UTF-8?q?=E5=9F=BA=E5=BB=BA=E5=B7=A5=E4=BD=9C=E5=8C=85#1-#4=20[nas]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipefail+SIGPIPE 三处(本机复现rc=141确定性,run724根因): - L82端口释放检测被反转(最危险): netstat大输出|grep -q命中即提前退出→上游tr吃SIGPIPE(141)→pipefail判整管道失败→走||分支谎报「已释放」→下一步/run撞10048竞态 - L91 API等待循环同雷=run724死因: 检测恒假→120轮耗尽exit1→收尾三步全跳过 - L98 /docs探活同族(输出仅3字节几乎不触发,顺手修) 修法: grep -q 'x' → grep 'x' >/dev/null(读完整流不提前退出,无SIGPIPE) workflow内裸ssh全量补 -o ConnectTimeout=20(对齐vps_restart_resident.sh写法): 单次ssh挂死原会让整步无限冻结; vps-deploy.yml 9处+ci-cd.yml 2处(NAS) 收尾三步抽幂等脚本 scripts/ci/vps_finalize.sh(工作包#2): - run681/715/724三实录: 部署本体成功但收尾被前序步骤失败整段跳过,只能人工拼三段补 - 现在workflow尾步自动调; 被跳过时Mac本机手跑即补: GTOKEN= bash scripts/ci/vps_finalize.sh <部署的sha> - 常驻重启失败不阻断tag/issue记账(代码已就位,末尾非零退出提醒); tag -f+push -f天然幂等; issue按真实gap开/关 关issue按真实gap(工作包#3,2026-08-21事故): - 旧代码硬编码VPS_LOG=""无脑关issue——用户dispatch填旧sha(f28f9cf≠be315a0)漏推P0后,gap非空也关→用户误以为推完 - 修=finalize算vps-deployed..origin/master真实gap,非空则issue保持open并列剩余清单 - gap改按subject算(--format='%h %s'|grep,与enforce-label同定义): --grep匹配整个message,commit body里引用的命令文本会误命中(曾误开issue#7); ci-cd.yml check-vps-needed同步改 - vps_pending_check.py: GITHUB_SHA缺失时回退VPS_DEPLOY_SHA(支持Mac手跑); 有上轮部署sha时body加「仍未上VPS」警示 --- .gitea/workflows/ci-cd.yml | 8 +++--- .gitea/workflows/vps-deploy.yml | 45 +++++++++++++++----------------- scripts/ci/vps_finalize.sh | 46 +++++++++++++++++++++++++++++++++ scripts/ci/vps_pending_check.py | 9 +++++-- 4 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 scripts/ci/vps_finalize.sh diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 2e1e4ce..9a2f5f7 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -84,7 +84,7 @@ jobs: rsync -az --delete frontend/dist/ sanguo-nas:/volume1/homes/admin/.sanguo_projects/sanguo_vnpy_v2/frontend/dist/ - name: restart NAS 容器(加载新后端代码) + 等 web 就绪 run: | - ssh sanguo-nas '/var/packages/Docker/target/usr/bin/docker restart sanguo_vnpy_v2' + ssh -o ConnectTimeout=20 sanguo-nas '/var/packages/Docker/target/usr/bin/docker restart sanguo_vnpy_v2' for i in $(seq 1 30); do curl -sf http://192.168.2.154:8000/ -o /dev/null && { echo "web ready"; exit 0; } sleep 2 @@ -126,7 +126,7 @@ jobs: echo "✅ /health 200, 存活探针通" - name: NAS 副本数据可读(dbbardata) run: | - ssh sanguo-nas "python3 -c 'import sqlite3 + ssh -o ConnectTimeout=20 sanguo-nas "python3 -c 'import sqlite3 c=sqlite3.connect(\"/volume1/stock/sanguo_vnpy_v2/data_backup/quant_trading.db\") n=c.execute(\"SELECT COUNT(*) FROM dbbardata WHERE symbol=?\",(\"600519\",)).fetchone()[0] print(\"dbbardata 600519 rows:\", n) @@ -144,7 +144,9 @@ jobs: RANGE="HEAD~50..HEAD" echo "⚠️ vps-deployed tag 不存在(首次/未推过 VPS), 查最近 50 commit" fi - export VPS_LOG="$(git log --format='%h %s' "$RANGE" --grep='\[vps\]' || true)" + # gap 按 subject 算(与 enforce-label 同定义): 不能用 --grep='\[vps\]'——它匹配整个 + # message, commit body 里引用的命令文本会误命中(曾误开 issue#7)。与 vps_finalize.sh 同款。 + export VPS_LOG="$(git log --format='%h %s' "$RANGE" | grep -E '\[vps\]' || true)" echo "::notice::NAS 验证通过 ✅ 。含 [vps] 则开 [待推VPS] issue(人工/agent 推 vps-deploy.yml)" python3 scripts/ci/vps_pending_check.py diff --git a/.gitea/workflows/vps-deploy.yml b/.gitea/workflows/vps-deploy.yml index 34c42ce..7698536 100644 --- a/.gitea/workflows/vps-deploy.yml +++ b/.gitea/workflows/vps-deploy.yml @@ -51,7 +51,7 @@ jobs: - name: promote 代码到 VPS (只推 VPS, 不动 NAS) run: bash scripts/nas_sync/promote.sh --target vps - name: VPS 冒烟 (LocalUnifiedProvider 真数据) - run: ssh 49.232.102.198 'cd C:\sanguo_vnpy_v2 && C:\Python310\python.exe -X utf8 scripts/data_platform/verify_unified_e2e.py' + run: ssh -o ConnectTimeout=20 49.232.102.198 'cd C:\sanguo_vnpy_v2 && C:\Python310\python.exe -X utf8 scripts/data_platform/verify_unified_e2e.py' - name: 前端构建 + 推 dist 到 VPS (公网前端无管线, 此处补齐) run: | set -e @@ -64,55 +64,52 @@ jobs: - name: VPS 替换 dist + 重启 sanguo-api (等端口释放防 10048 竞态, 再等起服) run: | set -e - ssh 49.232.102.198 'C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\nas_sync\vps_update_dist.py' | tee /tmp/dist_swap.log + ssh -o ConnectTimeout=20 49.232.102.198 'C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\nas_sync\vps_update_dist.py' | tee /tmp/dist_swap.log grep -q DIST_UPDATE_DONE /tmp/dist_swap.log # 停 API → 僵尸端口守卫 → 等端口彻底释放(否则 fail) → 重启 # 2026-08-16 实录: /end 杀不掉 uvicorn reload 孤儿子进程, 旧进程跨两轮 # 部署占 8000, 新实例静默抢不到端口, /docs 200 验的还是旧代码 → 500 旧 bug 重现 - ssh 49.232.102.198 'schtasks /end /tn sanguo-api' || true + ssh -o ConnectTimeout=20 49.232.102.198 'schtasks /end /tn sanguo-api' || true # 注意 || true 兜底: 端口已空时 grep 无匹配, set -e -o pipefail 会把整步炸掉 # (2026-08-16 run674 实录: /end 干净杀掉后 180ms 死在这行, API 停在半路) - ZPID=$(ssh 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000' | grep -i LISTENING | awk '{print $NF}' | sort -u | head -1 || true) + ZPID=$(ssh -o ConnectTimeout=20 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000' | grep -i LISTENING | awk '{print $NF}' | sort -u | head -1 || true) if [ -n "$ZPID" ]; then echo "僵尸 API 进程 PID=$ZPID 仍占 8000, 强杀" - ssh 49.232.102.198 "taskkill /F /T /PID $ZPID" || true + ssh -o ConnectTimeout=20 49.232.102.198 "taskkill /F /T /PID $ZPID" || true fi + # ⚠️ pipefail 雷区(2026-08-22 run724 根因, issue#33 评论1290): 本步 shell 是 -e -o pipefail, + # `大输出 | grep -q` 命中即提前退出 → 上游 tr 吃 SIGPIPE(141) → pipefail 把「匹配成功」判成 + # 整管道失败。本行曾因此被反转: 端口仍在监听却走 || 分支谎报「已释放」→ 下一步 /run 撞 10048。 + # 修法 = `grep 'x' >/dev/null`(读完整流不提前退出); 下面等待循环同雷同修(run724 死因)。 PORT_FREED=0 for i in $(seq 1 30); do - ssh 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep -q ':8000.*LISTEN' || { echo "port freed after $((i*2))s"; PORT_FREED=1; break; } + ssh -o ConnectTimeout=20 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000.*LISTEN' >/dev/null || { echo "port freed after $((i*2))s"; PORT_FREED=1; break; } sleep 2 done if [ "$PORT_FREED" != "1" ]; then echo "❌ 端口 8000 未释放, 强杀后重启会 10048, 终止"; exit 1; fi - ssh 49.232.102.198 'schtasks /run /tn sanguo-api' + ssh -o ConnectTimeout=20 49.232.102.198 'schtasks /run /tn sanguo-api' # 等新实例监听 + /docs 可用(上限 20 分钟:2026-08-16 run681 实录 19 个常驻进程 # 负载下 API 启动超 10 分钟,最终正常起服——10 分钟帽误杀,workflow 放弃时 # API 其实在路上,且跳过了常驻重启/tag/关issue 三步) for i in $(seq 1 120); do - if ssh 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep -q ':8000.*LISTEN'; then + # grep >/dev/null 而非 grep -q: 同上 pipefail+SIGPIPE 雷(run724 死因: 检测恒假→120 轮耗尽 exit 1→收尾三步全跳过) + if ssh -o ConnectTimeout=20 49.232.102.198 'netstat -ano -p tcp' | tr -d '\r' | grep ':8000.*LISTEN' >/dev/null; then echo "API listening after ~$((i*10))s"; break fi sleep 10 if [ "$i" = "120" ]; then echo "API 20 分钟未监听 8000"; exit 1; fi done for i in $(seq 1 12); do - ssh 49.232.102.198 'powershell -NoProfile -Command "(Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/docs).StatusCode"' 2>/dev/null | grep -q 200 && { echo "API /docs 200"; exit 0; } + ssh -o ConnectTimeout=20 49.232.102.198 'powershell -NoProfile -Command "(Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/docs).StatusCode"' 2>/dev/null | grep 200 >/dev/null && { echo "API /docs 200"; exit 0; } sleep 10 done echo "API 监听但 /docs 2 分钟未返回 200"; exit 1 - - name: 重启 VPS 常驻进程 (supervisor/影子柜台吃新代码; 盘中自动跳过) - run: bash scripts/nas_sync/vps_restart_resident.sh - - name: mark vps-deployed tag (VPS 版本真相, session 用 git log vps-deployed..HEAD 查未部署) - env: - TOKEN: ${{ github.token }} - run: | - set -e - SHA="${{ inputs.sha }}" - git tag -f vps-deployed "$SHA" - git push -f "http://oauth2:${TOKEN}@192.168.2.154:3000/${GITHUB_REPOSITORY}.git" refs/tags/vps-deployed - echo "✅ vps-deployed → ${SHA::7}" - - name: 关闭 [待推VPS] issue (tag=HEAD → gap 清空 = 绿) + # 收尾三步抽成幂等脚本(2026-08-22 基建工作包#2, issue#33): run681/715/724 三次实录 + # 部署成功但收尾被前序步骤失败整段跳过, 只能人工补。现在: 正常走到尾自动执行; + # 被跳过时 Mac 本机手跑即可补: GTOKEN= bash scripts/ci/vps_finalize.sh <部署的sha> + # issue 维护按 vps-deployed..origin/master 真实 gap 开/关(2026-08-21 事故: 部署旧 sha 后 + # 无脑关 issue, gap 非空也关 → 用户误以为推完)。 + - name: 收尾三步 (常驻重启 + vps-deployed tag + [待推VPS] issue, 幂等可重入) env: GTOKEN: ${{ github.token }} - run: | - export VPS_LOG="" - python3 scripts/ci/vps_pending_check.py + run: bash scripts/ci/vps_finalize.sh "${{ inputs.sha }}" diff --git a/scripts/ci/vps_finalize.sh b/scripts/ci/vps_finalize.sh new file mode 100644 index 0000000..bb65581 --- /dev/null +++ b/scripts/ci/vps_finalize.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# vps-deploy 收尾三步: 常驻滚动重启 + 打 vps-deployed tag + [待推VPS] issue 维护。 +# 为什么抽脚本(2026-08-22 基建工作包#2, issue#33): run681/715/724 三次实录——部署本体 +# 成功但前序步骤(端口等待/API探活)失败时, 收尾三步被整段跳过, 只能人工拼三段补。 +# 抽出后幂等可重入: +# - workflow 正常走到尾: vps-deploy.yml 最后一步自动调用 +# - 收尾被跳过时: Mac 本机手跑补, 不再人工拼命令: +# GTOKEN= bash scripts/ci/vps_finalize.sh <本轮部署的sha> +# 三步各自幂等: 常驻重启(盘中 09:15-15:30 自动跳过) / tag -f + push -f / issue 按真实 gap 开或关。 +# ⚠️ gap 按 subject 算(与 enforce-label 同定义, git log --format='%h %s' 再 grep): +# 不能用 --grep='\[vps\]'——它匹配整个 message, commit body 里引用的命令文本会误命中(曾误开 issue#7)。 +set -euo pipefail +VPS_DEPLOYED_TAG=vps-deployed +SHA="${1:?用法: GTOKEN= bash scripts/ci/vps_finalize.sh <本轮部署的sha(40位)>}" +export GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-sanguo/sanguo_vnpy_v2}" + +FAIL=0 + +# ---- 1) 常驻进程滚动重启(幂等; 失败不阻断 tag/issue 记账——代码已就位, 末尾非零退出提醒人工) ---- +if bash scripts/nas_sync/vps_restart_resident.sh; then + echo "✅ 收尾 1/3: 常驻进程已滚动重启(或盘中自动跳过)" +else + echo "❌ 收尾 1/3: 常驻重启失败(部署本体已成功, 代码已就位)——重跑本脚本即可补" + FAIL=1 +fi + +# ---- 2) 打 tag(VPS 版本真相; tag -f + push -f 幂等)。origin 在 workflow checkout 里 +# 已是带 token 的 remote, 手跑时是本仓库日常 remote, 两种上下文都直推 ---- +git tag -f "$VPS_DEPLOYED_TAG" "$SHA" +git push -f origin "refs/tags/$VPS_DEPLOYED_TAG" +echo "✅ 收尾 2/3: $VPS_DEPLOYED_TAG → ${SHA:0:7}" + +# ---- 3) [待推VPS] issue 维护: 按 vps-deployed..origin/master 真实 gap 决定开/关 ---- +# (2026-08-21 事故: 用户 dispatch 填了旧 sha, 收尾硬编码 VPS_LOG="" 无脑关 issue—— +# gap 非空也关, 用户误以为推完。修 = 算真实 gap, 非空则 issue 保持 open 并列出剩余清单) +if [ -z "${GTOKEN:-}" ]; then + echo "⚠️ 收尾 3/3: GTOKEN 未设置, 跳过 issue 维持(workflow 自带; 手跑需 GTOKEN)——issue 保持现状偏保守" +else + git fetch origin master --depth 200 2>/dev/null || git fetch origin master + export VPS_LOG="$(git log --format='%h %s' "$VPS_DEPLOYED_TAG..origin/master" | grep -E '\[vps\]' || true)" + export VPS_DEPLOY_SHA="$SHA" + python3 scripts/ci/vps_pending_check.py + echo "✅ 收尾 3/3: issue 维护完成(按真实 gap 开/关)" +fi + +exit $FAIL diff --git a/scripts/ci/vps_pending_check.py b/scripts/ci/vps_pending_check.py index 99fa186..60e8a74 100644 --- a/scripts/ci/vps_pending_check.py +++ b/scripts/ci/vps_pending_check.py @@ -21,7 +21,7 @@ API_BASE = "http://192.168.2.154:3000/api/v1/repos/" def main(): token = os.environ["GTOKEN"] repo = os.environ["GITHUB_REPOSITORY"] - sha = os.environ["GITHUB_SHA"] + sha = os.environ.get("GITHUB_SHA") or os.environ.get("VPS_DEPLOY_SHA") or "HEAD" vps_log = os.environ.get("VPS_LOG", "").strip() api = API_BASE + repo @@ -53,10 +53,15 @@ def main(): n = len([l for l in vps_log.splitlines() if l.strip()]) print(f"⚠️ 待推 VPS: {n} 个 [vps] commit") print(vps_log) + prev = os.environ.get("VPS_DEPLOY_SHA", "") body = ( f"nas-verify 绿 @{sha[:7]}, 以下 [vps] commit 待推 VPS:\n" f"```\n{vps_log}\n```\n\n" - f"推法: Gitea → Actions → vps-deploy.yml → Run workflow, sha={sha}\n" + + ( + f"⚠️ 上一轮 vps-deploy 只部署到 @{prev[:7]}, 上述 commit 仍未上 VPS —— issue 保持 open(不再无脑关)。\n\n" + if prev else "" + ) + + f"推法: Gitea → Actions → vps-deploy.yml → Run workflow, sha={sha}\n" f"agent: dispatch vps-deploy.yml inputs={{sha:{sha}}}" ) if pending: