From 48767272892f879858662d7cf724944b792d8d04 Mon Sep 17 00:00:00 2001
From: claude_dev <claude-dev@sanguo.local>
Date: Sat, 1 Aug 2026 02:04:19 +0800
Subject: [PATCH] =?UTF-8?q?fix(nas=5Fsync):=20sync=5Fvaluation=5Fdaily=20?=
 =?UTF-8?q?=E6=94=B9scp-r(=E5=8D=95tar=E8=B7=A8=E5=85=AC=E7=BD=91broken=20?=
 =?UTF-8?q?pipe)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

单tar 1.1GB scp跨公网broken pipe(连接断retry难救);改scp-r多文件独立传
(sync_parquet.sh验证6.24G鲁棒)。实测valuation+financial_abstract DONE,
8-1同步5535x2文件。踩坑:nohup &在tee命令里干扰stdin致部署空,分步tee。
---
 scripts/nas_sync/sync_valuation_daily.sh | 57 ++++++++++--------------
 1 file changed, 23 insertions(+), 34 deletions(-)

diff --git a/scripts/nas_sync/sync_valuation_daily.sh b/scripts/nas_sync/sync_valuation_daily.sh
index 66a1033..b61acef 100644
--- a/scripts/nas_sync/sync_valuation_daily.sh
+++ b/scripts/nas_sync/sync_valuation_daily.sh
@@ -1,48 +1,37 @@
 #!/bin/bash
-# NAS 端：每日同步 VPS valuation + financial_abstract（ak-eod 每日 --force 全量重写这俩）。
+# NAS 端：每日同步 VPS valuation + financial_abstract（ak-eod 每日 --force 全量重写）。
 #
-# 设计（2026-07-31 实测）：
-# - ak-eod 19:00 用 --force 每日全量重写 valuation/financial_abstract（估值随股价每日变）
-# - bsdtar --newer 按 mtime 增量不可行（VPS 慢盘 stat 11070 文件 >60s）
-# - python tarfile 全打包只 read 内容不 stat，实测 9.5s/1.1GB → scp 单 tar → 解包
+# 设计迭代（2026-07-31 实测）：
+# - ak-eod 每日 --force 重写 valuation/financial_abstract（估值随股价每日变）
+# - bsdtar --newer 慢盘 stat 不可行；python tarfile 打包 9.5s/1.1GB OK
+# - 但 scp 单 tar 1.1GB 跨公网 broken pipe（连接断全丢，retry 难救）
+# - 改 scp -r 目录：多文件独立传，单文件失败不牵连（sync_parquet.sh 验证 6.24G 鲁棒）
 #
-# 调度：ak-eod 19:00 跑完（~21:30）后延迟跑（22:00 或挂 id=2 次日 3:00）。
-# per-date 类（dragon_tiger 等）走 sync_parquet_daily.sh；weekly per-stock（northbound 等）走 sync_parquet.sh 周日全量。
+# 调度：ak-eod 19:00 跑完（~21:30）后延迟跑。
 set -u
 
 KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
 VPS=Administrator@49.232.102.198
 ROOT=/volume1/stock/sanguo_vnpy_v2
 DATA=$ROOT/data
-STAGE=$ROOT/data_backup/_staging
 LOG=$ROOT/data_backup/valuation_daily.log
-
-VPS_TAR='C:\sanguo_vnpy_v2\data\_sync_val.tar'
-VPS_TAR_SCP='C:/sanguo_vnpy_v2/data/_sync_val.tar'
+DIRS="valuation financial_abstract"
 
 TS=$(date '+%F %T')
-mkdir -p "$STAGE"
-echo "=== $TS valuation+financial_abstract daily ===" >> "$LOG"
+echo "=== $TS valuation daily (scp -r) ===" >> "$LOG"
 
-# VPS: python tarfile 打包（绕 bsdtar 命令解析/慢盘 stat 坑）
-ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" "C:\Python310\python.exe -X utf8 -" <<'PY' >> "$LOG" 2>&1
-import tarfile, os
-base = r'C:\sanguo_vnpy_v2\data'
-out = base + r'\_sync_val.tar'
-with tarfile.open(out, 'w') as t:
-    for d in ['valuation', 'financial_abstract']:
-        t.add(os.path.join(base, 'static', d), arcname='static/' + d)
-print(f'PACK size={os.path.getsize(out)/1024/1024:.1f}MB')
-PY
+for d in $DIRS; do
+  OK=0
+  for attempt in 1 2 3; do
+    if scp -r -i "$KEY" -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=20 \
+      "$VPS:C:/sanguo_vnpy_v2/data/static/$d" "$DATA/static/" >> "$LOG" 2>&1; then
+      OK=1
+      break
+    fi
+    echo "[$(date '+%T')] $d scp attempt $attempt fail, retry..." >> "$LOG"
+    sleep 5
+  done
+  echo "[$(date '+%T')] $d $( [ "$OK" = 1 ] && echo OK || echo FAIL )" >> "$LOG"
+done
 
-# scp tar 到 NAS + 解包
-if scp -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$VPS_TAR_SCP" "$STAGE/_sync_val.tar" >> "$LOG" 2>&1; then
-  tar -xf "$STAGE/_sync_val.tar" -C "$DATA" >> "$LOG" 2>&1
-  echo "[$(date '+%T')] valuation daily DONE" >> "$LOG"
-else
-  echo "[$(date '+%T')] valuation daily FAIL (scp)" >> "$LOG"
-fi
-
-# 清理（NAS 本地 tar + VPS 临时 tar）
-rm -f "$STAGE/_sync_val.tar"
-ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" "del \"$VPS_TAR\"" >> "$LOG" 2>&1
+echo "[$(date '+%T')] valuation daily DONE" >> "$LOG"