feat(nas_sync): parquet 断点续传脚本(逐文件scp+NAS size校验跳过,治VPS无rsync) [nas]

2026-08-12 08:45:24 +08:00
parent bf30283ac5
commit 4f6cd43b21
1 changed files with 52 additions and 0 deletions
@@ -0,0 +1,52 @@
+#!/bin/bash
+# NAS 端：VPS parquet(static+valuation) 文件级断点续传。
+# VPS 无 rsync，改逐文件 scp + NAS 端 size 校验跳过已传。中断重跑只续未传文件。
+# 脱离 ssh 会话: nohup setsid bash sync_parquet_resumable.sh </dev/null >/dev/null 2>&1 &
+set -u
+KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
+VPS=Administrator@49.232.102.198
+NAS_DATA=/volume1/stock/sanguo_vnpy_v2/data
+LOG=/volume1/stock/sanguo_vnpy_v2/data_backup/parquet.log
+MANIFEST=/volume1/stock/sanguo_vnpy_v2/data_backup/parquet_manifest.txt
+
+echo "=== resumable start $(date) ===" >> "$LOG"
+
+# 1. VPS 端 Python 列 parquet 清单(path size, 正斜杠), stdin 喂执行不落盘
+ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" 'C:\Python310\python.exe -X utf8 -' > "$MANIFEST" <<'PYEOF'
+import os
+for sub in ("static", "valuation_baostock"):
+    root = os.path.join(r"C:\sanguo_vnpy_v2\data", sub)
+    for dp, _, fs in os.walk(root):
+        for f in fs:
+            p = os.path.join(dp, f).replace("\\", "/")
+            try:
+                print(p, os.path.getsize(p))
+            except OSError:
+                pass
+PYEOF
+
+n=$(wc -l < "$MANIFEST")
+echo "[manifest] $n files" >> "$LOG"
+
+# 2. 逐文件 scp, NAS size 匹配则跳过(断点续传核心)
+done=0; skipped=0; failed=0
+while read -r vps_path size rest; do
+  [ -z "$vps_path" ] && continue
+  rel=${vps_path#C:/sanguo_vnpy_v2/data/}
+  nas_path="$NAS_DATA/$rel"
+  mkdir -p "$(dirname "$nas_path")"
+  if [ -f "$nas_path" ] && [ "$(stat -c%s "$nas_path" 2>/dev/null)" = "$size" ]; then
+    skipped=$((skipped+1))
+  elif scp -q -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$vps_path" "$nas_path" 2>>"$LOG"; then
+    done=$((done+1))
+    if [ $((done % 50)) -eq 0 ]; then echo "[progress] done=$done skipped=$skipped failed=$failed" >> "$LOG"; fi
+  else
+    failed=$((failed+1))
+    echo "[FAIL] $rel" >> "$LOG"
+  fi
+done < "$MANIFEST"
+
+echo "[result] done=$done skipped=$skipped failed=$failed" >> "$LOG"
+du -sh "$NAS_DATA/static" "$NAS_DATA/valuation_baostock" >> "$LOG" 2>&1
+echo "=== resumable done $(date) ===" >> "$LOG"
+echo "PARQUET_DONE $(date)" >> "$LOG"