diff --git a/scripts/nas_sync/sync_parquet_resumable.sh b/scripts/nas_sync/sync_parquet_resumable.sh new file mode 100644 index 0000000..df36d57 --- /dev/null +++ b/scripts/nas_sync/sync_parquet_resumable.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# NAS 端:VPS parquet(static+valuation) 文件级断点续传。 +# VPS 无 rsync,改逐文件 scp + NAS 端 size 校验跳过已传。中断重跑只续未传文件。 +# 脱离 ssh 会话: nohup setsid bash sync_parquet_resumable.sh /dev/null 2>&1 & +set -u +KEY=/var/services/homes/admin/.ssh/id_ed25519_nas +VPS=Administrator@49.232.102.198 +NAS_DATA=/volume1/stock/sanguo_vnpy_v2/data +LOG=/volume1/stock/sanguo_vnpy_v2/data_backup/parquet.log +MANIFEST=/volume1/stock/sanguo_vnpy_v2/data_backup/parquet_manifest.txt + +echo "=== resumable start $(date) ===" >> "$LOG" + +# 1. VPS 端 Python 列 parquet 清单(path size, 正斜杠), stdin 喂执行不落盘 +ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" 'C:\Python310\python.exe -X utf8 -' > "$MANIFEST" <<'PYEOF' +import os +for sub in ("static", "valuation_baostock"): + root = os.path.join(r"C:\sanguo_vnpy_v2\data", sub) + for dp, _, fs in os.walk(root): + for f in fs: + p = os.path.join(dp, f).replace("\\", "/") + try: + print(p, os.path.getsize(p)) + except OSError: + pass +PYEOF + +n=$(wc -l < "$MANIFEST") +echo "[manifest] $n files" >> "$LOG" + +# 2. 逐文件 scp, NAS size 匹配则跳过(断点续传核心) +done=0; skipped=0; failed=0 +while read -r vps_path size rest; do + [ -z "$vps_path" ] && continue + rel=${vps_path#C:/sanguo_vnpy_v2/data/} + nas_path="$NAS_DATA/$rel" + mkdir -p "$(dirname "$nas_path")" + if [ -f "$nas_path" ] && [ "$(stat -c%s "$nas_path" 2>/dev/null)" = "$size" ]; then + skipped=$((skipped+1)) + elif scp -q -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$vps_path" "$nas_path" 2>>"$LOG"; then + done=$((done+1)) + if [ $((done % 50)) -eq 0 ]; then echo "[progress] done=$done skipped=$skipped failed=$failed" >> "$LOG"; fi + else + failed=$((failed+1)) + echo "[FAIL] $rel" >> "$LOG" + fi +done < "$MANIFEST" + +echo "[result] done=$done skipped=$skipped failed=$failed" >> "$LOG" +du -sh "$NAS_DATA/static" "$NAS_DATA/valuation_baostock" >> "$LOG" 2>&1 +echo "=== resumable done $(date) ===" >> "$LOG" +echo "PARQUET_DONE $(date)" >> "$LOG"