53 lines
2.1 KiB
Bash
53 lines
2.1 KiB
Bash
#!/bin/bash
|
|
# NAS 端：VPS parquet(static+valuation) 文件级断点续传。
|
|
# VPS 无 rsync，改逐文件 scp + NAS 端 size 校验跳过已传。中断重跑只续未传文件。
|
|
# 脱离 ssh 会话: nohup setsid bash sync_parquet_resumable.sh </dev/null >/dev/null 2>&1 &
|
|
set -u
|
|
KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
|
|
VPS=Administrator@49.232.102.198
|
|
NAS_DATA=/volume1/stock/sanguo_vnpy_v2/data
|
|
LOG=/volume1/stock/sanguo_vnpy_v2/data_backup/parquet.log
|
|
MANIFEST=/volume1/stock/sanguo_vnpy_v2/data_backup/parquet_manifest.txt
|
|
|
|
echo "=== resumable start $(date) ===" >> "$LOG"
|
|
|
|
# 1. VPS 端 Python 列 parquet 清单(path size, 正斜杠), stdin 喂执行不落盘
|
|
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" 'C:\Python310\python.exe -X utf8 -' > "$MANIFEST" <<'PYEOF'
|
|
import os
|
|
for sub in ("static", "valuation_baostock"):
|
|
root = os.path.join(r"C:\sanguo_vnpy_v2\data", sub)
|
|
for dp, _, fs in os.walk(root):
|
|
for f in fs:
|
|
p = os.path.join(dp, f).replace("\\", "/")
|
|
try:
|
|
print(p, os.path.getsize(p))
|
|
except OSError:
|
|
pass
|
|
PYEOF
|
|
|
|
n=$(wc -l < "$MANIFEST")
|
|
echo "[manifest] $n files" >> "$LOG"
|
|
|
|
# 2. 逐文件 scp, NAS size 匹配则跳过(断点续传核心)
|
|
done=0; skipped=0; failed=0
|
|
while read -r vps_path size rest; do
|
|
[ -z "$vps_path" ] && continue
|
|
rel=${vps_path#C:/sanguo_vnpy_v2/data/}
|
|
nas_path="$NAS_DATA/$rel"
|
|
mkdir -p "$(dirname "$nas_path")"
|
|
if [ -f "$nas_path" ] && [ "$(stat -c%s "$nas_path" 2>/dev/null)" = "$size" ]; then
|
|
skipped=$((skipped+1))
|
|
elif scp -q -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$vps_path" "$nas_path" 2>>"$LOG"; then
|
|
done=$((done+1))
|
|
if [ $((done % 50)) -eq 0 ]; then echo "[progress] done=$done skipped=$skipped failed=$failed" >> "$LOG"; fi
|
|
else
|
|
failed=$((failed+1))
|
|
echo "[FAIL] $rel" >> "$LOG"
|
|
fi
|
|
done < "$MANIFEST"
|
|
|
|
echo "[result] done=$done skipped=$skipped failed=$failed" >> "$LOG"
|
|
du -sh "$NAS_DATA/static" "$NAS_DATA/valuation_baostock" >> "$LOG" 2>&1
|
|
echo "=== resumable done $(date) ===" >> "$LOG"
|
|
echo "PARQUET_DONE $(date)" >> "$LOG"
|