diff --git a/scripts/nas_sync/sync_parquet_daily.sh b/scripts/nas_sync/sync_parquet_daily.sh new file mode 100644 index 0000000..114764f --- /dev/null +++ b/scripts/nas_sync/sync_parquet_daily.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# NAS 端:VPS parquet 每日增量同步(仅 per-date 类,按日期精确拉,不扫 mtime)。 +# +# 设计背景(2026-07-31 实测): +# - VPS 机械盘扫 parquet 小文件 mtime 极慢(valuation+financial_abstract 11070 文件 >60s), +# bsdtar --newer 增量不可行(印证 phase3 当年弃 rsync 改 SQLite 增量的同一坑)。 +# - per-date 类文件名含日期(YYYYMMDD_type.parquet),可按日期精确拉(scp 单文件,不扫目录)。 +# - per-stock 类(valuation/三表/northbound 等)文件名无日期,走 sync_parquet.sh 周度全量。 +# +# 覆盖:ak-events 每日产出(dragon_tiger/block_trade/margin_sse/restricted)。 +# 每日拉最近 7 天窗口(容错漏跑几天;节假日无文件=scp 失败=skip 正常)。 +set -u + +KEY=/var/services/homes/admin/.ssh/id_ed25519_nas +VPS=Administrator@49.232.102.198 +ROOT=/volume1/stock/sanguo_vnpy_v2 +DATA=$ROOT/data +LOG=$ROOT/data_backup/parquet_daily.log + +TYPES="dragon_tiger block_trade margin_sse restricted" +WINDOW_DAYS=7 + +TS=$(date '+%F %T') +echo "=== $TS parquet daily (per-date, ${WINDOW_DAYS}day window) ===" >> "$LOG" + +PULLED=0 +SKIPPED=0 +for type in $TYPES; do + mkdir -p "$DATA/static/$type" + for i in $(seq 0 $((WINDOW_DAYS - 1))); do + d=$(date -d "$i days ago" +%Y%m%d) + file="${d}_${type}.parquet" + if scp -q -i "$KEY" -o StrictHostKeyChecking=no \ + "$VPS:C:/sanguo_vnpy_v2/data/static/$type/$file" \ + "$DATA/static/$type/" 2>/dev/null; then + PULLED=$((PULLED + 1)) + else + SKIPPED=$((SKIPPED + 1)) + fi + done +done + +echo "[$(date '+%T')] DAILY DONE pulled=$PULLED skipped=$SKIPPED (skip=节假日/未出,正常)" >> "$LOG"