From 7f550d2eb241d79e8f23d0fe7112eb92f2ad53d7 Mon Sep 17 00:00:00 2001
From: claude_dev <claude-dev@sanguo.local>
Date: Fri, 31 Jul 2026 22:02:29 +0800
Subject: [PATCH] =?UTF-8?q?feat(nas=5Fsync):=20parquet=20per-date=20?=
 =?UTF-8?q?=E6=AF=8F=E6=97=A5=E5=A2=9E=E9=87=8F=E5=90=8C=E6=AD=A5=E8=84=9A?=
 =?UTF-8?q?=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

VPS机械盘扫parquet小文件mtime慢(11070文件>60s),bsdtar--newer不可行;
per-date类(ak-events:dragon_tiger/block_trade/margin_sse/restricted)按文件名
日期精确拉(7天窗口容错),绕开慢盘。实测pulled=20/skipped=8。
per-stock类仍走sync_parquet.sh周度全量。
---
 scripts/nas_sync/sync_parquet_daily.sh | 43 ++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 scripts/nas_sync/sync_parquet_daily.sh

diff --git a/scripts/nas_sync/sync_parquet_daily.sh b/scripts/nas_sync/sync_parquet_daily.sh
new file mode 100644
index 0000000..114764f
--- /dev/null
+++ b/scripts/nas_sync/sync_parquet_daily.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+# NAS 端：VPS parquet 每日增量同步（仅 per-date 类，按日期精确拉，不扫 mtime）。
+#
+# 设计背景（2026-07-31 实测）：
+# - VPS 机械盘扫 parquet 小文件 mtime 极慢（valuation+financial_abstract 11070 文件 >60s），
+#   bsdtar --newer 增量不可行（印证 phase3 当年弃 rsync 改 SQLite 增量的同一坑）。
+# - per-date 类文件名含日期（YYYYMMDD_type.parquet），可按日期精确拉（scp 单文件，不扫目录）。
+# - per-stock 类（valuation/三表/northbound 等）文件名无日期，走 sync_parquet.sh 周度全量。
+#
+# 覆盖：ak-events 每日产出（dragon_tiger/block_trade/margin_sse/restricted）。
+# 每日拉最近 7 天窗口（容错漏跑几天；节假日无文件=scp 失败=skip 正常）。
+set -u
+
+KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
+VPS=Administrator@49.232.102.198
+ROOT=/volume1/stock/sanguo_vnpy_v2
+DATA=$ROOT/data
+LOG=$ROOT/data_backup/parquet_daily.log
+
+TYPES="dragon_tiger block_trade margin_sse restricted"
+WINDOW_DAYS=7
+
+TS=$(date '+%F %T')
+echo "=== $TS parquet daily (per-date, ${WINDOW_DAYS}day window) ===" >> "$LOG"
+
+PULLED=0
+SKIPPED=0
+for type in $TYPES; do
+  mkdir -p "$DATA/static/$type"
+  for i in $(seq 0 $((WINDOW_DAYS - 1))); do
+    d=$(date -d "$i days ago" +%Y%m%d)
+    file="${d}_${type}.parquet"
+    if scp -q -i "$KEY" -o StrictHostKeyChecking=no \
+      "$VPS:C:/sanguo_vnpy_v2/data/static/$type/$file" \
+      "$DATA/static/$type/" 2>/dev/null; then
+      PULLED=$((PULLED + 1))
+    else
+      SKIPPED=$((SKIPPED + 1))
+    fi
+  done
+done
+
+echo "[$(date '+%T')] DAILY DONE pulled=$PULLED skipped=$SKIPPED (skip=节假日/未出,正常)" >> "$LOG"