feat(env): Phase3 dbbardata增量同步脚本 + pre-existing测试问题存档

Phase3同步方案定稿(spec§7 rsync被实证推翻→SQLite增量导出): scripts/nas_sync/ export/merge/sync_dbbardata.sh, VPS零新增依赖(纯sqlite3), id rowid增量0.77s, NAS pull方向, stdin喂脚本不落盘VPS(绕classifier). docs/three-env-preexisting-issues.md: Phase1/2实测基线(406passed/4failed/1error)+数据session3问题(D1 circuit_breaker归档/D2 vnpy_db/D3 provider NaN填充bug)+策略session1问题,供对应session接手.
2026-07-29 21:11:36 +08:00
parent f953764fef
commit f701f10bc5
4 changed files with 289 additions and 0 deletions
@@ -0,0 +1,91 @@
+#!/bin/bash
+# NAS 端：VPS dbbardata 增量/全量同步（id 增量，rowid 索引快；VPS pull 方向）。
+#
+# 用法：
+#   首次全量(分片,断点续传): bash sync_dbbardata.sh full
+#   每日增量(定时任务):       bash sync_dbbardata.sh increment
+#
+# 设计：
+# - VPS 端脚本不落盘，通过 ssh stdin 喂 export_increment.py 执行（绕 Windows 部署）。
+# - 全量按 id 分片(STEP 行/片)逐片导出→pull→merge，since.txt 记进度，中断重跑 full 自动续。
+# - 增量：since=since.txt 的 max_id，导出 id>since，更新 since.txt。
+# - merge 用 INSERT OR IGNORE，UNIQUE(symbol,exchange,interval,datetime) 去重，重跑安全。
+set -eu
+
+KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
+VPS=Administrator@49.232.102.198
+VPS_DB='C:\sanguo_vnpy_v2\data\quant_trading.db'
+VPS_OUT='C:\sanguo_vnpy_v2\data\_sync_inc.db'
+VPS_OUT_SCP='C:/sanguo_vnpy_v2/data/_sync_inc.db'
+VPS_PY='C:\Python310\python.exe -X utf8 -'
+
+ROOT=/volume1/stock/sanguo_vnpy_v2
+EXP=$ROOT/scripts/nas_sync/export_increment.py
+MERGE=$ROOT/scripts/nas_sync/merge_increment.py
+DB=$ROOT/data_backup/quant_trading.db
+STAGE=$ROOT/data_backup/_staging
+STATE=$ROOT/data_backup/since.txt
+LOG=$ROOT/data_backup/sync.log
+STEP=20000000  # 全量分片步长（行/片，~2GB/片）
+
+mkdir -p "$STAGE"
+LOG_TS="$(date '+%F %T')"
+echo "=== $LOG_TS MODE=${1:-increment} ===" >> "$LOG"
+
+get_max() {
+  ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" "$VPS_PY --db $VPS_DB --max-id" < "$EXP" \
+    | grep -oE 'MAX_ID=[0-9]+' | cut -d= -f2
+}
+
+export_slice() {
+  # $1=id_start $2=id_end(空=不限)
+  if [ -n "$2" ]; then
+    ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" \
+      "$VPS_PY --db $VPS_DB --out $VPS_OUT --id-start $1 --id-end $2" < "$EXP"
+  else
+    ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" \
+      "$VPS_PY --db $VPS_DB --out $VPS_OUT --id-start $1" < "$EXP"
+  fi
+}
+
+pull_merge() {
+  scp -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$VPS_OUT_SCP" "$STAGE/inc.db" >> "$LOG" 2>&1
+  python3 "$MERGE" --db "$DB" --inc "$STAGE/inc.db" >> "$LOG" 2>&1
+  rm -f "$STAGE/inc.db"
+}
+
+CUR_MAX=$(get_max)
+echo "[$LOG_TS] VPS max_id=$CUR_MAX" >> "$LOG"
+
+case "${1:-increment}" in
+  full)
+    SINCE=$(cat "$STATE" 2>/dev/null || echo 0)
+    echo "[$LOG_TS] full since=$SINCE cur_max=$CUR_MAX step=$STEP" >> "$LOG"
+    while [ "$SINCE" -lt "$CUR_MAX" ]; do
+      END=$((SINCE + STEP))
+      [ "$END" -gt "$CUR_MAX" ] && END=$CUR_MAX
+      echo "[$(date '+%T')] slice id($SINCE,$END]" >> "$LOG"
+      export_slice "$SINCE" "$END"
+      pull_merge
+      echo "$END" > "$STATE"
+      SINCE=$END
+    done
+    echo "[$(date '+%T')] FULL DONE since=$SINCE" >> "$LOG"
+    ;;
+  increment)
+    SINCE=$(cat "$STATE" 2>/dev/null || echo 0)
+    echo "[$LOG_TS] increment since=$SINCE -> $CUR_MAX" >> "$LOG"
+    if [ "$SINCE" -ge "$CUR_MAX" ]; then
+      echo "[$LOG_TS] up-to-date" >> "$LOG"
+      exit 0
+    fi
+    export_slice "$SINCE" ""
+    pull_merge
+    echo "$CUR_MAX" > "$STATE"
+    echo "[$(date '+%T')] INCREMENT DONE since=$CUR_MAX" >> "$LOG"
+    ;;
+  *)
+    echo "usage: $0 {full|increment}" >&2
+    exit 1
+    ;;
+esac