From d76171434be221c1e1c97d38345c60a6de59e751 Mon Sep 17 00:00:00 2001
From: claude_dev <claude-dev@sanguo.local>
Date: Thu, 30 Jul 2026 18:08:42 +0800
Subject: [PATCH] =?UTF-8?q?fix(nas=5Fsync):=20Phase3=20full=20=E4=BF=AE?=
 =?UTF-8?q?=E5=A4=8D(STEP=E9=99=8D=E7=89=87+merge=20PRAGMA)+=E5=85=A8?=
 =?UTF-8?q?=E9=87=8F=E5=AE=8C=E6=88=902.02=E4=BA=BF=E8=A1=8C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- sync_dbbardata.sh: STEP 2000万→200万/片(首片2GB export+scp跨公网挂)
- merge_increment.py: 加 PRAGMA cache=500MB+WAL+NORMAL+synchronous
  (replica 2亿行 UNIQUE索引数GB,默认2MB cache致索引IO merge>120s超时)
- 全量完成: replica 2.019亿行(=dbbardata全量日线+15min+ETF)
- 踩坑链: ssh reset中断→timeout杀ssh留NAS merge残留持锁→since落后数据已在库
- 教训: timeout杀ssh不杀远程python;merge大表必加PRAGMA;断点续传since.txt救命

Co-Authored-By: Claude <noreply@anthropic.com>
---
 scripts/nas_sync/merge_increment.py | 4 ++++
 scripts/nas_sync/sync_dbbardata.sh  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/nas_sync/merge_increment.py b/scripts/nas_sync/merge_increment.py
index e980c08..92bd30f 100644
--- a/scripts/nas_sync/merge_increment.py
+++ b/scripts/nas_sync/merge_increment.py
@@ -30,6 +30,10 @@ def main():
 
     conn = sqlite3.connect(args.db)
     cur = conn.cursor()
+    # 大表(1.8亿行+)merge 提速: 默认2MB cache 致 UNIQUE索引(数GB)全磁盘IO, merge>120s超时
+    for _p in ("PRAGMA journal_mode=WAL", "PRAGMA synchronous=NORMAL",
+               "PRAGMA cache_size=-500000", "PRAGMA temp_store=MEMORY"):
+        cur.execute(_p)
     cur.execute(SCHEMA)
     cur.execute(
         "CREATE UNIQUE INDEX IF NOT EXISTS uq_dbbardata "
diff --git a/scripts/nas_sync/sync_dbbardata.sh b/scripts/nas_sync/sync_dbbardata.sh
index 473fa9c..f521040 100644
--- a/scripts/nas_sync/sync_dbbardata.sh
+++ b/scripts/nas_sync/sync_dbbardata.sh
@@ -26,7 +26,7 @@ DB=$ROOT/data_backup/quant_trading.db
 STAGE=$ROOT/data_backup/_staging
 STATE=$ROOT/data_backup/since.txt
 LOG=$ROOT/data_backup/sync.log
-STEP=20000000  # 全量分片步长（行/片，~2GB/片）
+STEP=2000000  # 全量分片步长（行/片，~200MB/片；首版2000万/2GB致首片export+scp跨公网挂,降至此）
 
 mkdir -p "$STAGE"
 LOG_TS="$(date '+%F %T')"