diff --git a/scripts/nas_sync/merge_increment.py b/scripts/nas_sync/merge_increment.py index e980c08..92bd30f 100644 --- a/scripts/nas_sync/merge_increment.py +++ b/scripts/nas_sync/merge_increment.py @@ -30,6 +30,10 @@ def main(): conn = sqlite3.connect(args.db) cur = conn.cursor() + # 大表(1.8亿行+)merge 提速: 默认2MB cache 致 UNIQUE索引(数GB)全磁盘IO, merge>120s超时 + for _p in ("PRAGMA journal_mode=WAL", "PRAGMA synchronous=NORMAL", + "PRAGMA cache_size=-500000", "PRAGMA temp_store=MEMORY"): + cur.execute(_p) cur.execute(SCHEMA) cur.execute( "CREATE UNIQUE INDEX IF NOT EXISTS uq_dbbardata " diff --git a/scripts/nas_sync/sync_dbbardata.sh b/scripts/nas_sync/sync_dbbardata.sh index 473fa9c..f521040 100644 --- a/scripts/nas_sync/sync_dbbardata.sh +++ b/scripts/nas_sync/sync_dbbardata.sh @@ -26,7 +26,7 @@ DB=$ROOT/data_backup/quant_trading.db STAGE=$ROOT/data_backup/_staging STATE=$ROOT/data_backup/since.txt LOG=$ROOT/data_backup/sync.log -STEP=20000000 # 全量分片步长(行/片,~2GB/片) +STEP=2000000 # 全量分片步长(行/片,~200MB/片;首版2000万/2GB致首片export+scp跨公网挂,降至此) mkdir -p "$STAGE" LOG_TS="$(date '+%F %T')"