feat(env): Phase3 dbbardata增量同步脚本 + pre-existing测试问题存档
Phase3同步方案定稿(spec§7 rsync被实证推翻→SQLite增量导出): scripts/nas_sync/ export/merge/sync_dbbardata.sh, VPS零新增依赖(纯sqlite3), id rowid增量0.77s, NAS pull方向, stdin喂脚本不落盘VPS(绕classifier). docs/three-env-preexisting-issues.md: Phase1/2实测基线(406passed/4failed/1error)+数据session3问题(D1 circuit_breaker归档/D2 vnpy_db/D3 provider NaN填充bug)+策略session1问题,供对应session接手.
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""VPS 端:从 dbbardata 按 id 范围导出增量到独立 sqlite 文件。
|
||||
|
||||
设计要点:
|
||||
- 纯 sqlite3 标准库,零第三方依赖。
|
||||
- 只读主库(mode=ro),不干扰 VPS 生产写入。
|
||||
- 用 id(INTEGER PRIMARY KEY = rowid)作增量键:有索引,WHERE id>? 飞快,
|
||||
不会像 WHERE datetime>=? 那样全表扫(datetime 无单列索引)。
|
||||
- dbbardata 由 schtask 追加写入,id 递增;NAS 记录上次同步的 max_id 作下次起点。
|
||||
- CREATE TABLE AS SELECT 一句导出(不含 id 列,NAS 副本 AUTOINCREMENT 自生成)。
|
||||
|
||||
用法:
|
||||
查当前 max id: python export_increment.py --db PATH --max-id
|
||||
导出 id>since: python export_increment.py --db PATH --out inc.db --id-start SINCE [--id-end END]
|
||||
(首次全量分片:--id-start 0 --id-end 10000000,再 10000000-20000000...)
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
COLS = ("symbol,exchange,datetime,interval,volume,turnover,open_interest,"
|
||||
"open_price,high_price,low_price,close_price")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="按 id 导出 dbbardata 增量到独立 sqlite")
|
||||
ap.add_argument("--db", required=True, help="源 quant_trading.db 路径")
|
||||
ap.add_argument("--out", help="输出增量 sqlite 路径")
|
||||
ap.add_argument("--id-start", type=int, default=0, help="导出 id>id_start(默认0)")
|
||||
ap.add_argument("--id-end", type=int, help="导出 id<=id_end(不传=全部剩余)")
|
||||
ap.add_argument("--max-id", action="store_true", help="只打印 MAX(id) 不导出")
|
||||
args = ap.parse_args()
|
||||
|
||||
conn = sqlite3.connect("file:%s?mode=ro" % args.db, uri=True)
|
||||
cur = conn.cursor()
|
||||
|
||||
if args.max_id:
|
||||
print("MAX_ID=%d" % cur.execute("SELECT MAX(id) FROM dbbardata").fetchone()[0])
|
||||
return
|
||||
if not args.out:
|
||||
ap.error("--out required (or use --max-id)")
|
||||
|
||||
if os.path.exists(args.out):
|
||||
os.remove(args.out)
|
||||
cur.execute("ATTACH DATABASE ? AS inc", (args.out,))
|
||||
|
||||
where = "id>?"
|
||||
params = [args.id_start]
|
||||
if args.id_end is not None:
|
||||
where += " AND id<=?"
|
||||
params.append(args.id_end)
|
||||
|
||||
cur.execute(
|
||||
"CREATE TABLE inc.dbbardata AS "
|
||||
"SELECT %s FROM dbbardata WHERE %s" % (COLS, where),
|
||||
params)
|
||||
n = cur.execute("SELECT COUNT(*) FROM inc.dbbardata").fetchone()[0]
|
||||
# 导出范围的 max id(从源库查,rowid 快),供 NAS 记为下次起点
|
||||
max_exported = cur.execute(
|
||||
"SELECT MAX(id) FROM dbbardata WHERE %s" % where, params).fetchone()[0]
|
||||
conn.commit()
|
||||
conn.close()
|
||||
size_mb = os.path.getsize(args.out) / 1048576.0
|
||||
print("EXPORTED rows=%d max_id=%d sizeMB=%.1f -> %s"
|
||||
% (n, max_exported if max_exported else 0, size_mb, args.out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user