claude_dev
f701f10bc5
Phase3同步方案定稿(spec§7 rsync被实证推翻→SQLite增量导出): scripts/nas_sync/ export/merge/sync_dbbardata.sh, VPS零新增依赖(纯sqlite3), id rowid增量0.77s, NAS pull方向, stdin喂脚本不落盘VPS(绕classifier). docs/three-env-preexisting-issues.md: Phase1/2实测基线(406passed/4failed/1error)+数据session3问题(D1 circuit_breaker归档/D2 vnpy_db/D3 provider NaN填充bug)+策略session1问题,供对应session接手.
56 lines
2.1 KiB
Python
56 lines
2.1 KiB
Python
"""NAS 端：把增量 sqlite 合并进本地 dbbardata 副本。
|
|
|
|
设计要点：
|
|
- 纯 sqlite3 标准库，零第三方依赖（NAS 宿主 Python 3.8 也能跑）。
|
|
- INSERT OR IGNORE ... SELECT：靠 UNIQUE(symbol,exchange,interval,datetime)
|
|
去重，重复行安全跳过（增量重跑、首全量分片重叠都不怕）。
|
|
- 首次合并自动建表+建唯一索引（与 VPS schema 对齐，id AUTOINCREMENT）。
|
|
|
|
用法：
|
|
python merge_increment.py --db /volume1/.../quant_trading.db --inc /tmp/inc.db
|
|
"""
|
|
import argparse
|
|
import sqlite3
|
|
|
|
SCHEMA = """CREATE TABLE IF NOT EXISTS dbbardata(
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
symbol TEXT, exchange TEXT, datetime TEXT, interval TEXT,
|
|
volume REAL, turnover REAL, open_interest REAL,
|
|
open_price REAL, high_price REAL, low_price REAL, close_price REAL)"""
|
|
|
|
COLS = ("symbol,exchange,datetime,interval,volume,turnover,open_interest,"
|
|
"open_price,high_price,low_price,close_price")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description="合并增量 sqlite 进本地 dbbardata 副本")
|
|
ap.add_argument("--db", required=True, help="NAS 本地 quant_trading.db 路径")
|
|
ap.add_argument("--inc", required=True, help="增量 sqlite 路径")
|
|
args = ap.parse_args()
|
|
|
|
conn = sqlite3.connect(args.db)
|
|
cur = conn.cursor()
|
|
cur.execute(SCHEMA)
|
|
cur.execute(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS uq_dbbardata "
|
|
"ON dbbardata(symbol,exchange,interval,datetime)")
|
|
cur.execute("ATTACH DATABASE ? AS inc", (args.inc,))
|
|
|
|
inc_count = cur.execute("SELECT COUNT(*) FROM inc.dbbardata").fetchone()[0]
|
|
before = cur.execute("SELECT COUNT(*) FROM main.dbbardata").fetchone()[0]
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO main.dbbardata(%s) SELECT %s FROM inc.dbbardata"
|
|
% (COLS, COLS))
|
|
conn.commit()
|
|
after = cur.execute("SELECT COUNT(*) FROM main.dbbardata").fetchone()[0]
|
|
conn.close()
|
|
|
|
inserted = after - before
|
|
skipped = inc_count - inserted
|
|
print("MERGED inc=%d inserted=%d skipped(dup)=%d before=%d after=%d"
|
|
% (inc_count, inserted, skipped, before, after))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|