feat(data): 方案A 数据层落地(spec §14)— DB唯一表+治幸存者偏差+权威源

spec §14 方案A 数据层迁移完成 + E2E 验证(read_db_daily: 在市/退市治偏差/ETF 全OK): - dbbardata('d') 1826万含退市(治回测幸存者偏差, INSERT OR REPLACE staging迁移, WHERE OHLC NOT NULL+COALESCE) - constituent_unified 7110行/9指数(300/500/50 baostock全集 + 深证4指 akshare cni union + 中证1000/2000 snapshot) - pe/pb 不进DB -> valuation_baostock/<year>.parquet 按年宽表(2003-2026) - 废弃 daily_baostock_full/bs_index_constituent(rename _old 保留); 旧4 schtask disabled - 新 schtask sanguo-bs-eod 18:05(baostock个股日线+15min+拆pe/pb DAILY_LIMIT 48000) + sanguo-xt-eod 18:40(ETF/基金xtata) - 权威源: baostock个股日线+估值+15min+复权+300/500/50 / xtata ETF+基金+当天实时 / akshare三表+事件+深证中证成份股 - 全程备份+staging+_old保留可回滚; 脚本 audit/probe/migrate/merge/cleanup/fix_config/verify/bs_eod/xt_eod/wrapper/register_schtasks - 待办(spec §6 使用层): LocalParquetProvider 接 constituent_unified+valuation_baostock + 实时拼接
2026-07-23 07:20:34 +08:00
parent b270faf4b9
commit c2a89d01a4
27 changed files with 2062 additions and 0 deletions
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""audit_data_layout.py — 只读盘点 VPS 本地数据布局(DB表行数/maxdate + data/目录)。
+
+用法:
+  python audit_data_layout.py [BASE_DIR]
+默认 BASE=C:\\sanguo_vnpy_v2 (VPS)。Mac 调试传本地路径。
+
+只 SELECT / 遍历目录,不写任何东西。
+"""
+import sqlite3
+import sys
+from pathlib import Path
+
+BASE = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(r"C:\sanguo_vnpy_v2")
+DB = BASE / "data" / "quant_trading.db"
+DATA = BASE / "data"
+
+
+def line(s=""):
+    print(s)
+
+
+# ======================== DB 表盘点 ========================
+line("=" * 72)
+line(f"DB: {DB}  exists={DB.exists()}")
+line("=" * 72)
+if DB.exists():
+    c = sqlite3.connect(str(DB), timeout=60)
+    tables = [r[0] for r in c.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")]
+    SKIP_COUNT = {"dbbardata", "daily_baostock_full"}  # 千万/亿级 COUNT 慢, 跳过
+    for t in tables:
+        if t in SKIP_COUNT:
+            line(f"  {t:42s} (skip COUNT, see max below)")
+            continue
+        try:
+            n = c.execute(f"SELECT COUNT(*) FROM '{t}'").fetchone()[0]
+        except Exception as e:
+            n = f"ERR {e}"
+        line(f"  {t:42s} {n}")
+
+    # 关键表 schema + 时间范围 (大表 dbbardata/daily_baostock_full 跳过慢查询)
+    for t in ("bs_index_constituent", "bs_adjust_factor"):
+        try:
+            cols = [r[1] for r in c.execute(f"PRAGMA table_info('{t}')")]
+            if not cols:
+                continue
+            line("")
+            line(f"[{t}] cols({len(cols)}): {cols}")
+            if "symbol" in cols:
+                nd = c.execute(f"SELECT COUNT(DISTINCT symbol) FROM '{t}'").fetchone()[0]
+                line(f"  distinct symbol = {nd}")
+            for dc in ("date", "datetime", "trade_date"):
+                if dc in cols:
+                    mn = c.execute(f"SELECT MIN({dc}) FROM '{t}'").fetchone()[0]
+                    mx = c.execute(f"SELECT MAX({dc}) FROM '{t}'").fetchone()[0]
+                    line(f"  {dc}: {mn} ~ {mx}")
+                    break
+        except Exception as e:
+            line(f"[{t}] ERR {e}")
+    c.close()
+
+# ======================== data/ 目录盘点 ========================
+line("")
+line("=" * 72)
+line(f"DATA DIR: {DATA}  exists={DATA.exists()}")
+line("=" * 72)
+if DATA.exists():
+    for sub in sorted(DATA.iterdir()):
+        name = sub.name
+        if name.startswith("."):
+            continue
+        try:
+            if sub.is_dir():
+                files = list(sub.rglob("*"))
+                n_files = sum(1 for f in files if f.is_file())
+                n_pq = sum(1 for f in files if f.suffix == ".parquet")
+                n_xls = sum(1 for f in files if f.suffix in (".xls", ".xlsx"))
+                line(f"  [DIR] {name:34s} files={n_files:6d} parquet={n_pq:6d} xls={n_xls}")
+            elif sub.is_file():
+                sz = sub.stat().st_size
+                unit, val = ("GB", sz / 1024 ** 3) if sz > 1024 ** 3 else (
+                    "MB", sz / 1024 ** 2)
+                line(f"  [FILE] {name:32s} {val:8.1f} {unit}")
+        except Exception as e:
+            line(f"  {name} ERR {e}")