claude_dev
41cc6d13bf
spec §6 使用层 provider — 读方案A 权威数据层, 零 online, 治幸存者偏差:
- get_price: dbbardata('d') raw + bs_adjust_factor 前复权(asof, qfq[t]=raw[t]*factor[t])
- get_index_stocks/get_constituent: constituent_unified 并集治偏差(300=940含被踢, 无date时点)
- get_fundamentals_df: baostock pe/pb/ps/pcf + akshare 市值 + 三表委托 LocalParquetProvider
- 辅助: trade_days/security_info/current_tick/split_dividend/all_securities
VPS E2E 实证修复(Mac fixture 盲区):
- dbbardata datetime 混合格式("2024-09-26" vs "2024-09-26 00:00:00")
→ pd.to_datetime format='mixed' + SQL substr(datetime,1,10) 比日期(字符串比漏边界)
- 补 TestMixedDatetimeFormat 单测覆盖
验证: VPS 真数据 E2E 全通过(600519在市raw/qfq复权/000005退市治偏差/510300ETF/
fundamentals市值+pe+eps全字段/辅助方法); Mac 37单测+149回归绿
交付: 使用说明 docs/portfolio_local_unified_provider.md(其他 session 直用)+
plan+probe+E2E 脚本
131 lines
5.2 KiB
Python
131 lines
5.2 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""probe_unified_schema.py — 探查方案A 新权威数据层 schema (LocalUnifiedProvider 实现依赖)。
|
|
|
|
输出 constituent_unified / dbbardata('d') / bs_adjust_factor / valuation_baostock 的
|
|
列结构 + 样本 + 覆盖, 供 spec §6 使用层 provider 实现参考。
|
|
"""
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
BASE = Path(r"C:\sanguo_vnpy_v2")
|
|
DB = BASE / "data" / "quant_trading.db"
|
|
VAL_DIR = BASE / "data" / "valuation_baostock"
|
|
|
|
|
|
def section(title):
|
|
print(f"\n===== {title} =====")
|
|
|
|
|
|
# 1. constituent_unified
|
|
section("constituent_unified schema")
|
|
c = sqlite3.connect(str(DB))
|
|
try:
|
|
info = c.execute("PRAGMA table_info(constituent_unified)").fetchall()
|
|
print("columns:", [(r[1], r[2]) for r in info])
|
|
print("count/index:",
|
|
c.execute("SELECT COUNT(*), COUNT(DISTINCT index_code) FROM constituent_unified").fetchone())
|
|
print("per-index (total / in_current / was_removed):")
|
|
for row in c.execute("SELECT index_code, COUNT(*), SUM(in_current), SUM(was_removed) "
|
|
"FROM constituent_unified GROUP BY index_code ORDER BY index_code"):
|
|
print(" ", row)
|
|
print("sample 300:")
|
|
for row in c.execute("SELECT * FROM constituent_unified WHERE index_code LIKE '%300%' LIMIT 3"):
|
|
print(" ", row)
|
|
finally:
|
|
c.close()
|
|
|
|
# 2. dbbardata('d') raw 个股 + ETF
|
|
section("dbbardata('d') sample")
|
|
c = sqlite3.connect(str(DB))
|
|
try:
|
|
print("columns:", [r[1] for r in c.execute("PRAGMA table_info(dbbardata)").fetchall()])
|
|
for sym, lbl in [("600519", "个股在市"), ("000005", "退市"), ("510300", "ETF")]:
|
|
r = c.execute("SELECT COUNT(*), MIN(datetime), MAX(datetime) FROM dbbardata "
|
|
"WHERE symbol=? AND interval='d'", (sym,)).fetchone()
|
|
print(f" {sym}({lbl}): {r}")
|
|
print("sample 600519 last 3:")
|
|
for row in c.execute("SELECT symbol,exchange,datetime,volume,turnover,open_price,close_price "
|
|
"FROM dbbardata WHERE symbol='600519' AND interval='d' "
|
|
"ORDER BY datetime DESC LIMIT 3"):
|
|
print(" ", row)
|
|
finally:
|
|
c.close()
|
|
|
|
# 3. bs_adjust_factor
|
|
section("bs_adjust_factor schema")
|
|
c = sqlite3.connect(str(DB))
|
|
try:
|
|
tabs = [r[0] for r in c.execute("SELECT name FROM sqlite_master WHERE type='table' "
|
|
"AND name LIKE '%adjust%'").fetchall()]
|
|
print("adjust tables:", tabs)
|
|
if "bs_adjust_factor" in tabs:
|
|
print("columns:", [r[1] for r in c.execute("PRAGMA table_info(bs_adjust_factor)").fetchall()])
|
|
print("count:", c.execute("SELECT COUNT(*) FROM bs_adjust_factor").fetchone())
|
|
print("sample 600519:")
|
|
for row in c.execute("SELECT * FROM bs_adjust_factor WHERE code LIKE '%600519%' LIMIT 3"):
|
|
print(" ", row)
|
|
finally:
|
|
c.close()
|
|
|
|
# 4. valuation_baostock parquet
|
|
section("valuation_baostock parquet")
|
|
print("years:", sorted(p.name for p in VAL_DIR.glob("*.parquet")) if VAL_DIR.exists() else "DIR MISSING")
|
|
p2026 = VAL_DIR / "2026.parquet"
|
|
if p2026.exists():
|
|
df = pd.read_parquet(p2026)
|
|
print("columns:", list(df.columns))
|
|
print("shape:", df.shape)
|
|
print("sample 600519:")
|
|
sub = df[df["symbol"].astype(str).str.contains("600519")] if "symbol" in df.columns else df.head(0)
|
|
print(sub.head(3).to_dict("records"))
|
|
|
|
# 5. static akshare 三表 + valuation (market_cap/total_share 来源 + 文件格式)
|
|
from collections import Counter
|
|
section("static akshare (格式 + market_cap/total_share 来源)")
|
|
for sub in ["valuation", "balance", "income"]:
|
|
d = BASE / "data" / "static" / sub
|
|
if not d.exists():
|
|
print(f"{sub}: DIR MISSING")
|
|
continue
|
|
allf = list(d.iterdir())
|
|
exts = Counter(p.suffix for p in allf)
|
|
print(f"{sub}: {len(allf)} files, ext分布={dict(exts)}")
|
|
matches = sorted([p for p in allf if "600519" in p.name])
|
|
if not matches:
|
|
print(" 600519 无文件")
|
|
continue
|
|
f0 = matches[0]
|
|
with open(f0, "rb") as fh:
|
|
head = fh.read(8)
|
|
print(f" 600519 file={f0.name} magic={head[:4]!r}")
|
|
if head[:4] == b"PAR1":
|
|
df = pd.read_parquet(f0)
|
|
print(f" parquet cols({len(df.columns)}):", list(df.columns)[:20])
|
|
if sub == "valuation":
|
|
hit = [c for c in df.columns if any(k in str(c) for k in ("市值", "股本"))]
|
|
print(" 市值/股本列:", hit, "sample:", df.head(1).to_dict("records"))
|
|
if sub == "balance":
|
|
hit = [c for c in df.columns if any(k in str(c) for k in ("TOTAL_SHARES", "TOTAL_SHARE", "总股本", "实收资本"))]
|
|
print(" 股本相关列:", hit)
|
|
else:
|
|
print(f" 非 parquet (magic={head[:4]!r})")
|
|
|
|
# 6. 复权因子构造验证 (600519 foreAdjustFactor 语义)
|
|
section("复权因子 foreAdjustFactor 语义验证")
|
|
c = sqlite3.connect(str(DB))
|
|
try:
|
|
rows = c.execute("SELECT dividOperateDate, foreAdjustFactor FROM bs_adjust_factor "
|
|
"WHERE code='sh.600519' ORDER BY dividOperateDate").fetchall()
|
|
print("600519 除权事件数:", len(rows))
|
|
print("前3:", rows[:3])
|
|
print("后3:", rows[-3:])
|
|
print("语义推断: foreAdjustFactor 递增=累计前复权因子; 最新事件后=1.0; "
|
|
"qfq[t]=raw[t]*factor[t]")
|
|
finally:
|
|
c.close()
|
|
|
|
print("\nPROBE DONE")
|