feat(portfolio): LocalUnifiedProvider spec §6 使用层落地 + VPS E2E(Task6)
spec §6 使用层 provider — 读方案A 权威数据层, 零 online, 治幸存者偏差:
- get_price: dbbardata('d') raw + bs_adjust_factor 前复权(asof, qfq[t]=raw[t]*factor[t])
- get_index_stocks/get_constituent: constituent_unified 并集治偏差(300=940含被踢, 无date时点)
- get_fundamentals_df: baostock pe/pb/ps/pcf + akshare 市值 + 三表委托 LocalParquetProvider
- 辅助: trade_days/security_info/current_tick/split_dividend/all_securities
VPS E2E 实证修复(Mac fixture 盲区):
- dbbardata datetime 混合格式("2024-09-26" vs "2024-09-26 00:00:00")
→ pd.to_datetime format='mixed' + SQL substr(datetime,1,10) 比日期(字符串比漏边界)
- 补 TestMixedDatetimeFormat 单测覆盖
验证: VPS 真数据 E2E 全通过(600519在市raw/qfq复权/000005退市治偏差/510300ETF/
fundamentals市值+pe+eps全字段/辅助方法); Mac 37单测+149回归绿
交付: 使用说明 docs/portfolio_local_unified_provider.md(其他 session 直用)+
plan+probe+E2E 脚本
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""probe_unified_schema.py — 探查方案A 新权威数据层 schema (LocalUnifiedProvider 实现依赖)。
|
||||
|
||||
输出 constituent_unified / dbbardata('d') / bs_adjust_factor / valuation_baostock 的
|
||||
列结构 + 样本 + 覆盖, 供 spec §6 使用层 provider 实现参考。
|
||||
"""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
BASE = Path(r"C:\sanguo_vnpy_v2")
|
||||
DB = BASE / "data" / "quant_trading.db"
|
||||
VAL_DIR = BASE / "data" / "valuation_baostock"
|
||||
|
||||
|
||||
def section(title):
|
||||
print(f"\n===== {title} =====")
|
||||
|
||||
|
||||
# 1. constituent_unified
|
||||
section("constituent_unified schema")
|
||||
c = sqlite3.connect(str(DB))
|
||||
try:
|
||||
info = c.execute("PRAGMA table_info(constituent_unified)").fetchall()
|
||||
print("columns:", [(r[1], r[2]) for r in info])
|
||||
print("count/index:",
|
||||
c.execute("SELECT COUNT(*), COUNT(DISTINCT index_code) FROM constituent_unified").fetchone())
|
||||
print("per-index (total / in_current / was_removed):")
|
||||
for row in c.execute("SELECT index_code, COUNT(*), SUM(in_current), SUM(was_removed) "
|
||||
"FROM constituent_unified GROUP BY index_code ORDER BY index_code"):
|
||||
print(" ", row)
|
||||
print("sample 300:")
|
||||
for row in c.execute("SELECT * FROM constituent_unified WHERE index_code LIKE '%300%' LIMIT 3"):
|
||||
print(" ", row)
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
# 2. dbbardata('d') raw 个股 + ETF
|
||||
section("dbbardata('d') sample")
|
||||
c = sqlite3.connect(str(DB))
|
||||
try:
|
||||
print("columns:", [r[1] for r in c.execute("PRAGMA table_info(dbbardata)").fetchall()])
|
||||
for sym, lbl in [("600519", "个股在市"), ("000005", "退市"), ("510300", "ETF")]:
|
||||
r = c.execute("SELECT COUNT(*), MIN(datetime), MAX(datetime) FROM dbbardata "
|
||||
"WHERE symbol=? AND interval='d'", (sym,)).fetchone()
|
||||
print(f" {sym}({lbl}): {r}")
|
||||
print("sample 600519 last 3:")
|
||||
for row in c.execute("SELECT symbol,exchange,datetime,volume,turnover,open_price,close_price "
|
||||
"FROM dbbardata WHERE symbol='600519' AND interval='d' "
|
||||
"ORDER BY datetime DESC LIMIT 3"):
|
||||
print(" ", row)
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
# 3. bs_adjust_factor
|
||||
section("bs_adjust_factor schema")
|
||||
c = sqlite3.connect(str(DB))
|
||||
try:
|
||||
tabs = [r[0] for r in c.execute("SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name LIKE '%adjust%'").fetchall()]
|
||||
print("adjust tables:", tabs)
|
||||
if "bs_adjust_factor" in tabs:
|
||||
print("columns:", [r[1] for r in c.execute("PRAGMA table_info(bs_adjust_factor)").fetchall()])
|
||||
print("count:", c.execute("SELECT COUNT(*) FROM bs_adjust_factor").fetchone())
|
||||
print("sample 600519:")
|
||||
for row in c.execute("SELECT * FROM bs_adjust_factor WHERE code LIKE '%600519%' LIMIT 3"):
|
||||
print(" ", row)
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
# 4. valuation_baostock parquet
|
||||
section("valuation_baostock parquet")
|
||||
print("years:", sorted(p.name for p in VAL_DIR.glob("*.parquet")) if VAL_DIR.exists() else "DIR MISSING")
|
||||
p2026 = VAL_DIR / "2026.parquet"
|
||||
if p2026.exists():
|
||||
df = pd.read_parquet(p2026)
|
||||
print("columns:", list(df.columns))
|
||||
print("shape:", df.shape)
|
||||
print("sample 600519:")
|
||||
sub = df[df["symbol"].astype(str).str.contains("600519")] if "symbol" in df.columns else df.head(0)
|
||||
print(sub.head(3).to_dict("records"))
|
||||
|
||||
# 5. static akshare 三表 + valuation (market_cap/total_share 来源 + 文件格式)
|
||||
from collections import Counter
|
||||
section("static akshare (格式 + market_cap/total_share 来源)")
|
||||
for sub in ["valuation", "balance", "income"]:
|
||||
d = BASE / "data" / "static" / sub
|
||||
if not d.exists():
|
||||
print(f"{sub}: DIR MISSING")
|
||||
continue
|
||||
allf = list(d.iterdir())
|
||||
exts = Counter(p.suffix for p in allf)
|
||||
print(f"{sub}: {len(allf)} files, ext分布={dict(exts)}")
|
||||
matches = sorted([p for p in allf if "600519" in p.name])
|
||||
if not matches:
|
||||
print(" 600519 无文件")
|
||||
continue
|
||||
f0 = matches[0]
|
||||
with open(f0, "rb") as fh:
|
||||
head = fh.read(8)
|
||||
print(f" 600519 file={f0.name} magic={head[:4]!r}")
|
||||
if head[:4] == b"PAR1":
|
||||
df = pd.read_parquet(f0)
|
||||
print(f" parquet cols({len(df.columns)}):", list(df.columns)[:20])
|
||||
if sub == "valuation":
|
||||
hit = [c for c in df.columns if any(k in str(c) for k in ("市值", "股本"))]
|
||||
print(" 市值/股本列:", hit, "sample:", df.head(1).to_dict("records"))
|
||||
if sub == "balance":
|
||||
hit = [c for c in df.columns if any(k in str(c) for k in ("TOTAL_SHARES", "TOTAL_SHARE", "总股本", "实收资本"))]
|
||||
print(" 股本相关列:", hit)
|
||||
else:
|
||||
print(f" 非 parquet (magic={head[:4]!r})")
|
||||
|
||||
# 6. 复权因子构造验证 (600519 foreAdjustFactor 语义)
|
||||
section("复权因子 foreAdjustFactor 语义验证")
|
||||
c = sqlite3.connect(str(DB))
|
||||
try:
|
||||
rows = c.execute("SELECT dividOperateDate, foreAdjustFactor FROM bs_adjust_factor "
|
||||
"WHERE code='sh.600519' ORDER BY dividOperateDate").fetchall()
|
||||
print("600519 除权事件数:", len(rows))
|
||||
print("前3:", rows[:3])
|
||||
print("后3:", rows[-3:])
|
||||
print("语义推断: foreAdjustFactor 递增=累计前复权因子; 最新事件后=1.0; "
|
||||
"qfq[t]=raw[t]*factor[t]")
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
print("\nPROBE DONE")
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""verify_unified_e2e.py — LocalUnifiedProvider VPS 真数据 E2E(spec §6 Task6)。
|
||||
|
||||
直接实例化 provider 测三大接口(不跑 bullet_trade 全回测,精确验证):
|
||||
- get_price 读 dbbardata('d'): 在市(600519)/退市(000005 治偏差)/ETF(510300) + raw vs qfq 复权
|
||||
- get_index_stocks 并集(constituent_unified,含被踢)
|
||||
- get_fundamentals_df 市值(akshare)+ pe/pb(baostock)+ 三表
|
||||
每步 flush(ssh 非交互 stdout block-buffered)。避开 get_all_securities(全表 distinct 28GB 慢)。
|
||||
"""
|
||||
import sys
|
||||
sys.path.insert(0, r"C:\sanguo_vnpy_v2")
|
||||
try:
|
||||
sys.stdout.reconfigure(line_buffering=True) # 每行 flush
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from sanguo_portfolio.providers import LocalUnifiedProvider
|
||||
|
||||
p = LocalUnifiedProvider()
|
||||
|
||||
|
||||
def step(title):
|
||||
print(f"\n===== {title} =====", flush=True)
|
||||
|
||||
|
||||
step("1. get_price 读 dbbardata('d') 混合 datetime 格式")
|
||||
df = p.get_price("600519.XSHG", start_date="2024-09-25", end_date="2024-09-30", fq="raw")
|
||||
print(f"600519 raw: {len(df)} 行", df.tail(1).to_dict("records") if len(df) else "EMPTY", flush=True)
|
||||
df_q = p.get_price("600519.XSHG", start_date="2024-09-25", end_date="2024-09-30", fq="qfq")
|
||||
print(f"600519 qfq(前复权): {len(df_q)} 行", df_q.tail(1).to_dict("records") if len(df_q) else "EMPTY", flush=True)
|
||||
df_d = p.get_price("000005.XSHE", start_date="2024-04-20", end_date="2024-04-30")
|
||||
print(f"000005 退市(治偏差): {len(df_d)} 行", "✅有数据" if len(df_d) else "❌空!偏差未治", flush=True)
|
||||
df_e = p.get_price("510300.SH", start_date="2024-09-25", end_date="2024-09-30")
|
||||
print(f"510300 ETF(xtata源): {len(df_e)} 行", flush=True)
|
||||
df_p = p.get_price(["600519.XSHG", "000001.XSHE"], end_date="2024-09-30", count=2, panel=False, fields=["close"])
|
||||
print(f"panel=False 长表: {len(df_p)} 行, 列={list(df_p.columns) if len(df_p) else 'EMPTY'}", flush=True)
|
||||
|
||||
step("2. get_index_stocks 并集(constituent_unified 治偏差)")
|
||||
for idx in ["000300.XSHG", "000905.XSHG", "000016.XSHG"]:
|
||||
s = p.get_index_stocks(idx)
|
||||
print(f" {idx}: {len(s)} 只(含被踢) e.g. {s[:2]}", flush=True)
|
||||
print(f" get_constituent 别名 300: {len(p.get_constituent('000300'))} 只", flush=True)
|
||||
|
||||
step("3. get_fundamentals_df(市值 akshare + pe/pb baostock + 三表)")
|
||||
fund = p.get_fundamentals_df(["600519.XSHG", "000001.XSHE"], date="2024-09-30")
|
||||
cols = ["code", "market_cap", "circulating_market_cap", "pe_ratio", "pb_ratio", "ps_ratio", "eps"]
|
||||
print(fund[cols].to_string(), flush=True)
|
||||
|
||||
step("4. 辅助方法(轻量,避全表扫)")
|
||||
print("get_trade_days(count=3):", [d.strftime("%Y-%m-%d") for d in p.get_trade_days(count=3)], flush=True)
|
||||
print("get_security_info 600519:", p.get_security_info("600519.XSHG"), flush=True)
|
||||
tick = p.get_current_tick("600519.XSHG")
|
||||
print("get_current_tick:", {k: tick[k] for k in ("close", "high_limit", "low_limit")} if tick else None, flush=True)
|
||||
print("get_split_dividend 600519(2024):", len(p.get_split_dividend("600519.XSHG", "2024-01-01", "2024-12-31")), "事件", flush=True)
|
||||
|
||||
print("\nE2E DONE — LocalUnifiedProvider VPS 真数据验证通过", flush=True)
|
||||
Reference in New Issue
Block a user