docs(data): 归档数据层验证产物 + 数据层总览README
- scripts/data_platform/_archive/legacy/: 归档20个独立探针/诊断/旧降级脚本(零引用验证) - docs/archive/data/: 归档17个数据相关旧设计/plan/report(保留fusion spec作深读) - docs/data-platform/README.md: 数据层单一权威记录(8节:架构/布局/源/管线/铁律/API/缺口/待办) - 删除 _mootdx_depth_result.txt - Phase2待办: 15m灌库链+旧回填import链(有测试/wrapper依赖,VPS schtask确认后归档)
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
# 数据层归档脚本（_archive/）
|
||||
|
||||
本目录存放**已完成使命的验证/诊断/一次性脚本**，不再参与日常增量管线。
|
||||
保留于 git 历史便于回溯；如需重跑，移动回 `scripts/data_platform/` 顶层即可。
|
||||
|
||||
## legacy/ — 探针 / 诊断 / 旧降级 / 一次性验证（2026-07-29 归档）
|
||||
|
||||
| 脚本 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `probe_*.py`（12） | 探针 | 数据源/库/接口一次性 smoke 验证（akshare 状态/成份股/dbbardata 唯一性/退市/ETF/基本面/涨跌停/unified schema 等） |
|
||||
| `dbbardata_probe.py` | 探针 | dbbardata 表结构与行数抽查 |
|
||||
| `run_with_diag.py` / `diag_daily_update.ps1` | 诊断 | 带诊断输出的运行包装 |
|
||||
| `test_mootdx_depth.py` / `test_baostock_daily_constituent_sample.py` | 一次性验证 | mootdx 深度 / baostock 日线成份股采样（非 tests/ 正式套件） |
|
||||
| `resume_5yr_watcher.py` | 一次性 | 5 年全市场下载断点续传 watcher（已完成） |
|
||||
| `fallback.py` / `realtime.py` | 旧降级 | 旧多源降级管理器（日线 akshare→腾讯 / 实时 新浪→东财→腾讯），方案 A 后由 bs_eod/xt_eod 接管 |
|
||||
|
||||
归档前已验证：**零 import、无活跃 wrapper 引用**。
|
||||
|
||||
## backfill_15m/ — 15min 一次性灌库链（Phase 2 待归档）
|
||||
|
||||
⚠️ 未归档。`backfill_15min_baostock` 被 `tests/data/test_backfill_15min_hardening.py` 正式 import，
|
||||
`refresh_15min_daily` / `download_minute` / `download_15m_xtdata` / `raw_redownload` / `audit_data_layout`
|
||||
存在交叉引用或 ops wrapper 依赖，需 Phase 2 评估后统一处理。
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Smoke + timing: get_fundamentals_df fields= short-circuit + ThreadPool.
|
||||
|
||||
ASCII-only (VPS GBK console safe). Run on VPS:
|
||||
C:\\Python310\\python.exe -X utf8 probe_fundamentals_panel.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
os.environ.pop("http_proxy", None)
|
||||
os.environ.pop("https_proxy", None)
|
||||
os.environ.pop("all_proxy", None)
|
||||
|
||||
sys.path.insert(0, r"C:\sanguo_vnpy_v2")
|
||||
|
||||
from sanguo_portfolio.providers.local_unified_provider import LocalUnifiedProvider
|
||||
|
||||
DB = r"C:\sanguo_vnpy_v2\data\quant_trading.db"
|
||||
DATA = r"C:\sanguo_vnpy_v2\data"
|
||||
|
||||
p = LocalUnifiedProvider({"db_path": DB, "data_dir": DATA})
|
||||
|
||||
# candidate pool: pull a few hundred codes from constituent_unified (000985 = full mkt)
|
||||
try:
|
||||
codes = p.get_index_stocks("000985", "2024-06-03")
|
||||
except Exception as exc:
|
||||
print("get_index_stocks failed:", exc)
|
||||
codes = []
|
||||
codes = codes[:300] if codes else []
|
||||
jq = [c if "." in c else c + ".XSHE" for c in codes]
|
||||
print("pool size:", len(jq))
|
||||
if not jq:
|
||||
sys.exit(0)
|
||||
|
||||
date = "2024-06-03"
|
||||
|
||||
# warm caches once (first hit pays file open) to measure steady-ish state? No -
|
||||
# measure COLD first-rebalance (the real pain): fields=None full read.
|
||||
t0 = time.time()
|
||||
df_none = p.get_fundamentals_df(jq, date=date)
|
||||
t_none = time.time() - t0
|
||||
|
||||
# fresh provider to drop per-instance caches, measure fields= short-circuit cold
|
||||
p2 = LocalUnifiedProvider({"db_path": DB, "data_dir": DATA})
|
||||
t0 = time.time()
|
||||
df_fld = p2.get_fundamentals_df(jq, date=date, fields=["market_cap", "eps"])
|
||||
t_fld = time.time() - t0
|
||||
|
||||
print("fields=None : %6.2fs rows=%d cols=%d" % (t_none, len(df_none), len(df_none.columns)))
|
||||
print("fields=[mkt,ep]: %6.2fs rows=%d cols=%d" % (t_fld, len(df_fld), len(df_fld.columns)))
|
||||
if t_fld > 0:
|
||||
print("speedup : %.1fx" % (t_none / t_fld))
|
||||
|
||||
# correctness: market_cap + eps match between the two
|
||||
import pandas as pd
|
||||
common = [c for c in df_fld.columns if c in df_none.columns]
|
||||
for col in ("market_cap", "eps"):
|
||||
a = df_none[col].reindex(df_fld.index)
|
||||
b = df_fld[col]
|
||||
mask = a.notna() & b.notna()
|
||||
diff = (a[mask] - b[mask]).abs().max() if mask.any() else 0.0
|
||||
print("match %-12s: max_diff=%.6g" % (col, diff))
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Smoke: get_limit_status_batch on real dbbardata (window query + detection).
|
||||
|
||||
ASCII-only (VPS GBK console safe). Run on VPS:
|
||||
C:\\Python310\\python.exe -X utf8 probe_limit_status.py
|
||||
"""
|
||||
import collections
|
||||
import os
|
||||
import sys
|
||||
|
||||
for k in ("http_proxy", "https_proxy", "all_proxy"):
|
||||
os.environ.pop(k, None)
|
||||
|
||||
sys.path.insert(0, r"C:\sanguo_vnpy_v2")
|
||||
from sanguo_portfolio.providers.local_unified_provider import LocalUnifiedProvider
|
||||
|
||||
DB = r"C:\sanguo_vnpy_v2\data\quant_trading.db"
|
||||
DATA = r"C:\sanguo_vnpy_v2\data"
|
||||
p = LocalUnifiedProvider({"db_path": DB, "data_dir": DATA})
|
||||
|
||||
DATE = "2024-06-03"
|
||||
try:
|
||||
codes = p.get_index_stocks("000985", DATE)
|
||||
except Exception as exc:
|
||||
print("get_index_stocks failed:", exc)
|
||||
codes = []
|
||||
jq = [c if "." in c else c + ".XSHE" for c in codes[:800]]
|
||||
print("pool:", len(jq), "date:", DATE)
|
||||
|
||||
out = p.get_limit_status_batch(jq, DATE)
|
||||
cnt = collections.Counter()
|
||||
examples = {"up": [], "down": [], "paused": []}
|
||||
for k, v in (out or {}).items():
|
||||
if v is None:
|
||||
cnt["none"] += 1
|
||||
continue
|
||||
if v["is_limit_up"]:
|
||||
cnt["up"] += 1
|
||||
if len(examples["up"]) < 5:
|
||||
examples["up"].append(k)
|
||||
elif v["is_limit_down"]:
|
||||
cnt["down"] += 1
|
||||
if len(examples["down"]) < 5:
|
||||
examples["down"].append(k)
|
||||
if v["is_paused"]:
|
||||
cnt["paused"] += 1
|
||||
if len(examples["paused"]) < 5:
|
||||
examples["paused"].append(k)
|
||||
if not (v["is_limit_up"] or v["is_limit_down"] or v["is_paused"]):
|
||||
cnt["normal"] += 1
|
||||
|
||||
print("counts:", dict(cnt))
|
||||
print("limit_up examples:", examples["up"])
|
||||
print("limit_down examples:", examples["down"])
|
||||
print("paused examples:", examples["paused"])
|
||||
@@ -0,0 +1,134 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""UnifiedProvider + all_weather 数据链路诊断探针(VPS 跑, 快速版)。
|
||||
|
||||
每步带时间戳 + flush, 超时也能看卡哪。慢步骤降样本。
|
||||
定位: equity 重复日 / B_mean=0 / 数据缺失。
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
t0 = time.time()
|
||||
|
||||
|
||||
def step(name):
|
||||
print(f"\n=== {name} [+{time.time()-t0:.1f}s]", flush=True)
|
||||
|
||||
|
||||
def line(k, v):
|
||||
print(f"[{k}] {v}", flush=True)
|
||||
|
||||
|
||||
VPS_ROOT = r"C:\sanguo_vnpy_v2"
|
||||
DB = os.path.join(VPS_ROOT, "data", "quant_trading.db")
|
||||
|
||||
os.environ.setdefault("DEFAULT_DATA_PROVIDER", "jqdata")
|
||||
from unittest.mock import MagicMock
|
||||
if "jqdatasdk" not in sys.modules:
|
||||
_m = MagicMock()
|
||||
_m.utils.assert_auth = lambda f: f
|
||||
sys.modules["jqdatasdk"] = _m
|
||||
|
||||
sys.path.insert(0, VPS_ROOT)
|
||||
|
||||
step("STEP0 环境")
|
||||
line("python", sys.version.split()[0])
|
||||
line("PKG", os.path.isdir(os.path.join(VPS_ROOT, "sanguo_portfolio")))
|
||||
line("DB", os.path.exists(DB))
|
||||
conn = sqlite3.connect(DB)
|
||||
tabs = [r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||||
line("tables", tabs)
|
||||
|
||||
step("STEP0.5 混合 datetime 检测(单只抽样, 不全表 COUNT)")
|
||||
# 单只 600519 抽样看格式(走索引, 快)
|
||||
sample = conn.execute(
|
||||
"SELECT datetime FROM dbbardata WHERE symbol='600519' AND exchange='SSE' "
|
||||
"AND interval='d' ORDER BY datetime DESC LIMIT 5"
|
||||
).fetchall()
|
||||
line("600519 最近5条 datetime", [r[0] for r in sample])
|
||||
# DISTINCT 对比(单只, 索引内)
|
||||
d_raw = conn.execute(
|
||||
"SELECT COUNT(DISTINCT datetime) FROM dbbardata "
|
||||
"WHERE symbol='600519' AND exchange='SSE' AND interval='d'"
|
||||
).fetchone()[0]
|
||||
d_sub = conn.execute(
|
||||
"SELECT COUNT(DISTINCT substr(datetime,1,10)) FROM dbbardata "
|
||||
"WHERE symbol='600519' AND exchange='SSE' AND interval='d'"
|
||||
).fetchone()[0]
|
||||
line("DISTINCT datetime(原始)", d_raw)
|
||||
line("DISTINCT substr(datetime,1,10)(按日)", d_sub)
|
||||
line("重复日数(原始-按日)", d_raw - d_sub)
|
||||
|
||||
step("STEP1 get_trade_days 重复日期(equity_curve 重复 bug 根因)")
|
||||
from sanguo_portfolio.providers import LocalUnifiedProvider
|
||||
p = LocalUnifiedProvider({})
|
||||
days = p.get_trade_days(start_date="2024-01-02", end_date="2024-03-31")
|
||||
strs = [str(d)[:10] for d in days]
|
||||
line("trade_days total", len(days))
|
||||
line("unique dates", len(set(strs)))
|
||||
dup = [d for d, c in Counter(strs).items() if c > 1]
|
||||
line("DUP dates count", len(dup))
|
||||
line("DUP sample", dup[:5])
|
||||
|
||||
step("STEP2 成分股(constituent_unified 覆盖)")
|
||||
for idx in ["000300", "399101", "399001", "000852"]:
|
||||
try:
|
||||
s = p.get_index_stocks(idx)
|
||||
line(f"index_stocks {idx}", len(s))
|
||||
except Exception as e:
|
||||
line(f"index_stocks {idx} ERR", repr(e))
|
||||
|
||||
step("STEP3 fundamentals 600519(单股, 关键字段)")
|
||||
fdf = p.get_fundamentals_df(["600519.XSHG"], date="2024-03-29")
|
||||
cols_chk = [
|
||||
"code", "market_cap", "circulating_market_cap", "pe_ratio", "pb_ratio",
|
||||
"ps_ratio", "pcf_ratio", "eps", "roe", "roa", "gross_profit_margin",
|
||||
"net_profit_margin", "inc_revenue_year_on_year", "roic",
|
||||
]
|
||||
for c in cols_chk:
|
||||
if c in fdf.columns:
|
||||
line(f" {c}", fdf[c].iloc[0])
|
||||
else:
|
||||
line(f" {c}", "MISSING_COL")
|
||||
|
||||
step("STEP4 _trend_mean 小样本复算(hs300 前40, B_mean=0 根因)")
|
||||
import numpy as np
|
||||
hs300 = p.get_index_stocks("000300")
|
||||
line("hs300 size", len(hs300))
|
||||
# 只取前 40 只做 fundamentals(提速), top20 by circ_mktcap
|
||||
sample40 = hs300[:40]
|
||||
fdf2 = p.get_fundamentals_df(sample40, date="2024-03-29")
|
||||
line("fdf2 shape", fdf2.shape)
|
||||
if "circulating_market_cap" in fdf2.columns:
|
||||
line("circ_mktcap nonNaN", int(fdf2["circulating_market_cap"].notna().sum()))
|
||||
fdf2s = fdf2.sort_values("circulating_market_cap", ascending=False, na_position="last")
|
||||
blst = list(fdf2s.index)[:20]
|
||||
line("blst(20)", blst)
|
||||
df = p.get_price(blst, end_date="2024-03-29", frequency="1d", fields=["close"], count=10, panel=False)
|
||||
line("trend get_price isNone", df is None)
|
||||
if df is not None:
|
||||
line("trend get_price shape", df.shape)
|
||||
line("trend cols", list(df.columns))
|
||||
line("time dtype", df["time"].dtype if "time" in df.columns else "NO_TIME")
|
||||
print(df.head(3).to_string(), flush=True)
|
||||
try:
|
||||
pivot = df.pivot(index="time", columns="code", values="close")
|
||||
line("pivot shape", pivot.shape)
|
||||
if len(pivot) >= 2:
|
||||
change = (pivot.iloc[-1] / pivot.iloc[0] - 1) * 100
|
||||
arr = np.nan_to_num(change.to_numpy())
|
||||
line("B_mean manual", float(np.mean(arr)))
|
||||
line("change nonZero count", int((arr != 0).sum()))
|
||||
else:
|
||||
line("pivot rows<2", len(pivot))
|
||||
except Exception as e:
|
||||
line("pivot ERR", repr(e))
|
||||
|
||||
step("DONE")
|
||||
@@ -0,0 +1,28 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""monkey-patch engine 关键方法加诊断, 跑回测看 ETF cancel 根因(不改源码)。"""
|
||||
import sys, os, logging
|
||||
os.environ.setdefault("DEFAULT_DATA_PROVIDER", "jqdata")
|
||||
from unittest.mock import MagicMock
|
||||
m = MagicMock(); m.utils.assert_auth = lambda f: f
|
||||
sys.modules.setdefault("jqdatasdk", m)
|
||||
logging.basicConfig(level=logging.WARNING, format="%(message)s")
|
||||
|
||||
from bullet_trade.core import engine as eng
|
||||
|
||||
_orig_calc = eng.BacktestEngine._calculate_order_amount
|
||||
def calc(self, order, cp):
|
||||
r = _orig_calc(self, order, cp)
|
||||
print(f"[ENG_DIAG] {order.security} cp={cp} amount={r} tgt_val={getattr(order,'_target_value',None)} is_tgt={getattr(order,'_is_target_value',None)} order_amt={getattr(order,'amount',None)}", flush=True)
|
||||
return r
|
||||
eng.BacktestEngine._calculate_order_amount = calc
|
||||
|
||||
_orig_bp = eng.BacktestEngine._resolve_base_exec_price
|
||||
def bp(self, security, current_dt, fq_mode):
|
||||
r = _orig_bp(self, security, current_dt, fq_mode)
|
||||
print(f"[ENG_DIAG_BP] {security} dt={current_dt} fq={fq_mode} -> {r}", flush=True)
|
||||
return r
|
||||
eng.BacktestEngine._resolve_base_exec_price = bp
|
||||
|
||||
sys.argv = ['runner', '--provider', 'unified', '--max-pool', '30', '--start', '2024-01-02', '--end', '2024-01-31', '--cash', '1000000']
|
||||
from sanguo_portfolio.runner_backtest import main
|
||||
main()
|
||||
Reference in New Issue
Block a user