Files
sanguo_vnpy_v2/sanguo_trader/shadow/reconcile_report.py
T
claude_dev 31596f58f9
CI/CD / test (push) Successful in 14s
CI/CD / nas-deploy (push) Successful in 24s
CI/CD / nas-verify (push) Successful in 1s
fix(shadow): 双轨对账count_match改按票+方向聚合总量——QMT部分成交拆行致27vs5恒False纯计数噪音 [vps]
08-24实录: momentum一笔3800股被QMT拆19笔fill,对账27vs5恒count_match=False,总量却分毫不差。旧口径按原始行数比(注释自称'差异如实呈现待归因',实则部分成交属成交粒度非分歧)。修=按聚合桶live_volume==shadow_volume判;原始行数保留在live_count/shadow_count供归因。真量差仍判False(新增测试钉死)。trader 254绿。
2026-08-24 23:07:12 +08:00

492 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""双轨日终对账报表(影子柜台 vs 实盘模拟,设计 §8.2,影子 P3 前半)。
同一 db 文件(VPS ``backtest_results.db``)里 live_* 实盘侧与 paper_* 影子侧
同库共存,本模块按日对账四项指标:
| 对比项 | 一致标准 |
|--------|---------|
| 成交笔数 | 完全相同(rejected 影子单不计) |
| 每笔成交价差 | 平均 <10bps(按 symbol+side 聚合 vwap 对比) |
| 收盘持仓 | 逐只股票+数量相同 |
| 净值偏差 | 月累计 <0.5%(月内首基线 → 当日,双侧收益率差) |
差异即策略从纸面到真实的真实滑点成本——影子柜台的核心产出之一。
用法:
python -m sanguo_trader.shadow.reconcile_report --db <db> [--date YYYY-MM-DD]
API: GET /paper/reconcile(见 sanguo_api/routes_paper.py)
"""
from __future__ import annotations
import json
import logging
import sqlite3
from datetime import datetime
from typing import Any, Dict, List, Optional
logger = logging.getLogger(__name__)
# §8.2 一致标准
PRICE_DIFF_BPS_MAX = 10.0 # 每笔成交价差平均上限(bps)
NAV_MTD_PCT_MAX = 0.5 # 净值月累计偏差上限(%)
def _norm_symbol(symbol: str) -> str:
"""两侧符号口径不同(live=QMT '510300.SH',shadow=jq '510300.XSHG')→ 6 位码。"""
return str(symbol or "").split(".", 1)[0].strip()
def _norm_side(direction: str) -> str:
"""live 'buy'/'sell' vs shadow 'long'/'short' → B/S。"""
d = str(direction or "").lower()
if d in ("buy", "long", "", "b"):
return "B"
return "S"
def _month_start(date: str) -> str:
return f"{date[:7]}-01"
def _trade_rows(conn: sqlite3.Connection, table: str, account_id: int,
date: str) -> List[Dict[str, Any]]:
"""按日取成交行。live.traded_at 是 QMT 原样字符串(可能 '2026-08-15 09:35:00'
或紧凑格式),用两种 LIKE 兜;shadow 用 bar_date 精确匹配。"""
if table == "live_trades":
q = ("SELECT symbol, direction, price, volume, traded_at FROM live_trades "
"WHERE account_id=? AND (substr(traded_at,1,10)=? OR traded_at LIKE ?)")
rows = conn.execute(q, (account_id, date, f"{date.replace('-', '')}%")).fetchall()
return [{"symbol": r[0], "direction": r[1], "price": r[2], "volume": r[3]}
for r in rows]
q = ("SELECT symbol, direction, price, volume FROM paper_trades "
"WHERE account_id=? AND bar_date=? AND (rejected IS NULL OR rejected=0)")
rows = conn.execute(q, (account_id, date)).fetchall()
return [{"symbol": r[0], "direction": r[1], "price": r[2], "volume": r[3]}
for r in rows]
def _aggregate(trades: List[Dict[str, Any]]) -> Dict[str, Dict[str, float]]:
"""(norm_symbol, side) → {volume, notional} 聚合(vwap = notional/volume)。"""
agg: Dict[str, Dict[str, float]] = {}
for t in trades:
key = f"{_norm_symbol(t['symbol'])}:{_norm_side(t['direction'])}"
a = agg.setdefault(key, {"volume": 0.0, "notional": 0.0})
vol = float(t["volume"] or 0)
a["volume"] += vol
a["notional"] += vol * float(t["price"] or 0)
return agg
def _reconcile_trades(conn: sqlite3.Connection, live_id: int, shadow_id: int,
date: str) -> Dict[str, Any]:
live_rows = _trade_rows(conn, "live_trades", live_id, date)
shadow_rows = _trade_rows(conn, "paper_trades", shadow_id, date)
live = _aggregate(live_rows)
shadow = _aggregate(shadow_rows)
rows: List[Dict[str, Any]] = []
diffs: List[float] = []
for key in sorted(set(live) | set(shadow)):
symbol, side = key.rsplit(":", 1)
lv, sv = live.get(key), shadow.get(key)
lv_vwap = lv["notional"] / lv["volume"] if lv and lv["volume"] else None
sv_vwap = sv["notional"] / sv["volume"] if sv and sv["volume"] else None
bps = None
if lv_vwap and sv_vwap:
bps = (lv_vwap - sv_vwap) / sv_vwap * 1e4 # 带符号:稳定偏一侧→重标滑点
diffs.append(abs(bps))
rows.append({
"symbol": symbol, "side": side,
"live_volume": lv["volume"] if lv else 0,
"shadow_volume": sv["volume"] if sv else 0,
"live_vwap": round(lv_vwap, 4) if lv_vwap else None,
"shadow_vwap": round(sv_vwap, 4) if sv_vwap else None,
"price_diff_bps": round(bps, 2) if bps is not None else None,
})
avg_bps = sum(diffs) / len(diffs) if diffs else 0.0
# 笔数口径(2026-08-24 修订):count_match 按 票+方向 聚合桶总量比较——实盘
# QMT 部分成交会把一笔 3800 股拆 19 行(08-24 momentum 27 vs 5 恒 False,
# 纯计数噪音);原始行数仍如实呈现(live_count/shadow_count)供归因。
volume_match = all(r["live_volume"] == r["shadow_volume"] for r in rows)
live_count = len(live_rows)
shadow_count = len(shadow_rows)
return {
"live_count": live_count,
"shadow_count": shadow_count,
"count_match": volume_match,
"rows": rows,
"avg_price_diff_bps": round(avg_bps, 2),
"pass_price": avg_bps <= PRICE_DIFF_BPS_MAX,
}
def _reconcile_positions(conn: sqlite3.Connection, live_id: int,
shadow_id: int) -> Dict[str, Any]:
lv = {r[0]: r[1] for r in conn.execute(
"SELECT symbol, volume FROM live_positions WHERE account_id=?", (live_id,))}
sv = {r[0]: r[1] for r in conn.execute(
"SELECT symbol, volume FROM paper_positions "
"WHERE account_id=? AND scope='account' AND date=("
" SELECT MAX(date) FROM paper_positions WHERE account_id=? AND scope='account')",
(shadow_id, shadow_id))}
lv_n = {_norm_symbol(s): v for s, v in lv.items()}
sv_n = {_norm_symbol(s): v for s, v in sv.items()}
rows = []
match = True
for sym in sorted(set(lv_n) | set(sv_n)):
lvol = int(lv_n.get(sym) or 0)
svol = int(sv_n.get(sym) or 0)
if lvol != svol:
match = False
rows.append({"symbol": sym, "live_volume": lvol,
"shadow_volume": svol, "volume_diff": lvol - svol})
return {"rows": rows, "match": match}
def _reconcile_nav(conn: sqlite3.Connection, live_id: int, shadow_id: int,
date: str) -> Dict[str, Any]:
"""月累计净值偏差:月内(含月前最后一条)首基线 → 当日,双侧收益率差(%)。"""
def _series(table: str, col: str, aid: int) -> Dict[str, float]:
q = (f"SELECT date, {col} FROM {table} WHERE account_id=? "
f"AND date<=? ORDER BY date")
return {r[0]: float(r[1]) for r in conn.execute(q, (aid, date))}
live_s = _series("live_balance", "total", live_id)
shadow_s = _series("paper_daily_balance", "total_equity", shadow_id)
if not live_s or not shadow_s or date not in live_s or date not in shadow_s:
return {"live_total": live_s.get(date), "shadow_total": shadow_s.get(date),
"mtd_deviation_pct": None, "pass_nav": None, "note": "净值序列不全"}
ms = _month_start(date)
def _baseline(series: Dict[str, float]) -> Optional[float]:
prior = [v for d, v in series.items() if d < ms]
return prior[-1] if prior else next(
(v for d, v in sorted(series.items()) if d >= ms), None)
lb, sb = _baseline(live_s), _baseline(shadow_s)
if not lb or not sb:
return {"live_total": live_s[date], "shadow_total": shadow_s[date],
"mtd_deviation_pct": None, "pass_nav": None, "note": "无月内基线"}
live_ret = live_s[date] / lb - 1
shadow_ret = shadow_s[date] / sb - 1
dev = abs(live_ret - shadow_ret) * 100
return {
"live_total": live_s[date],
"shadow_total": shadow_s[date],
"live_mtd_return_pct": round(live_ret * 100, 4),
"shadow_mtd_return_pct": round(shadow_ret * 100, 4),
"mtd_deviation_pct": round(dev, 4),
"pass_nav": dev <= NAV_MTD_PCT_MAX,
}
def build_reconcile_report(db: str, live_account_id: int, shadow_account_id: int,
date: Optional[str] = None) -> Dict[str, Any]:
"""构建并返回某日双轨对账报告(纯读,不落库)。"""
date = date or datetime.now().strftime("%Y-%m-%d")
with sqlite3.connect(db) as conn:
trades = _reconcile_trades(conn, live_account_id, shadow_account_id, date)
positions = _reconcile_positions(conn, live_account_id, shadow_account_id)
nav = _reconcile_nav(conn, live_account_id, shadow_account_id, date)
passed = bool(
trades["count_match"] and trades["pass_price"]
and positions["match"] and nav["pass_nav"] is not False
)
return {
"date": date,
"live_account_id": live_account_id,
"shadow_account_id": shadow_account_id,
"trades": trades,
"positions": positions,
"nav": nav,
"passed": passed,
}
# ===== B5 恒等式对账(spec §multi-strategy-instance-budget §B5) =====
# 全账户 = Σ实例账本 + 未归因遗留仓;先过恒等式,再做逐对 live↔shadow 行为对比。
IDENTITY_TOL_PCT = 0.5 # 未归因占账户市值容忍上限(%;价格时点差)
def build_identity_report(db: str, date: Optional[str] = None) -> Dict[str, Any]:
"""账户恒等式报告:每 QMT 账号一行,snapshot 市值 vs Σ实例账本市值。
- Σ实例市值 = 同账号各实盘实例 live_balance 最新一条 market_value 之和
(dae56e2 起为实例账本视图;旧全账户行会如实呈现为大额负未归因)。
- 未归因仓 = 按 6 位码逐票对账:snapshot 持仓 Σ实例持仓(live_positions)。
- 无快照/无实例的账号如实标注(snapshot_missing / no_instances)。
"""
date = date or datetime.now().strftime("%Y-%m-%d")
rows: List[Dict[str, Any]] = []
with sqlite3.connect(db) as conn:
conn.row_factory = sqlite3.Row
snapshots = {r["account"]: dict(r) for r in conn.execute(
"SELECT * FROM qmt_account_snapshot")}
# 同 QMT 账号分组:实例最新账本 + 实例持仓视图
inst_rows = [dict(r) for r in conn.execute(
"SELECT id, name, account FROM live_accounts")]
by_acc: Dict[str, List[Dict[str, Any]]] = {}
for r in inst_rows:
by_acc.setdefault((r.get("account") or "").strip(), []).append(r)
accounts = sorted(set(snapshots) | {a for a in by_acc if a})
for acc in accounts:
snap = snapshots.get(acc)
insts = by_acc.get(acc, [])
mv_total = 0.0
for inst in insts:
last = conn.execute(
"SELECT market_value FROM live_balance "
"WHERE account_id=? ORDER BY id DESC LIMIT 1",
(inst["id"],)).fetchone()
inst["mv"] = float(last[0] or 0) if last else 0.0
mv_total += inst["mv"]
snap_mv = float(snap["market_value"]) if snap else None
unattr_mv = (snap_mv - mv_total) if snap_mv is not None else None
unattr_pct = (abs(unattr_mv) / snap_mv * 100
if snap_mv not in (None, 0) and unattr_mv is not None
else None)
# 逐票未归因:快照持仓 − Σ实例持仓(6 位码对齐)
snap_pos: Dict[str, float] = {}
if snap:
try:
for p in json.loads(snap.get("positions") or "[]"):
snap_pos[_norm_symbol(p.get("symbol"))] = float(
p.get("volume") or 0)
except (ValueError, TypeError):
snap_pos = {}
inst_pos: Dict[str, float] = {}
for inst in insts:
for r in conn.execute(
"SELECT symbol, volume FROM live_positions "
"WHERE account_id=?", (inst["id"],)):
inst_pos[_norm_symbol(r[0])] = inst_pos.get(
_norm_symbol(r[0]), 0.0) + float(r[1] or 0)
unattr_positions = [
{"symbol": s, "snapshot_volume": v,
"instance_volume": inst_pos.get(s, 0.0),
"diff": v - inst_pos.get(s, 0.0)}
for s, v in sorted(snap_pos.items())
if abs(v - inst_pos.get(s, 0.0)) > 0.5
]
if snap is None:
status = "snapshot_missing"
elif not insts:
status = "no_instances"
elif unattr_pct is not None and unattr_pct <= IDENTITY_TOL_PCT:
status = "pass"
else:
status = "unattributed_over_tol"
rows.append({
"account": acc, "date": date, "status": status,
"snapshot_mv": snap_mv, "instance_mv_total": mv_total,
"unattributed_mv": unattr_mv,
"unattributed_pct": round(unattr_pct, 4) if unattr_pct is not None else None,
"tolerance_pct": IDENTITY_TOL_PCT,
"instances": [{"id": i["id"], "name": i["name"], "mv": i["mv"]}
for i in insts],
"unattributed_positions": unattr_positions,
})
passed = all(r["status"] in ("pass", "no_instances") for r in rows) \
and bool(rows)
return {"date": date, "rows": rows, "identity_passed": passed}
_IDENTITY_SCHEMA = """
CREATE TABLE IF NOT EXISTS identity_reconcile (
id INTEGER PRIMARY KEY AUTOINCREMENT,
account TEXT,
date TEXT,
passed INTEGER,
report TEXT,
created_at TEXT,
UNIQUE(account, date)
)
"""
def save_identity_report(db: str, report: Dict[str, Any]) -> None:
"""恒等式报告落库(同账号同日覆盖)。"""
with sqlite3.connect(db) as conn:
conn.execute(_IDENTITY_SCHEMA)
for row in report["rows"]:
conn.execute(
"INSERT INTO identity_reconcile (account, date, passed, report, created_at) "
"VALUES (?,?,?,?,?) ON CONFLICT(account, date) DO UPDATE SET "
"passed=excluded.passed, report=excluded.report, "
"created_at=excluded.created_at",
(row["account"], report["date"],
1 if row["status"] in ("pass", "no_instances") else 0,
json.dumps(row, ensure_ascii=False),
datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
)
conn.commit()
def load_identity_report(db: str, date: str) -> List[Dict[str, Any]]:
"""读已存恒等式行(无 → 空表)。"""
with sqlite3.connect(db) as conn:
try:
rows = conn.execute(
"SELECT report FROM identity_reconcile WHERE date=? ORDER BY account",
(date,)).fetchall()
except sqlite3.OperationalError:
return []
return [json.loads(r[0]) for r in rows]
def find_dual_track_pairs(db: str) -> List[Dict[str, Any]]:
"""自动配对(2026-08-16 v2):优先 instance_id+周期精确配对,无 instance 回退策略名。
v1 按策略名建 dict 收敛——同策略多 live 账户(live#10/#11 都是 channel_test)
后者覆盖前者 → live#10 漏配、live#11 被配两次,15:10 日终对账配错对
(VPS 8对舰队实测)。同实例的 live↔shadow 才是真双轨。
"""
pairs: List[Dict[str, Any]] = []
with sqlite3.connect(db) as conn:
conn.row_factory = sqlite3.Row
lives = [dict(r) for r in conn.execute(
"SELECT id, strategy_class, instance_id, interval "
"FROM live_accounts WHERE status='running'")]
for r in conn.execute(
"SELECT id, strategies, instance_id, interval FROM paper_accounts "
"WHERE mode='shadow' AND status='running'"):
sh = dict(r)
try:
name = (json.loads(sh.get("strategies") or "[]") or [{}])[0].get("name", "")
except (json.JSONDecodeError, IndexError):
continue
cand = None
if sh.get("instance_id"):
# 同实例同周期 → 精确双轨;退而求其次同实例
cand = next((l for l in lives
if l.get("instance_id") == sh["instance_id"]
and l.get("interval") == sh.get("interval")), None)
if cand is None:
cand = next((l for l in lives
if l.get("instance_id") == sh["instance_id"]), None)
if cand is None:
same = [l for l in lives if l["strategy_class"] == name]
if len(same) == 1:
cand = same[0]
elif len(same) > 1:
# 同策略多 live 无 instance 可辨 → 周期一致才配,宁缺毋错
cand = next((l for l in same
if l.get("interval") == sh.get("interval")), None)
if cand:
pairs.append({"live_account_id": cand["id"],
"shadow_account_id": sh["id"], "strategy": name})
return pairs
_SCHEMA = """
CREATE TABLE IF NOT EXISTS dual_track_reconcile (
id INTEGER PRIMARY KEY AUTOINCREMENT,
live_account_id INTEGER,
shadow_account_id INTEGER,
date TEXT,
passed INTEGER,
report TEXT,
created_at TEXT,
UNIQUE(live_account_id, shadow_account_id, date)
)
"""
def save_reconcile_report(db: str, report: Dict[str, Any]) -> None:
"""落库(upsert 同配对同日覆盖)。"""
with sqlite3.connect(db) as conn:
conn.execute(_SCHEMA)
conn.execute(
"INSERT INTO dual_track_reconcile "
"(live_account_id, shadow_account_id, date, passed, report, created_at) "
"VALUES (?,?,?,?,?,?) "
"ON CONFLICT(live_account_id, shadow_account_id, date) "
"DO UPDATE SET passed=excluded.passed, report=excluded.report, "
"created_at=excluded.created_at",
(report["live_account_id"], report["shadow_account_id"], report["date"],
1 if report.get("passed") else 0,
json.dumps(report, ensure_ascii=False),
datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
)
conn.commit()
def load_reconcile_report(db: str, live_account_id: int, shadow_account_id: int,
date: str, *, as_row: bool = False) -> Any:
"""读已存报告;无 → None。as_row=True 返回表行(含 created_at)而非解析 JSON。"""
with sqlite3.connect(db) as conn:
conn.execute(_SCHEMA)
row = conn.execute(
"SELECT live_account_id, shadow_account_id, date, passed, report, "
"created_at FROM dual_track_reconcile WHERE live_account_id=? "
"AND shadow_account_id=? AND date=?",
(live_account_id, shadow_account_id, date),
).fetchone()
if row is None:
return None
if as_row:
keys = ("live_account_id", "shadow_account_id", "date", "passed",
"report", "created_at")
return [dict(zip(keys, row))]
return json.loads(row[4])
def main() -> None:
"""CLI:python -m sanguo_trader.shadow.reconcile_report --db <db> [--date ...]
不传配对 → find_dual_track_pairs 自动配对全部跑一遍并落库。
"""
import argparse
logging.basicConfig(level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
p = argparse.ArgumentParser(description="双轨日终对账报表")
p.add_argument("--db", required=True)
p.add_argument("--date", default=None)
p.add_argument("--live", type=int, default=None, help="显式配对:live 账户 id")
p.add_argument("--shadow", type=int, default=None, help="显式配对:影子账户 id")
args = p.parse_args()
if args.live and args.shadow:
pairs = [{"live_account_id": args.live, "shadow_account_id": args.shadow,
"strategy": "(explicit)"}]
else:
pairs = find_dual_track_pairs(args.db)
if not pairs:
logger.warning("未找到运行中的双轨配对(影子 mode=shadow ↔ live 同策略名)")
# B5 恒等式先行:全账户 = Σ实例账本 + 未归因,再做逐对行为对比
identity = build_identity_report(args.db, args.date)
save_identity_report(args.db, identity)
for row in identity["rows"]:
logger.info(
"[恒等式] %s %s: 快照市值=%.0f Σ实例=%.0f 未归因=%.0f(%s%%) "
"未归因票%d只 → %s",
identity["date"], row["account"],
row["snapshot_mv"] or 0, row["instance_mv_total"],
row["unattributed_mv"] or 0, row["unattributed_pct"],
len(row["unattributed_positions"]), row["status"],
)
for pair in pairs:
r = build_reconcile_report(args.db, pair["live_account_id"],
pair["shadow_account_id"], args.date)
save_reconcile_report(args.db, r)
t = r["trades"]
logger.info(
"[对账] %s live#%s vs shadow#%s(%s): 笔数 %s/%s=%s 价差%.1fbps=%s "
"持仓=%s 净值月偏差=%s%%%s",
r["date"], pair["live_account_id"], pair["shadow_account_id"],
pair["strategy"], t["live_count"], t["shadow_count"],
"" if t["count_match"] else "",
t["avg_price_diff_bps"], "" if t["pass_price"] else "超限",
"" if r["positions"]["match"] else "",
r["nav"]["mtd_deviation_pct"],
"PASS" if r["passed"] else "FAIL",
)
if __name__ == "__main__":
main()