848 lines
33 KiB
Python
848 lines
33 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""xcheck.py — 跨 vendor 对账域唯一入口(spec §20.12.4, 2026-09-22 用户拍板)。
|
||
|
||
独立子域 corpus/xcheck/ 只读对照、永不 merge 进权威层(独立性=永不合并)。
|
||
时机=写后即照(事件驱动主档;兜底漂移档 Phase 2b)。
|
||
|
||
--lane corpus_side 挂 corpus 班尾: ①ann_em 拉取(np-anotice T-1~T 全市场) +
|
||
公告披露日对照(ann_meta vs ann_em) ②腾讯 qt 全市场估值快照
|
||
落盘 + peTTM/pbMRQ 对照(vs valuation_baostock)
|
||
--lane fund_side 挂 dmsk 班尾(wash 完赛后): 读 wash manifests 当日命中股 →
|
||
新浪三表近12期拉取落盘 + 键完整性对照(vs static) +
|
||
NOTICE_DATE 对照(dmsk vs static, 本地零拉取)
|
||
--report [--date D] findings 汇总(人工查询)
|
||
--dry-run 只出报告零触发(L3 通道另有 L3_AUTO 开关, 默认关)
|
||
|
||
可追溯(spec §20.12.4): findings 每行内嵌对照时刻三方原值(values JSON);
|
||
trace_id=xc-YYYYMMDD-NNNN 贯穿 发现→仲裁→wash 重洗→复验; 根因分类学
|
||
root_cause∈{schema_drift,unit_scale,timezone,mapping_error,source_bug,
|
||
real_data_error,unknown}; 报告 tmp+rename 原子写; 读权威层只读。
|
||
|
||
退出码: 0=跑完(含发现 L2, 报警≠失败) 1=致命 3=锁让路。
|
||
"""
|
||
import argparse
|
||
import datetime as dt
|
||
import fcntl
|
||
import json
|
||
import logging
|
||
import os
|
||
import random
|
||
import re
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import pandas as pd
|
||
import requests
|
||
|
||
XCHECK_ROOT = Path(os.environ.get("CORPUS_ROOT", "/volume1/stock/corpus")) / "xcheck"
|
||
STATIC_ROOT = Path(os.environ.get("XCHECK_STATIC_ROOT",
|
||
"/volume1/stock/sanguo_vnpy_v2/data/static"))
|
||
DMSK_ROOT = Path(os.environ.get("XCHECK_DMSK_ROOT",
|
||
"/volume1/stock/fundamentals/dmsk"))
|
||
VB_ROOT = Path(os.environ.get("XCHECK_VB_ROOT",
|
||
"/volume1/stock/sanguo_vnpy_v2/data/valuation_baostock"))
|
||
WASH_MANIFEST_DIR = Path(os.environ.get(
|
||
"XCHECK_WASH_MANIFEST",
|
||
"/volume1/stock/sanguo_vnpy_v2/data_backup/static_overlay/manifests"))
|
||
|
||
NP_ANN_URL = ("https://np-anotice-stock.eastmoney.com/api/security/ann"
|
||
"?sr=-1&page_size={ps}&page_index={page}&ann_type=A"
|
||
"&client_source=web&f_node=0&s_node=0")
|
||
QT_URL = "https://qt.gtimg.cn/q="
|
||
SINA_FIN_URL = ("https://quotes.sina.cn/cn/api/openapi.php/"
|
||
"CompanyFinanceService.getFinanceReport2022"
|
||
"?paperCode={sym}&source={rtype}&type=0&page=1&num=12")
|
||
|
||
ANN_EM_MAX_PAGES = 60 # 全市场 T-1~T 窗 ~2-5k 条, 100/页封顶 60 页
|
||
QT_BATCH = 60 # qt 批量每股数(URL 长度约束)
|
||
QT_TIMEOUT = 12
|
||
SINA_TYPES = [("lrb", "income"), ("fzb", "balance"), ("llb", "cashflow")]
|
||
SINA_RECENT_PERIODS = 12 # 键完整性对照窗(与拉取 num=12 对齐)
|
||
|
||
RATE = {"em": 1.0, "qt": 0.5, "sina": 1.0}
|
||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
|
||
EM_HEADERS = {**UA, "Referer": "https://data.eastmoney.com/"}
|
||
SINA_HEADERS = {**UA, "Referer": "https://finance.sina.com.cn"}
|
||
|
||
VAL_TOL_NOTE = 0.001 # <0.1% 静默(L1 档下界)
|
||
VAL_TOL_WARN = 0.01 # >1% 报警
|
||
L3_AUTO = os.environ.get("XCHECK_L3_AUTO", "0") == "1" # 多数派自动仲裁, 默认关
|
||
|
||
# 新浪行项 item_field → EM 声明列映射(Phase 2b 2026-09-22 勘测定稿)。
|
||
# 变体按「值非空回退」解: 普通/银行(600036)/保险(601318)三 schema
|
||
# 数值级 EQ 锚定(PARENETP|NETPARECOMPPROF|NETPARESHARPROF 等);
|
||
# FE_INTEREST_EXPENSE=其中:利息费用(非财务费用全项, 财务费用=FINEXPE 勘误排除);
|
||
# SHORT_FIN_PAYABLE 为 EM 2019 准则合并前旧列(茅台 0/103 格力 null 全实证),
|
||
# 真数据在 NOTE_ACCOUNTS_PAYABLE(声明外附加行, 为 factor IBD 观察供料)。
|
||
SINA_FIELD_MAP = {
|
||
"income": {
|
||
"TOTAL_OPERATE_INCOME": ["BIZTOTINCO"],
|
||
"OPERATE_COST": ["BIZCOST"],
|
||
"PARENT_NETPROFIT": ["PARENETP", "NETPARECOMPPROF", "NETPARESHARPROF"],
|
||
"TOTAL_PROFIT": ["TOTPROFIT"],
|
||
"INVEST_INCOME": ["INVEINCO", "NETINVINCO"],
|
||
"FAIRVALUE_CHANGE_INCOME": ["VALUECHGLOSS"],
|
||
"ASSET_IMPAIRMENT_LOSS": ["ASSEIMPALOSSPROFIT", "ASSEIMPALOSS"],
|
||
"CREDIT_IMPAIRMENT_LOSS": ["CREDITIMPLOSSEPROFIT", "CREDITIMPLOSSE"],
|
||
"RESEARCH_EXPENSE": ["DEVEEXPE"],
|
||
"SALE_EXPENSE": ["SALESEXPE"],
|
||
"FE_INTEREST_EXPENSE": ["INTERESTEXPENSE"],
|
||
"INCOME_TAX": ["INCOTAXEXPE", "INCOTAX"],
|
||
"BASIC_EPS": ["BASICEPS"],
|
||
},
|
||
"balance": {
|
||
"TOTAL_ASSETS": ["TOTASSET"],
|
||
"TOTAL_PARENT_EQUITY": ["PARESHARRIGH"],
|
||
"ACCOUNTS_RECE": ["ACCORECE"],
|
||
"OTHER_RECE": ["OTHERRECE", "OTHERRECETOT"],
|
||
"GOODWILL": ["GOODWILL"],
|
||
"SHARE_CAPITAL": ["PAIDINCAPI"],
|
||
"SHORT_LOAN": ["SHORTTERMBORR"],
|
||
"SHORT_FIN_PAYABLE": ["NOTESPAYA"],
|
||
"NOTE_ACCOUNTS_PAYABLE": ["NOTESACCOPAYA"],
|
||
"NONCURRENT_LIAB_1YEAR": ["DUENONCLIAB"],
|
||
"LONG_LOAN": ["LONGBORR"],
|
||
"BOND_PAYABLE": ["BDSPAYA"],
|
||
"LEASE_LIAB": ["LEASELIAB"],
|
||
"INVENTORY": ["INVE"],
|
||
"MONETARYFUNDS": ["CURFDS"],
|
||
"ACCOUNTS_PAYABLE": ["ACCOPAYA"],
|
||
},
|
||
"cashflow": {
|
||
"NETCASH_OPERATE": ["MANANETR"],
|
||
"SALES_SERVICES": ["LABORGETCASH"],
|
||
"ACCEPT_INVEST_CASH": ["INVERETUGETCASH"],
|
||
"CONSTRUCT_LONG_ASSET": ["ACQUASSETCASH"],
|
||
"RECEIVE_LOAN_CASH": ["RECEFROMLOAN"],
|
||
"ISSUE_BOND": ["ISSBDRECECASH"],
|
||
"PAY_DEBT_CASH": ["DEBTPAYCASH"],
|
||
},
|
||
}
|
||
# 渠道不可对照(09-22 勘测): 扣非不在三表; DA 四件=间接法补充资料,
|
||
# openapi getFinanceReport2022 仅直编法主表(type/source 参数面已全扫)。
|
||
SINA_UNMAPPABLE = ("DEDUCT_PARENT_NETPROFIT", "FA_IR_DEPR", "IA_AMORTIZE",
|
||
"LPE_AMORTIZE", "USERIGHT_ASSET_AMORTIZE")
|
||
|
||
# 兜底漂移档(Phase 2b): 估值周一/三表月初窗, 全市场分布级+抽样聚合
|
||
DRIFT_STMT_WINDOW_DAYS = 7
|
||
DRIFT_SAMPLE_N = 30
|
||
DRIFT_L2_RATE = 0.05
|
||
|
||
ROOT_CAUSES = ("schema_drift", "unit_scale", "timezone", "mapping_error",
|
||
"source_bug", "real_data_error", "unknown")
|
||
|
||
FINDING_COLS = ["trace_id", "ts", "level", "check", "symbol", "report_date",
|
||
"column", "values", "deviation", "root_cause", "action", "status"]
|
||
|
||
log = logging.getLogger("xcheck")
|
||
|
||
|
||
# ---------- 薄节流客户端(与 corpus_download 同纪律; 不 import 生产入口防反向耦合) ----------
|
||
|
||
class Client:
|
||
def __init__(self, name):
|
||
self.name = name
|
||
self.session = requests.Session()
|
||
self.session.trust_env = False # 直连不走代理
|
||
self._last = 0.0
|
||
|
||
def get(self, url, **kw):
|
||
wait = self._last + RATE.get(self.name, 1.0) + random.uniform(0, 0.3) \
|
||
- time.monotonic()
|
||
if wait > 0:
|
||
time.sleep(wait)
|
||
self._last = time.monotonic()
|
||
r = self.session.get(url, timeout=QT_TIMEOUT, **kw)
|
||
r.raise_for_status()
|
||
return r
|
||
|
||
|
||
# ---------- trace / findings(可追溯核心) ----------
|
||
|
||
def _state_dir():
|
||
return XCHECK_ROOT / "state"
|
||
|
||
|
||
def next_trace_ids(day, n):
|
||
"""xc-YYYYMMDD-NNNN 递增(读计数器, 用后回写; 进程内预分 n 个防并发)。"""
|
||
st = _state_dir() / f"trace_counter_{day}.json"
|
||
st.parent.mkdir(parents=True, exist_ok=True)
|
||
cur = 0
|
||
if st.exists():
|
||
try:
|
||
cur = int(json.loads(st.read_text(encoding="utf-8"))["n"])
|
||
except (ValueError, KeyError):
|
||
cur = 0
|
||
ids = ["xc-%s-%04d" % (day, i + 1) for i in range(cur, cur + n)]
|
||
tmp = st.with_suffix(".tmp")
|
||
tmp.write_text(json.dumps({"n": cur + n}), encoding="utf-8")
|
||
os.replace(tmp, st)
|
||
return ids
|
||
|
||
|
||
def append_findings(findings):
|
||
"""追加当日 findings(结构化证据链, tmp+rename 原子)。返回落盘行数。"""
|
||
if not findings:
|
||
return 0
|
||
day = dt.date.today().isoformat()
|
||
d = XCHECK_ROOT / "reports" / f"dt={day}"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
existing = sorted(d.glob("part-*.parquet"))
|
||
idx = -1
|
||
for p in existing:
|
||
try:
|
||
idx = max(idx, int(p.stem.split("-")[1]))
|
||
except (ValueError, IndexError):
|
||
continue
|
||
df = pd.DataFrame(findings, columns=FINDING_COLS)
|
||
tmp = d / f".part-{idx + 1}.{os.getpid()}.tmp"
|
||
df.to_parquet(tmp, index=False)
|
||
os.replace(tmp, d / f"part-{idx + 1}.parquet")
|
||
return len(df)
|
||
|
||
|
||
def read_findings(day=None):
|
||
base = XCHECK_ROOT / "reports"
|
||
if day:
|
||
frames = [pd.read_parquet(p) for p in
|
||
sorted((base / f"dt={day}").glob("part-*.parquet"))]
|
||
else:
|
||
frames = [pd.read_parquet(p) for p in sorted(base.glob("dt=*/part-*.parquet"))]
|
||
return (pd.concat(frames, ignore_index=True) if frames
|
||
else pd.DataFrame(columns=FINDING_COLS))
|
||
|
||
|
||
# ---------- 对照纯函数(四层漏斗: 键→披露日→数值; 阈值分档+五级路由) ----------
|
||
|
||
def compare_notice_dates(symbol, static_rows, dmsk_rows):
|
||
"""层② PIT 披露日(两源: static vs dmsk)。同键期 NOTICE_DATE 差≥1天 → L2。"""
|
||
out = []
|
||
|
||
def _d(s):
|
||
return str(s)[:10] if s is not None and str(s) != "nan" else None
|
||
|
||
sm = {str(r["REPORT_DATE"])[:10]: _d(r.get("NOTICE_DATE"))
|
||
for r in static_rows}
|
||
dm = {str(r["REPORT_DATE"])[:10]: _d(r.get("NOTICE_DATE"))
|
||
for r in dmsk_rows}
|
||
for period in sorted(set(sm) & set(dm)):
|
||
a, b = sm[period], dm[period]
|
||
if a and b and a != b:
|
||
out.append({
|
||
"check": "notice_date", "symbol": symbol, "report_date": period,
|
||
"column": "NOTICE_DATE", "values": json.dumps(
|
||
{"static": a, "dmsk": b}, ensure_ascii=False),
|
||
"deviation": "1d+", "level": "L2",
|
||
"root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"})
|
||
return out
|
||
|
||
|
||
def compare_period_keys(symbol, sina_periods, static_periods):
|
||
"""层① 键完整性(新浪 vs static, 近12期窗)。单侧缺 → L2(可抓 wash 漏洗/新浪滞后)。
|
||
窗=并集最近 SINA_RECENT_PERIODS 期(static 全史老期不入窗,
|
||
修 09-22 冒烟发现的预跑 bug: 12 期对 103 期全史必刷 91 条噪声)。"""
|
||
out = []
|
||
sp, tp = set(sina_periods), set(static_periods)
|
||
window = set(sorted(sp | tp)[-SINA_RECENT_PERIODS:])
|
||
only_sina = sorted(sp & window - tp)
|
||
only_static = sorted(tp & window - sp)
|
||
if only_sina or only_static:
|
||
out.append({
|
||
"check": "period_keys", "symbol": symbol, "report_date": None,
|
||
"column": "REPORT_DATE", "values": json.dumps(
|
||
{"sina_only": only_sina, "static_only": only_static,
|
||
"window_periods": sorted(window)},
|
||
ensure_ascii=False),
|
||
"deviation": "sina_only=%d static_only=%d" % (len(only_sina),
|
||
len(only_static)),
|
||
"level": "L2", "root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"})
|
||
return out
|
||
|
||
|
||
def compare_valuation(symbol, qt_pe, qt_pb, vb_row):
|
||
"""层③ 数值(估值): qt 快照 peTTM/pbMRQ vs valuation_baostock 最近交易日行。
|
||
口径同为 TTM/MRQ, 慢变量日频对照; price/涨跌绝不对照(时点不同)。"""
|
||
out = []
|
||
for col, qt_v in (("peTTM", qt_pe), ("pbMRQ", qt_pb)):
|
||
vb_v = vb_row.get(col) if vb_row is not None else None
|
||
if not qt_v or vb_v in (None, 0, float("nan")):
|
||
continue
|
||
dev = abs(float(qt_v) - float(vb_v)) / abs(float(vb_v))
|
||
if dev <= VAL_TOL_NOTE:
|
||
continue # L0/L1 下界: 舍入级静默
|
||
level = "L2" if dev > VAL_TOL_WARN else "L1"
|
||
out.append({
|
||
"check": "valuation", "symbol": symbol, "report_date": None,
|
||
"column": col, "values": json.dumps(
|
||
{"qt": float(qt_v), "vb": float(vb_row[col]),
|
||
"vb_date": str(vb_row.get("date"))}, ensure_ascii=False),
|
||
"deviation": "%.4f" % dev, "level": level,
|
||
"root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"})
|
||
return out
|
||
|
||
|
||
def _num(v):
|
||
"""标量 → float|None(nan/空串/None 统一 None)。"""
|
||
try:
|
||
if v is None or pd.isna(v):
|
||
return None
|
||
return float(v)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _norm_sina_period(k):
|
||
"""新浪期键 20260630 → 2026-06-30(与 static str()[:10] 可交;
|
||
修复层①键对照空转 bug: 两格式永不相交, 09-22 勘测实锤)。"""
|
||
s = str(k)
|
||
if len(s) == 8 and s.isdigit():
|
||
return "%s-%s-%s" % (s[:4], s[4:6], s[6:8])
|
||
return s[:10]
|
||
|
||
|
||
def resolve_sina_value(vals, fields):
|
||
"""变体回退解: 按 fields 序取首个非空值(银行/保险 schema 行项名不同)。"""
|
||
for fld in fields:
|
||
v = _num((vals or {}).get(fld))
|
||
if v is not None:
|
||
return v
|
||
return None
|
||
|
||
|
||
def compare_statement_values(symbol, domain, sina_table, static_rows):
|
||
"""层③ 数值(三表): 新浪 vs static, 近12期交集 × SINA_FIELD_MAP 列。
|
||
阈值沿用估值域(<0.1% 静默/0.1~1% L1/>1% L2); static 缺列=银行/保险
|
||
schema 变体跳过; 单侧空或 static=0 → L1 coverage(攒 vendor 稀疏档案,
|
||
EM CREDIT_IMPAIRMENT_LOSS 对茅台整列死 0/103 属此类非数据丢失)。"""
|
||
out = []
|
||
cols = SINA_FIELD_MAP.get(domain, {})
|
||
st = {}
|
||
for r in static_rows:
|
||
st[str(r.get("REPORT_DATE"))[:10]] = r
|
||
for period in sorted(set(sina_table) & set(st))[-SINA_RECENT_PERIODS:]:
|
||
vals = sina_table[period]
|
||
row = st[period]
|
||
for em_col, fields in cols.items():
|
||
if em_col not in row:
|
||
continue # schema 变体(EM 银行/保险缺列)
|
||
em_v = _num(row.get(em_col))
|
||
sn_v = resolve_sina_value(vals, fields)
|
||
hit = next((f for f in fields if _num(vals.get(f)) is not None),
|
||
fields[0])
|
||
if em_v is None and sn_v is None:
|
||
continue
|
||
base = {"check": "statement_value", "symbol": symbol,
|
||
"report_date": period, "column": em_col,
|
||
"root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"}
|
||
if em_v is None or sn_v is None or em_v == 0:
|
||
out.append({**base, "level": "L1", "deviation": "one_side_null",
|
||
"values": json.dumps(
|
||
{"sina": sn_v, "static": em_v, "field": hit},
|
||
ensure_ascii=False)})
|
||
continue
|
||
dev = abs(em_v - sn_v) / abs(em_v)
|
||
if dev <= VAL_TOL_NOTE:
|
||
continue
|
||
out.append({**base,
|
||
"level": "L2" if dev > VAL_TOL_WARN else "L1",
|
||
"deviation": "%.4f" % dev,
|
||
"values": json.dumps(
|
||
{"sina": sn_v, "static": em_v, "field": hit},
|
||
ensure_ascii=False)})
|
||
return out
|
||
|
||
|
||
def route_l3(findings):
|
||
"""L3 多数派仲裁路由(三源数值域适用; 首期两源域不产生 L3)。
|
||
L3_AUTO=0 默认只报告; =1 时写 wash pending(复用 §20.11.7 通道)。"""
|
||
queued = [f for f in findings if f.get("level") == "L3"]
|
||
if not queued or not L3_AUTO:
|
||
return 0
|
||
pend = WASH_MANIFEST_DIR.parent / "pending.json"
|
||
try:
|
||
j = json.loads(pend.read_text(encoding="utf-8")) if pend.exists() else {}
|
||
except ValueError:
|
||
j = {}
|
||
units = j.setdefault("units", [])
|
||
for f in queued:
|
||
units.append({"symbol": f["symbol"], "domain": "income",
|
||
"trace_id": f["trace_id"], "reason": "xcheck_l3"})
|
||
tmp = pend.with_suffix(".tmp")
|
||
tmp.write_text(json.dumps(j, ensure_ascii=False), encoding="utf-8")
|
||
os.replace(tmp, pend)
|
||
for f in queued:
|
||
f["action"], f["status"] = "queued_wash", "queued_wash"
|
||
return len(queued)
|
||
|
||
|
||
# ---------- 拉取层(测试 mock 面) ----------
|
||
|
||
def fetch_ann_em(client, days=2):
|
||
"""np-anotice 全市场(新→旧), 拉到条目日期 < 今日-days+1 即止。"""
|
||
cutoff = (dt.date.today() - dt.timedelta(days=days - 1)).isoformat()
|
||
out = []
|
||
for page in range(1, ANN_EM_MAX_PAGES + 1):
|
||
r = client.get(NP_ANN_URL.format(ps=100, page=page), headers=EM_HEADERS)
|
||
rows = (r.json().get("data") or {}).get("list") or []
|
||
if not rows:
|
||
break
|
||
out.extend(rows)
|
||
oldest = min(str(it.get("notice_date") or "9999")[:10] for it in rows)
|
||
if oldest < cutoff:
|
||
break
|
||
return out
|
||
|
||
|
||
def _qt_symbol(code, exchange):
|
||
"""vb exchange(SSE/SZSE/BSE) → qt 前缀; BSE(北交) 首期跳过。"""
|
||
if exchange == "SSE":
|
||
return "sh" + code
|
||
if exchange == "SZSE":
|
||
return "sz" + code
|
||
return None
|
||
|
||
|
||
def parse_qt_payload(text):
|
||
"""qt 批量响应 → {code: {pe_ttm, pb}}; 字段位 idx39=PE(TTM) idx46=PB(§20.12.1)。"""
|
||
out = {}
|
||
for line in (text or "").split(";"):
|
||
if "=" not in line or '"' not in line:
|
||
continue
|
||
key = line.split("=")[0].strip().split("_")[-1]
|
||
parts = line.split('"')[1].split("~")
|
||
|
||
def _f(i):
|
||
try:
|
||
v = parts[i].strip()
|
||
return float(v) if v and v != "-" else None
|
||
except (ValueError, IndexError):
|
||
return None
|
||
out[key[2:]] = {"pe_ttm": _f(39) if len(parts) > 39 else None,
|
||
"pb": _f(46) if len(parts) > 46 else None}
|
||
return out
|
||
|
||
|
||
def fetch_qt_snapshot(client, universe):
|
||
"""universe=[(code, exchange)] → {code: {pe_ttm, pb}}。分批 QT_BATCH 只/请求。"""
|
||
out = {}
|
||
syms = [s for s in (_qt_symbol(c, e) for c, e in universe) if s]
|
||
for i in range(0, len(syms), QT_BATCH):
|
||
batch = syms[i:i + QT_BATCH]
|
||
try:
|
||
r = client.get(QT_URL + ",".join(batch), headers=UA)
|
||
out.update(parse_qt_payload(r.content.decode("gbk", errors="replace")))
|
||
except Exception as e:
|
||
log.warning("qt batch %d fail: %s", i // QT_BATCH, e)
|
||
return out
|
||
|
||
|
||
def fetch_sina_table(client, symbol_with_suffix, rtype):
|
||
"""新浪单表全行项: {归一期: {item_field: float|None}}。
|
||
期键归一(20260630→2026-06-30); page1/num12=SINA_RECENT_PERIODS。"""
|
||
code = symbol_with_suffix.split(".")[0]
|
||
prefix = "sh" if symbol_with_suffix.endswith(".SH") else "sz"
|
||
r = client.get(SINA_FIN_URL.format(sym=prefix + code, rtype=rtype),
|
||
headers=SINA_HEADERS)
|
||
rl = ((r.json().get("result") or {}).get("data") or {}).get("report_list") or {}
|
||
out = {}
|
||
for k, per in rl.items():
|
||
rows = (per or {}).get("data") or []
|
||
out[_norm_sina_period(k)] = {
|
||
row.get("item_field"): _num(row.get("item_value"))
|
||
for row in rows if row.get("item_field")}
|
||
return out
|
||
|
||
|
||
# ---------- 本地读(权威层只读) ----------
|
||
|
||
def _latest_vb_rows(year=None):
|
||
p = VB_ROOT / f"{year or dt.date.today().year}.parquet"
|
||
if not p.exists():
|
||
return pd.DataFrame()
|
||
df = pd.read_parquet(p)
|
||
return df.groupby("symbol").tail(1).set_index("symbol")
|
||
|
||
|
||
def _static_statement_rows(symbol_with_suffix, domain="income"):
|
||
"""static 三表行(只留 REPORT_DATE+映射列)。None=文件不存在。"""
|
||
p = STATIC_ROOT / domain / f"{symbol_with_suffix}_{domain}.parquet"
|
||
if not p.exists():
|
||
return None
|
||
df = pd.read_parquet(p)
|
||
keep = [c for c in df.columns if c == "REPORT_DATE"
|
||
or c in SINA_FIELD_MAP.get(domain, {})]
|
||
return df[keep].to_dict("records")
|
||
|
||
|
||
def _static_notice_rows(symbol_with_suffix, domain="income"):
|
||
p = STATIC_ROOT / domain / f"{symbol_with_suffix}_{domain}.parquet"
|
||
if not p.exists():
|
||
return []
|
||
df = pd.read_parquet(p, columns=["REPORT_DATE", "NOTICE_DATE"])
|
||
return df.sort_values("REPORT_DATE").tail(6).to_dict("records")
|
||
|
||
|
||
def _dmsk_notice_rows(code, domain="income"):
|
||
out = []
|
||
base = DMSK_ROOT / domain
|
||
if not base.exists():
|
||
return out
|
||
for d in sorted(base.glob("dt=*"))[-4:]: # 近 4 期分区
|
||
for pf in sorted(d.glob("*.parquet")):
|
||
try:
|
||
df = pd.read_parquet(
|
||
pf, columns=["SECURITY_CODE", "REPORT_DATE", "NOTICE_DATE"],
|
||
filters=[("SECURITY_CODE", "=", code)])
|
||
except Exception:
|
||
continue
|
||
out.extend(df.to_dict("records"))
|
||
return out
|
||
|
||
|
||
def _ann_meta_latest_by_code(days=2):
|
||
"""ann_meta 近 days 日分区 → {sec_code: 最新 ann_time[:10]}。"""
|
||
base = XCHECK_ROOT.parent / "ann_meta"
|
||
codes = {}
|
||
if not base.exists():
|
||
return codes
|
||
cutoff = (dt.date.today() - dt.timedelta(days=days)).isoformat()
|
||
for d in base.glob("dt=*"):
|
||
if d.name[3:] < cutoff:
|
||
continue
|
||
for pf in sorted(d.glob("part-*.parquet")):
|
||
try:
|
||
df = pd.read_parquet(
|
||
pf, columns=["sec_code", "ann_time"]).dropna(subset=["sec_code"])
|
||
except Exception:
|
||
continue
|
||
for r in df.itertuples():
|
||
day = str(r.ann_time)[:10]
|
||
if day >= codes.get(r.sec_code, ""):
|
||
codes[r.sec_code] = day
|
||
return codes
|
||
|
||
|
||
def _wash_hit_symbols():
|
||
"""wash manifests 未归档清单(=当日命中) → [(symbol_with_suffix, domain)]。"""
|
||
hits = []
|
||
if not WASH_MANIFEST_DIR.exists():
|
||
return hits
|
||
for lst in sorted(WASH_MANIFEST_DIR.glob("*.list")):
|
||
for line in lst.read_text(encoding="utf-8").splitlines():
|
||
line = line.strip()
|
||
m = re.match(r"^(\w+)/(\d{6}\.(?:SH|SZ))_\w+\.parquet$", line)
|
||
if m:
|
||
hits.append((m.group(2), m.group(1)))
|
||
return hits
|
||
|
||
|
||
# ---------- lane ----------
|
||
|
||
def run_corpus_side():
|
||
findings = []
|
||
day = dt.date.today().isoformat()
|
||
# ① ann_em 拉取+披露日对照
|
||
cl_em = Client("em")
|
||
try:
|
||
rows = fetch_ann_em(cl_em)
|
||
d = XCHECK_ROOT / "ann_em" / f"dt={day}"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
if rows:
|
||
pd.DataFrame(rows).to_parquet(d / "part-0.parquet", index=False)
|
||
em_latest = {}
|
||
for it in rows:
|
||
for c in it.get("codes") or []:
|
||
code = str(c.get("stock_code") or "")
|
||
nd = str(it.get("notice_date") or "")[:10]
|
||
if code and nd >= em_latest.get(code, ""):
|
||
em_latest[code] = nd
|
||
am_latest = _ann_meta_latest_by_code()
|
||
common = set(em_latest) & set(am_latest)
|
||
for code in sorted(common):
|
||
if em_latest[code] != am_latest[code]:
|
||
findings.append({
|
||
"check": "ann_date", "symbol": code, "report_date": None,
|
||
"column": "latest_notice", "values": json.dumps(
|
||
{"ann_meta": am_latest[code], "ann_em": em_latest[code]},
|
||
ensure_ascii=False),
|
||
"deviation": "date_diff", "level": "L2",
|
||
"root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"})
|
||
log.info("ann_em: fetched=%d em_codes=%d am_codes=%d common=%d findings=%d",
|
||
len(rows), len(em_latest), len(am_latest), len(common),
|
||
sum(1 for f in findings if f["check"] == "ann_date"))
|
||
except Exception as e:
|
||
log.warning("ann_em fail: %s", e)
|
||
# ② qt 快照+估值对照
|
||
try:
|
||
vb = _latest_vb_rows()
|
||
universe = [(str(s), str(e)) for s, e in
|
||
zip(vb.index.tolist(), vb["exchange"].tolist())]
|
||
cl_qt = Client("qt")
|
||
snap = fetch_qt_snapshot(cl_qt, universe)
|
||
d = XCHECK_ROOT / "quote_tencent" / f"dt={day}"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
if snap:
|
||
pd.DataFrame(
|
||
[{"symbol": k, **v} for k, v in snap.items()]
|
||
).to_parquet(d / "part-0.parquet", index=False)
|
||
n_val = 0
|
||
for code, q in snap.items():
|
||
if code not in vb.index:
|
||
continue
|
||
fs = compare_valuation(code, q.get("pe_ttm"), q.get("pb"),
|
||
vb.loc[code].to_dict())
|
||
findings.extend(fs)
|
||
n_val += len(fs)
|
||
log.info("qt: universe=%d snap=%d val_findings=%d", len(universe),
|
||
len(snap), n_val)
|
||
# ③ 兜底漂移档·估值(周一, 复用同班 snap/vb 零额外请求)
|
||
if _drift_gate_valuation(dt.date.today()):
|
||
try:
|
||
findings.extend(drift_valuation(snap, vb))
|
||
except Exception as e:
|
||
log.warning("valuation drift fail: %s", e)
|
||
except Exception as e:
|
||
log.warning("qt side fail: %s", e)
|
||
return findings
|
||
|
||
|
||
def _save_sina_raw(symbol, tables):
|
||
"""新浪全行项原料 dt 分区落盘(证据快照, 回放不依赖重拉)。"""
|
||
day = dt.date.today().isoformat()
|
||
d = XCHECK_ROOT / "sina_fin" / f"dt={day}"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
for domain, table in tables.items():
|
||
rows = [{"report_period": p, "item_field": f, "item_value": v}
|
||
for p, vals in table.items() for f, v in vals.items()]
|
||
if rows:
|
||
pd.DataFrame(rows).to_parquet(
|
||
d / f"{symbol}_{domain}.parquet", index=False)
|
||
|
||
|
||
def _drift_gate_valuation(today):
|
||
"""兜底档·估值: 周一(XCHECK_DRIFT=0 可整体关)。"""
|
||
return (os.environ.get("XCHECK_DRIFT", "1") == "1"
|
||
and today.isoweekday() == 1)
|
||
|
||
|
||
def _drift_gate_statement(today):
|
||
"""兜底档·三表: 月初 DRIFT_STMT_WINDOW_DAYS 天窗(抓休市错位)。"""
|
||
return (os.environ.get("XCHECK_DRIFT", "1") == "1"
|
||
and today.day <= DRIFT_STMT_WINDOW_DAYS)
|
||
|
||
|
||
def drift_valuation(snap, vb):
|
||
"""兜底: qt vs vb 全市场分位数漂移(分布级, 抓口径漂移/单点对照盲区)。
|
||
中位数偏差>1% → 单条 __MARKET__ L2(不逐股刷屏)。"""
|
||
out = []
|
||
for col, kk in (("peTTM", "pe_ttm"), ("pbMRQ", "pb")):
|
||
a, b = [], []
|
||
for c, q in snap.items():
|
||
if c not in vb.index:
|
||
continue
|
||
x, y = _num(q.get(kk)), _num(vb.loc[c, col])
|
||
if x is None or y is None or x == 0 or y == 0:
|
||
continue
|
||
a.append(x)
|
||
b.append(y)
|
||
if len(b) < 100:
|
||
continue
|
||
ma, mb = pd.Series(a).median(), pd.Series(b).median()
|
||
dev = abs(ma - mb) / abs(mb)
|
||
if dev <= VAL_TOL_WARN:
|
||
continue
|
||
out.append({
|
||
"check": "drift_valuation", "symbol": "__MARKET__",
|
||
"report_date": None, "column": col, "values": json.dumps(
|
||
{"qt_median": float(ma), "vb_median": float(mb),
|
||
"n": len(b)}, ensure_ascii=False),
|
||
"deviation": "%.4f" % dev, "level": "L2",
|
||
"root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"})
|
||
return out
|
||
|
||
|
||
def _sample_static_symbols(n):
|
||
"""static income 树抽样(按日 seed 确定性, 同日可复现)。"""
|
||
base = STATIC_ROOT / "income"
|
||
if not base.exists():
|
||
return []
|
||
syms = sorted(p.name.split("_")[0] for p in base.glob("*.parquet"))
|
||
if not syms:
|
||
return []
|
||
rng = random.Random("drift-" + dt.date.today().isoformat())
|
||
return rng.sample(syms, min(n, len(syms)))
|
||
|
||
|
||
def drift_statement_aggregate(stats):
|
||
"""月初兜底聚合: 抽样 L2 率>DRIFT_L2_RATE → 单条 __SAMPLE__ L2。"""
|
||
checked = stats.get("checked") or 0
|
||
if not checked:
|
||
return []
|
||
rate = stats["l2"] / checked
|
||
if rate <= DRIFT_L2_RATE:
|
||
return []
|
||
return [{
|
||
"check": "drift_statement", "symbol": "__SAMPLE__",
|
||
"report_date": None, "column": "l2_rate", "values": json.dumps(
|
||
{"checked": checked, "l2": stats["l2"],
|
||
"offenders": stats.get("offenders", [])[:5]},
|
||
ensure_ascii=False),
|
||
"deviation": "%.3f" % rate, "level": "L2",
|
||
"root_cause": "unknown", "action": "reported",
|
||
"status": "discovered"}]
|
||
|
||
|
||
def run_statement_drift(client):
|
||
"""三表月初兜底: 抽样 DRIFT_SAMPLE_N 股 × 三表数值对照, 聚合不逐股刷屏。"""
|
||
stats = {"checked": 0, "l2": 0, "offenders": []}
|
||
for symbol in _sample_static_symbols(DRIFT_SAMPLE_N):
|
||
for rtype, domain in SINA_TYPES:
|
||
try:
|
||
table = fetch_sina_table(client, symbol, rtype)
|
||
rows = _static_statement_rows(symbol, domain)
|
||
if not rows:
|
||
continue
|
||
fs = compare_statement_values(symbol, domain, table, rows)
|
||
n_l2 = sum(1 for f in fs if f["level"] == "L2")
|
||
stats["checked"] += 1
|
||
if n_l2:
|
||
stats["l2"] += 1
|
||
stats["offenders"].append(
|
||
{"symbol": symbol, "domain": domain, "n_l2": n_l2})
|
||
except Exception as e:
|
||
log.warning("drift %s %s fail: %s", symbol, domain, e)
|
||
log.info("statement drift: checked=%d l2=%d", stats["checked"], stats["l2"])
|
||
return drift_statement_aggregate(stats)
|
||
|
||
|
||
def run_fund_side():
|
||
findings = []
|
||
hits = _wash_hit_symbols()
|
||
log.info("fund_side: wash hits=%d (manifests 未归档=当日)", len(hits))
|
||
if not hits:
|
||
return findings
|
||
cl_sina = Client("sina")
|
||
tables_cache = {}
|
||
for symbol, domain in hits:
|
||
# ① 新浪三表拉取(全行项原料落盘) + 键完整性 + 层③数值对照
|
||
try:
|
||
if symbol not in tables_cache:
|
||
tables_cache[symbol] = {
|
||
dom: fetch_sina_table(cl_sina, symbol, rtype)
|
||
for rtype, dom in SINA_TYPES}
|
||
_save_sina_raw(symbol, tables_cache[symbol])
|
||
table = tables_cache[symbol].get(domain) or {}
|
||
rows = _static_statement_rows(symbol, domain)
|
||
static_periods = (sorted({str(r["REPORT_DATE"])[:10] for r in rows})
|
||
if rows else [])
|
||
findings.extend(compare_period_keys(
|
||
symbol, sorted(table), static_periods))
|
||
if rows:
|
||
findings.extend(compare_statement_values(
|
||
symbol, domain, table, rows))
|
||
except Exception as e:
|
||
log.warning("sina %s fail: %s", symbol, e)
|
||
# ② NOTICE_DATE 对照(dmsk vs static, 本地零拉取)
|
||
code = symbol.split(".")[0]
|
||
srows = _static_notice_rows(symbol, domain)
|
||
drows = _dmsk_notice_rows(code, domain)
|
||
findings.extend(compare_notice_dates(symbol, srows, drows))
|
||
# ③ 兜底漂移档·三表(月初窗, 全市场抽样非 wash 命中面)
|
||
if _drift_gate_statement(dt.date.today()):
|
||
try:
|
||
findings.extend(run_statement_drift(cl_sina))
|
||
except Exception as e:
|
||
log.warning("statement drift fail: %s", e)
|
||
return findings
|
||
|
||
|
||
def _stamp(findings):
|
||
day = dt.date.today().strftime("%Y%m%d")
|
||
n = len(findings)
|
||
ids = next_trace_ids(day, n) if n else []
|
||
ts = dt.datetime.now().isoformat(timespec="seconds")
|
||
for f, tid in zip(findings, ids):
|
||
f.setdefault("trace_id", tid)
|
||
f.setdefault("ts", ts)
|
||
f.setdefault("root_cause", "unknown")
|
||
f.setdefault("action", "reported")
|
||
f.setdefault("status", "discovered")
|
||
return findings
|
||
|
||
|
||
def run_report(day=None):
|
||
df = read_findings(day or dt.date.today().isoformat())
|
||
if df.empty:
|
||
print("findings: 0")
|
||
return
|
||
print("findings: %d" % len(df))
|
||
print(df.groupby(["level", "check"]).size().to_string())
|
||
print("--- L2 明细(最新 20) ---")
|
||
cols = ["trace_id", "check", "symbol", "report_date", "column",
|
||
"deviation", "values"]
|
||
print(df[df["level"] == "L2"][cols].tail(20).to_string(index=False))
|
||
|
||
|
||
def _ensure_log():
|
||
d = XCHECK_ROOT / "logs"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s %(levelname)s %(message)s",
|
||
handlers=[logging.FileHandler(
|
||
d / f"xcheck_{dt.date.today().isoformat()}.log", encoding="utf-8")])
|
||
|
||
|
||
def acquire_lock():
|
||
p = XCHECK_ROOT / "state" / "xcheck.lock"
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
fd = os.open(p, os.O_CREAT | os.O_RDWR, 0o644)
|
||
try:
|
||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except OSError:
|
||
os.close(fd)
|
||
return None
|
||
return fd
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser(description="跨 vendor 对账域(spec §20.12.4)")
|
||
ap.add_argument("--lane", choices=["corpus_side", "fund_side"])
|
||
ap.add_argument("--report", action="store_true")
|
||
ap.add_argument("--date", default=None)
|
||
ap.add_argument("--dry-run", action="store_true",
|
||
help="只出报告零触发(L3 另有 XCHECK_L3_AUTO 开关)")
|
||
args = ap.parse_args()
|
||
_ensure_log()
|
||
if args.report:
|
||
run_report(args.date)
|
||
return 0
|
||
fd = acquire_lock()
|
||
if fd is None:
|
||
print("另一实例持有锁, 让路退出", flush=True)
|
||
return 3
|
||
try:
|
||
findings = (run_corpus_side() if args.lane == "corpus_side"
|
||
else run_fund_side())
|
||
_stamp(findings)
|
||
if not args.dry_run:
|
||
route_l3(findings)
|
||
n = append_findings(findings)
|
||
log.info("lane=%s done: findings=%d appended=%d dry_run=%s l3_auto=%s",
|
||
args.lane, len(findings), n, args.dry_run, L3_AUTO)
|
||
print("findings=%d appended=%d" % (len(findings), n), flush=True)
|
||
return 0
|
||
finally:
|
||
os.close(fd)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|