claude_dev
f4404c17b5
backfill 210unit/1771发/零失败,verify键唯一PASS,BJ 1万+行/域,净新增383只镜像外代码;边界=2009前退市双源皆无(源头缺失)
61 lines
2.5 KiB
Python
61 lines
2.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""verify_dmsk.py — dmsk 补缺层只读验收器(spec §19.5 ①覆盖+键唯一硬门)。
|
|
|
|
用法: python3 verify_dmsk.py [--root /volume1/stock/fundamentals]
|
|
检查(退出码 0=PASS 1=FAIL):
|
|
1. 每域期目录数/part 数/行数/北交行数/最新 NOTICE_DATE;
|
|
2. 行键 (SECURITY_CODE,REPORT_DATE) 全局唯一(重键=FAIL, 同 corpus verify 纪律);
|
|
3. 退市股可查性(000003 三表至少一域有行 → 否则 WARNING 不 FAIL, 取决于回补深度);
|
|
4. 北交所含性(920/43/83/87/88 前缀行数>0 → 含; =0 → 单列说明不 FAIL)。
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
BJ_PREFIXES = ("43", "83", "87", "88", "92")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--root", default="/volume1/stock/fundamentals")
|
|
args = ap.parse_args()
|
|
root = Path(args.root) / "dmsk"
|
|
fails = []
|
|
report = {}
|
|
for domain in ("balance", "income", "cashflow"):
|
|
base = root / domain
|
|
parts = sorted(base.glob("dt=*/part-*.parquet")) if base.exists() else []
|
|
if not parts:
|
|
fails.append(f"{domain}: 零 part 文件")
|
|
continue
|
|
frames = [pd.read_parquet(p, columns=["SECURITY_CODE", "REPORT_DATE",
|
|
"NOTICE_DATE"]) for p in parts]
|
|
df = pd.concat(frames, ignore_index=True)
|
|
dups = int(df.duplicated(["SECURITY_CODE", "REPORT_DATE"]).sum())
|
|
n_bj = int(df["SECURITY_CODE"].astype(str).str[:2]
|
|
.isin(BJ_PREFIXES).sum())
|
|
delisted = int((df["SECURITY_CODE"].astype(str) == "000003").sum())
|
|
notice = df["NOTICE_DATE"].dropna()
|
|
# 注意 astype(str).max() 遇 NaN 会得 "None"(ASCII 大于日期串)——先 dropna
|
|
latest_notice = str(notice.max())[:10] if len(notice) else "n/a"
|
|
report[domain] = dict(periods=len({p.parent.name for p in parts}),
|
|
parts=len(parts), rows=len(df), dup_keys=dups,
|
|
bj_rows=n_bj, delisted_000003_rows=delisted,
|
|
notice_nonnull=int(notice.count()),
|
|
latest_notice_date=latest_notice)
|
|
if dups:
|
|
fails.append(f"{domain}: {dups} 重键行(键唯一硬门)")
|
|
print(json.dumps(report, ensure_ascii=False, indent=1))
|
|
if fails:
|
|
print("FAIL:", "; ".join(fails))
|
|
sys.exit(1)
|
|
print("PASS: 键唯一全过, 三域在库")
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|