claude_dev
236bd30eb8
- fundamentals_dmsk_download.py 单脚本(2009→今backfill+近3期daily NOTIC_DATE>=T-7),corpus骨架六修正全带入 - marker修正: daily按日折叠(§19.4原文「一次永逸」=day-2同款坑,52756e3教训);backfill键=域+期(域缺席连带跳过,首测抓出) - 次新期金丝雀防「filter失效vs空期」不分导致的整段静默空跑 - run_dmsk_standalone.sh(DSM 07:10 sanguo-fundamentals-daily,一次性容器同corpus款)+verify_dmsk.py(键唯一硬门) - 21测全mock绿,全量295绿;W1'镜像对账全绿+vb2026 H1缺口入档
58 lines
2.3 KiB
Python
58 lines
2.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""verify_dmsk.py — dmsk 补缺层只读验收器(spec §19.5 ①覆盖+键唯一硬门)。
|
|
|
|
用法: python3 verify_dmsk.py [--root /volume1/stock/fundamentals]
|
|
检查(退出码 0=PASS 1=FAIL):
|
|
1. 每域期目录数/part 数/行数/北交行数/最新 NOTICE_DATE;
|
|
2. 行键 (SECURITY_CODE,REPORT_DATE) 全局唯一(重键=FAIL, 同 corpus verify 纪律);
|
|
3. 退市股可查性(000003 三表至少一域有行 → 否则 WARNING 不 FAIL, 取决于回补深度);
|
|
4. 北交所含性(920/43/83/87/88 前缀行数>0 → 含; =0 → 单列说明不 FAIL)。
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
BJ_PREFIXES = ("43", "83", "87", "88", "92")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--root", default="/volume1/stock/fundamentals")
|
|
args = ap.parse_args()
|
|
root = Path(args.root) / "dmsk"
|
|
fails = []
|
|
report = {}
|
|
for domain in ("balance", "income", "cashflow"):
|
|
base = root / domain
|
|
parts = sorted(base.glob("dt=*/part-*.parquet")) if base.exists() else []
|
|
if not parts:
|
|
fails.append(f"{domain}: 零 part 文件")
|
|
continue
|
|
frames = [pd.read_parquet(p, columns=["SECURITY_CODE", "REPORT_DATE",
|
|
"NOTICE_DATE"]) for p in parts]
|
|
df = pd.concat(frames, ignore_index=True)
|
|
dups = int(df.duplicated(["SECURITY_CODE", "REPORT_DATE"]).sum())
|
|
n_bj = int(df["SECURITY_CODE"].astype(str).str[:2]
|
|
.isin(BJ_PREFIXES).sum())
|
|
delisted = int((df["SECURITY_CODE"].astype(str) == "000003").sum())
|
|
latest_notice = str(df["NOTICE_DATE"].astype(str).max())[:10]
|
|
report[domain] = dict(periods=len({p.parent.name for p in parts}),
|
|
parts=len(parts), rows=len(df), dup_keys=dups,
|
|
bj_rows=n_bj, delisted_000003_rows=delisted,
|
|
latest_notice_date=latest_notice)
|
|
if dups:
|
|
fails.append(f"{domain}: {dups} 重键行(键唯一硬门)")
|
|
print(json.dumps(report, ensure_ascii=False, indent=1))
|
|
if fails:
|
|
print("FAIL:", "; ".join(fails))
|
|
sys.exit(1)
|
|
print("PASS: 键唯一全过, 三域在库")
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|