feat(data): 财务136 W3' dmsk双lane落地+spec§19.8实施记录 [nas]

- fundamentals_dmsk_download.py 单脚本(2009→今backfill+近3期daily NOTIC_DATE>=T-7),corpus骨架六修正全带入 - marker修正: daily按日折叠(§19.4原文「一次永逸」=day-2同款坑,52756e3教训);backfill键=域+期(域缺席连带跳过,首测抓出) - 次新期金丝雀防「filter失效vs空期」不分导致的整段静默空跑 - run_dmsk_standalone.sh(DSM 07:10 sanguo-fundamentals-daily,一次性容器同corpus款)+verify_dmsk.py(键唯一硬门) - 21测全mock绿,全量295绿;W1'镜像对账全绿+vb2026 H1缺口入档
2026-09-08 00:29:23 +08:00
parent 2dcbdbe6fc
commit 236bd30eb8
5 changed files with 1178 additions and 0 deletions
@@ -0,0 +1,326 @@
+# -*- coding: utf-8 -*-
+"""TDD for fundamentals_dmsk_download.py — 财务三表 dmsk 补缺层(spec §19.4)。
+
+全 mock 零网络;日期全部相对 today(禁写死);DMSK_ROOT 重定向 tmp_path。
+契约来源: spec §19.2 11 发实测(全市场单期≈5223行/57列/NOTICE_DATE/含退市)。
+"""
+import datetime as dt
+import json
+import os
+from unittest.mock import MagicMock
+
+import pandas as pd
+import pytest
+
+from scripts.data_platform import fundamentals_dmsk_download as fdd
+
+
+# ---------- Fixtures ----------
+
+@pytest.fixture(autouse=True)
+def _fast(monkeypatch):
+    """限速归零 + 抖动归零: 测试不 sleep。"""
+    monkeypatch.setattr(fdd, "RATE", {k: 0.0 for k in fdd.RATE})
+    monkeypatch.setattr(fdd, "JITTER", (0.0, 0.0))
+
+
+@pytest.fixture
+def root(tmp_path, monkeypatch):
+    monkeypatch.setattr(fdd, "DMSK_ROOT", tmp_path)
+    for sub in ("dmsk", "state", "logs"):
+        (tmp_path / sub).mkdir(parents=True, exist_ok=True)
+    return tmp_path
+
+
+def _raw(code="600519", period="2024-12-31 00:00:00", notice="2025-04-03 00:00:00",
+         **over):
+    row = {
+        "SECURITY_CODE": code, "SECURITY_NAME_ABBR": "贵州茅台",
+        "REPORT_DATE": period, "NOTICE_DATE": notice,
+        "TOTAL_PARENT_EQUITY": 233105984399.47,
+    }
+    row.update(over)
+    return row
+
+
+class FakeResp:
+    def __init__(self, payload):
+        self.status_code = 200
+        self._payload = payload
+
+    def json(self):
+        return self._payload
+
+
+# ---------- 期枚举 ----------
+
+def test_quarter_end_dates_desc_and_bounds():
+    out = fdd.quarter_end_dates(2009, dt.date(2026, 9, 8))
+    assert out[0] == dt.date(2026, 6, 30)      # 未来期 09-30 不入列
+    assert out[1] == dt.date(2026, 3, 31)
+    assert out[-1] == dt.date(2009, 3, 31)
+    assert out == sorted(out, reverse=True)
+
+
+def test_quarter_end_dates_count():
+    out = fdd.quarter_end_dates(2009, dt.date(2026, 9, 8), count=3)
+    assert out == [dt.date(2026, 6, 30), dt.date(2026, 3, 31),
+                   dt.date(2025, 12, 31)]
+
+
+# ---------- 归一化 ----------
+
+def test_norm_row_truncates_and_zfills():
+    r = fdd.norm_row(_raw(code="600519"), "2026-09-08")
+    assert r["REPORT_DATE"] == "2024-12-31"
+    assert r["SECURITY_CODE"] == "600519"
+    assert r["fetch_date"] == "2026-09-08"
+    assert r["TOTAL_PARENT_EQUITY"] == 233105984399.47   # 其余列原样透传
+
+
+def test_norm_row_zfills_short_numeric_code():
+    assert fdd.norm_row(_raw(code="92001"), "d")["SECURITY_CODE"] == "092001"
+
+
+def test_is_bj_code():
+    assert fdd.is_bj_code("920001")
+    assert fdd.is_bj_code("833171")
+    assert fdd.is_bj_code("430047")
+    assert not fdd.is_bj_code("600519")
+    assert not fdd.is_bj_code("000001")
+    assert not fdd.is_bj_code("")
+    assert not fdd.is_bj_code(None)
+
+
+# ---------- fetch 层 ----------
+
+def test_fetch_period_rows_paginates_to_pages_end(root, monkeypatch):
+    calls = []
+
+    def fake_request(method, url, **kw):
+        calls.append(kw["params"])
+        page = int(kw["params"]["pageNum"])
+        return FakeResp({"result": {"pages": 3, "count": 6, "data": [
+            _raw(code=f"60000{page}")] * 2}, "success": True})
+
+    cl = fdd.DomainClient("datacenter")
+    monkeypatch.setattr(cl, "request", fake_request)
+    rows = fdd.fetch_period_rows(cl, "balance", "2024-12-31")
+    assert len(rows) == 6
+    assert len(calls) == 3
+    assert calls[0]["reportName"] == "RPT_DMSK_FN_BALANCE"
+    assert calls[0]["filter"] == "(REPORT_DATE='2024-12-31')"
+
+
+def test_fetch_period_rows_result_none_is_legal_empty(root):
+    cl = fdd.DomainClient("datacenter")
+    cl.session = MagicMock()
+    resp = FakeResp({"result": None, "success": False, "message": ""})
+    cl.session.request.return_value = resp
+    cl._pace = lambda: None
+    assert fdd.fetch_period_rows(cl, "balance", "2009-03-31") == []
+
+
+# ---------- backfill lane ----------
+
+def _patch_fetch(monkeypatch, responder):
+    calls = []
+
+    def fake(client, domain, report_date):
+        calls.append((domain, report_date))
+        return responder(domain, report_date)
+
+    monkeypatch.setattr(fdd, "fetch_period_rows", fake)
+    return calls
+
+
+def test_backfill_writes_parts_marks_and_resumes(root, monkeypatch):
+    _patch_fetch(monkeypatch, lambda d, p: [_raw(period=p + " 00:00:00")])
+    rc1 = fdd.run_lane("backfill", start_year=2024)
+    assert rc1 == 0
+    part = root / "dmsk" / "balance" / "dt=2024-12-31" / "part-0.parquet"
+    assert part.exists()
+    df = pd.read_parquet(part)
+    assert df["SECURITY_CODE"].iloc[0] == "600519"
+    # 全期 marker 落平铺(一次永逸, 键=域+期)
+    m = (root / "state" / "markers" / "backfill" / "period"
+         / "balance_2024-12-31.done")
+    assert m.exists()
+    # 重跑: 全 done → 零 fetch
+    calls2 = _patch_fetch(monkeypatch, lambda d, p: [_raw()])
+    rc2 = fdd.run_lane("backfill", start_year=2024)
+    assert rc2 == 0
+    assert calls2 == []
+
+
+def test_backfill_resumes_from_oldest_gap(root, monkeypatch):
+    _patch_fetch(monkeypatch, lambda d, p: [_raw(period=p + " 00:00:00")])
+    fdd.run_lane("backfill", start_year=2024)
+    (root / "state" / "markers" / "backfill" / "period"
+     / "balance_2025-12-31.done").unlink()       # 挖一个洞
+    calls = _patch_fetch(monkeypatch, lambda d, p: [_raw(period=p + " 00:00:00")])
+    fdd.run_lane("backfill", start_year=2024)
+    assert calls == [("balance", "2025-12-31")]
+
+
+def test_backfill_ledger_prevents_rekey_across_runs(root, monkeypatch):
+    _patch_fetch(monkeypatch, lambda d, p: [_raw(period=p + " 00:00:00")])
+    fdd.run_lane("backfill", start_year=2024)
+    # 清光 marker 重跑: 账本挡住 → 0 新行(键唯一硬门)
+    for p in (root / "state" / "markers" / "backfill").rglob("*.done"):
+        p.unlink()
+    fdd.run_lane("backfill", start_year=2024)
+    for part in (root / "dmsk").rglob("part-*.parquet"):
+        df = pd.read_parquet(part)
+        assert df.duplicated(["SECURITY_CODE", "REPORT_DATE"]).sum() == 0
+        assert len(df) == 1                      # 只有首跑那一行
+
+
+def test_in_batch_dedup_same_key_twice_in_response(root, monkeypatch):
+    # 同 (code,period) 两条同响应 → 账本外批内 seen 截留(corpus 六修正之三)
+    _patch_fetch(monkeypatch, lambda d, p: [
+        _raw(period=p + " 00:00:00"), _raw(period=p + " 00:00:00")])
+    fdd.run_lane("backfill", start_year=2025)
+    df = pd.read_parquet(root / "dmsk" / "balance"
+                         / "dt=2025-12-31" / "part-0.parquet")
+    assert len(df) == 1
+
+
+def test_backfill_canary_empty_fails_and_never_marks(root, monkeypatch):
+    # 次新期空=契约漂移疑云: failed+1, 不标 done 不落空库
+    def responder(d, p):
+        return [] if p == "2026-03-31" else [_raw(period=p + " 00:00:00")]
+    _patch_fetch(monkeypatch, responder)
+    rc = fdd.run_lane("backfill", start_year=2026)
+    assert rc == 1
+    assert not (root / "state" / "markers" / "backfill" / "period"
+                / "balance_2026-03-31.done").exists()
+
+
+def test_backfill_empty_early_period_is_legal_done(root, monkeypatch):
+    # 2009 年内非年报期早于首份数据: 合法空期照样标 done(金丝雀期有数据)
+    def responder(d, p):
+        if p.startswith("2009") and not p.endswith("12-31"):
+            return []
+        return [_raw(period=p + " 00:00:00")]
+    _patch_fetch(monkeypatch, responder)
+    rc = fdd.run_lane("backfill", start_year=2009)
+    assert rc == 0
+    assert (root / "state" / "markers" / "backfill" / "period"
+            / "balance_2009-03-31.done").exists()
+    assert not (root / "dmsk" / "balance" / "dt=2009-03-31").exists()
+
+
+def test_backfill_bj_inclusion_artifact(root, monkeypatch):
+    _patch_fetch(monkeypatch, lambda d, p: [
+        _raw(code="920001", period=p + " 00:00:00")])
+    fdd.run_lane("backfill", start_year=2025)
+    j = json.loads((root / "state" / "bj_inclusion.json").read_text("utf-8"))
+    assert j["balance"]["2025-12-31"] >= 1
+
+
+def test_limit_never_marks(root, monkeypatch):
+    _patch_fetch(monkeypatch, lambda d, p: [_raw(period=p + " 00:00:00")])
+    fdd.run_lane("backfill", limit=1, start_year=2024)
+    assert list((root / "state" / "markers" / "backfill").rglob("*.done")) == []
+
+
+# ---------- daily lane ----------
+
+def test_daily_notice_filter_and_period_dir(root, monkeypatch):
+    today = dt.date.today()
+    fresh = (today - dt.timedelta(days=1)).isoformat()
+    stale = (today - dt.timedelta(days=30)).isoformat()
+
+    def responder(d, p):
+        return [_raw(code="600519", period=p + " 00:00:00", notice=fresh),
+                _raw(code="000001", period=p + " 00:00:00", notice=stale)]
+    _patch_fetch(monkeypatch, responder)
+    rc = fdd.run_lane("daily")
+    assert rc == 0
+    # 落点=dt=<REPORT_DATE>/ 而非 dt=today; 近3期各得 1 行新披露
+    pdirs = sorted(p.name for p in (root / "dmsk" / "balance").glob("dt=*"))
+    assert len(pdirs) == 3
+    assert f"dt={today.isoformat()}" not in pdirs
+    df = pd.read_parquet(root / "dmsk" / "balance" / pdirs[0] / "part-0.parquet")
+    assert list(df["SECURITY_CODE"]) == ["600519"]     # NOTICE_DATE<T-7 被滤
+
+
+def test_daily_marker_date_scoped_next_day_refetches(root, monkeypatch):
+    """day-2 坑回归(corpus 52756e3 同款): 无日期维度会让次日整段跳过增量。"""
+    real_date = dt.date
+
+    class FakeDate(real_date):
+        @classmethod
+        def today(cls):
+            return real_date(2026, 9, 8)
+
+    monkeypatch.setattr(fdd.dt, "date", FakeDate)
+    _patch_fetch(monkeypatch, lambda d, p: [
+        _raw(code="600519", period=p + " 00:00:00",
+             notice="2026-09-07 00:00:00")])
+    fdd.run_lane("daily")
+    m1 = root / "state" / "markers" / "daily" / "period" / "dt=2026-09-08"
+    assert list(m1.glob("*.done"))
+
+    class FakeDate2(real_date):
+        @classmethod
+        def today(cls):
+            return real_date(2026, 9, 9)
+
+    monkeypatch.setattr(fdd.dt, "date", FakeDate2)
+    calls = _patch_fetch(monkeypatch, lambda d, p: [
+        _raw(code="600519", period=p + " 00:00:00",
+             notice="2026-09-08 00:00:00")])
+    fdd.run_lane("daily")
+    assert calls                                    # 次日真重扫,不被昨日 marker 跳过
+    m2 = root / "state" / "markers" / "daily" / "period" / "dt=2026-09-09"
+    assert list(m2.glob("*.done"))
+    assert not m1.exists()                          # 昨日 dt= 目录已被日初清理
+
+
+def test_daily_prunes_flat_legacy_markers(root, monkeypatch):
+    base = root / "state" / "markers" / "daily" / "period"
+    base.mkdir(parents=True)
+    (base / "balance_2026-06-30.done").write_text("legacy", encoding="utf-8")
+    stale = base / "dt=2026-09-07"
+    stale.mkdir()
+    (stale / "x.done").write_text("", encoding="utf-8")
+    fdd.prune_daily_markers()
+    assert not (base / "balance_2026-06-30.done").exists()
+    assert not stale.exists()
+
+
+# ---------- 账本对账 ----------
+
+def test_reconcile_ledger_from_parts(root):
+    d = root / "dmsk" / "balance" / "dt=2024-12-31"
+    d.mkdir(parents=True)
+    pd.DataFrame([{"SECURITY_CODE": "600519", "REPORT_DATE": "2024-12-31"}]
+                 ).to_parquet(d / "part-0.parquet", index=False)
+    led = fdd.IdLedger("balance")
+    added = fdd.reconcile_ledger_from_parts("balance", led)
+    assert added == 1
+    led.flush()
+    assert led.has_any(["600519|2024-12-31"]) == [True]
+
+
+# ---------- 锁/墙钟 ----------
+
+def test_lock_second_instance_yields(root):
+    fd1 = fdd.acquire_lock()
+    assert fd1 is not None
+    try:
+        assert fdd.acquire_lock() is None
+    finally:
+        os.close(fd1)
+    fd2 = fdd.acquire_lock()
+    assert fd2 is not None
+    os.close(fd2)
+
+
+def test_wallclock_checkpoint_rc(root):
+    ctx = fdd.Ctx("daily", until="00:00")           # 已过的墙钟
+    with pytest.raises(fdd.WallClockStop):
+        ctx.stop_now()
+    assert ctx.rc() == 3