feat(data): T5 backfill ③全文 ④PDF+容量闸门 [nas]

spec §18.7 T5(backfill lane 补完,四段齐): - ③新闻全文回补: 扫 news_meta 库存→未入账 art 补正文;账本即进度(免 260万 marker 文件);404=墓碑行 content_text=None 入账不重试;文章域独立限速 2s(RATE['article'],spec @2/s 慢车道) - ④PDF六类回补(新→旧): ann_meta 库存标题谓词筛六类→ann_time desc; 文件在=跳过(文件即进度);404 墓碑 state/pdf_missing.json;索引复用 _append_pdf_index(daily/backfill 同管道) - 容量闸门: 段首+每50GB 自查 _disk_free_gb,<200G 暂停 PDF 段+落 capacity_paused.json 告警(其余段不受影响,空间回升自动续) - 测试51个(root fixture 默认 _disk_free_gb=645:曾吃 Mac 真实磁盘 8G 余量误触发闸门);全套 268 绿
2026-09-06 23:24:29 +08:00
parent 10c486c64d
commit 1c3a61eb12
2 changed files with 375 additions and 19 deletions
@@ -27,6 +27,8 @@ def _fast(monkeypatch):
@pytest.fixture
 def root(tmp_path, monkeypatch):
    monkeypatch.setattr(cd, "CORPUS_ROOT", tmp_path)
+    # 容量闸门默认放行: 本机磁盘余量与测试无关(曾因 Mac 真实 free<200G 误暂停 ④)
+    monkeypatch.setattr(cd, "_disk_free_gb", lambda path: 645.0)
    for sub in ("news_meta", "news_fulltext", "ann_meta", "ann_pdf", "state", "logs"):
        (tmp_path / sub).mkdir(parents=True, exist_ok=True)
    return tmp_path
@@ -609,10 +611,13 @@ def test_backfill_ann_unit_fail_not_marked(root, monkeypatch):


 def _wire_news_backfill(monkeypatch, pages):
-    """pages: {page_idx: rows} — 翻页回放。"""
+    """pages: {page_idx: rows} — 翻页回放。③④ 段 fetch 一并锁死(零网络铁律)。"""
    monkeypatch.setattr(cd, "load_stock_pool", lambda **kw: [("600519", "o1")])
    monkeypatch.setattr(cd, "fetch_cninfo_announcements",
                        MagicMock(return_value=[]))
+    monkeypatch.setattr(cd, "fetch_article_html",
+                        MagicMock(return_value="<body>x</body>"))
+    monkeypatch.setattr(cd, "download_pdf", MagicMock(return_value=12))
    urls = []

    def fake(client, code, mkt, page):
@@ -683,3 +688,176 @@ def test_backfill_wallclock_rc3(root, monkeypatch):
    rc = cd.run_lane("backfill", until=past)
    assert rc == 3
    assert cd.is_done("backfill", "ann", "000001_2000")  # 首 unit 完成后停
+
+
+# ---------- backfill lane ③news_fulltext ④PDF+容量闸门(T5) ----------
+
+def _seed_news_meta(root, rows):
+    """往今天的 news_meta 分区落行(模拟 ①② 已产出的库存)。"""
+    import pandas as pd
+    day = dt.date.today().isoformat()
+    d = root / "news_meta" / f"dt={day}"
+    d.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(rows).to_parquet(d / "part-0.parquet", index=False)
+
+
+_NEWS_SEED = [
+    {"art_code": "f1", "stock_code": "600519", "show_time": "2024-01-01 00:00:00",
+     "title": "t1", "summary": None, "media_name": None,
+     "url": "http://finance.eastmoney.com/a/f1.html", "first_seen_date": "2026-09-06"},
+    {"art_code": "f2", "stock_code": "600519", "show_time": "2024-01-02 00:00:00",
+     "title": "t2", "summary": None, "media_name": None,
+     "url": "http://finance.eastmoney.com/a/f2.html", "first_seen_date": "2026-09-06"},
+]
+
+
+def _wire_backfill_fulltext(monkeypatch, html_by_url):
+    monkeypatch.setattr(cd, "load_stock_pool", lambda **kw: POOL)
+    monkeypatch.setattr(cd, "fetch_cninfo_announcements",
+                        MagicMock(return_value=[]))
+    monkeypatch.setattr(cd, "fetch_news_backfill_page",
+                        MagicMock(return_value=[]))
+    fetched = []
+
+    def fake(client, url):
+        fetched.append(url)
+        v = html_by_url.get(url, "<html><body>ok</body></html>")
+        if isinstance(v, Exception):
+            raise v
+        return v
+
+    monkeypatch.setattr(cd, "fetch_article_html", fake)
+    return fetched
+
+
+def test_fulltext_backfill_fills_from_news_meta(root, monkeypatch):
+    import pandas as pd
+    _seed_news_meta(root, _NEWS_SEED)
+    fetched = _wire_backfill_fulltext(monkeypatch, {})
+    rc = cd.run_lane("backfill")
+    assert rc == 0
+    assert sorted(fetched) == ["http://finance.eastmoney.com/a/f1.html",
+                               "http://finance.eastmoney.com/a/f2.html"]
+    today = dt.date.today().isoformat()
+    ft = pd.read_parquet(root / "news_fulltext" / f"dt={today}" / "part-0.parquet")
+    assert len(ft) == 2 and (ft["content_text"] == "ok").all()
+    # 幂等: 账本已知 → 重扫零 fetch
+    fetched2 = _wire_backfill_fulltext(monkeypatch, {})
+    cd.run_lane("backfill")
+    assert fetched2 == []
+
+
+def test_fulltext_backfill_404_tombstone_no_retry(root, monkeypatch):
+    import pandas as pd
+    _seed_news_meta(root, _NEWS_SEED)
+    fetched = _wire_backfill_fulltext(
+        monkeypatch, {"http://finance.eastmoney.com/a/f1.html":
+                      cd.HttpDeterministicError(404, "x")})
+    cd.run_lane("backfill")
+    today = dt.date.today().isoformat()
+    ft = pd.read_parquet(root / "news_fulltext" / f"dt={today}" / "part-0.parquet")
+    by_id = {r["art_code"]: r for r in ft.to_dict("records")}
+    assert by_id["f1"]["content_text"] is None      # 墓碑行(已处理无正文)
+    assert by_id["f2"]["content_text"] == "ok"
+    fetched2 = _wire_backfill_fulltext(monkeypatch, {})
+    cd.run_lane("backfill")
+    assert fetched2 == []                           # 墓碑也进账本,不再重试
+
+
+def _seed_ann_meta(root, rows):
+    import pandas as pd
+    day = dt.date.today().isoformat()
+    d = root / "ann_meta" / f"dt={day}"
+    d.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(rows).to_parquet(d / "part-0.parquet", index=False)
+
+
+def _ann_norm_row(aid, title, ann_time):
+    return {"announcement_id": aid, "sec_code": "600519", "sec_name": None,
+            "org_id": None, "title": title, "short_title": None, "content": None,
+            "ann_time": ann_time, "ann_type": None, "ann_type_name": None,
+            "column_id": None, "important": None,
+            "adjunct_url": f"finalpage/2026-04-30/{aid}.PDF",
+            "adjunct_size": 244, "batch_num": None,
+            "first_seen_date": dt.date.today().isoformat()}
+
+
+def _wire_pdf_backfill(monkeypatch):
+    monkeypatch.setattr(cd, "load_stock_pool", lambda **kw: POOL)
+    monkeypatch.setattr(cd, "fetch_cninfo_announcements",
+                        MagicMock(return_value=[]))
+    monkeypatch.setattr(cd, "fetch_news_backfill_page",
+                        MagicMock(return_value=[]))
+    monkeypatch.setattr(cd, "fetch_article_html", MagicMock())
+    downloads = []
+
+    def fake_dl(client, url, dest):
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(b"%PDF-fake")
+        downloads.append((url, str(dest)))
+        return 12
+
+    monkeypatch.setattr(cd, "download_pdf", fake_dl)
+    return downloads
+
+
+def test_pdf_backfill_selects_six_types_new_first(root, monkeypatch):
+    import pandas as pd
+    _seed_ann_meta(root, [
+        _ann_norm_row("p_new", "2025年年度报告", "2025-04-30 10:00:00"),
+        _ann_norm_row("p_mid", "2023年度业绩快报", "2023-04-20 10:00:00"),
+        _ann_norm_row("p_old", "2020年半年度报告", "2020-08-28 10:00:00"),
+        _ann_norm_row("p_skip", "第三届董事会决议公告", "2024-01-02 10:00:00"),
+    ])
+    downloads = _wire_pdf_backfill(monkeypatch)
+    rc = cd.run_lane("backfill")
+    assert rc == 0
+    aids = [u.split("/")[-1].replace(".PDF", "") for u, _ in downloads]
+    assert aids == ["p_new", "p_mid", "p_old"]       # 新→旧,非六类不选
+    assert (root / "ann_pdf" / "2025" / "p_new.pdf").exists()
+    idx = pd.read_parquet(root / "state" / "ann_pdf_index.parquet")
+    assert len(idx) == 3
+    assert idx.iloc[0]["announcement_id"] == "p_new"
+    # 幂等: 文件在=跳过,零重下
+    downloads2 = _wire_pdf_backfill(monkeypatch)
+    cd.run_lane("backfill")
+    assert downloads2 == []
+
+
+def test_pdf_backfill_404_tombstone(root, monkeypatch):
+    _seed_ann_meta(root, [_ann_norm_row("p404", "2024年年度报告",
+                                        "2024-04-30 10:00:00")])
+
+    def boom(client, url, dest):
+        raise cd.HttpDeterministicError(404, url)
+
+    monkeypatch.setattr(cd, "load_stock_pool", lambda **kw: POOL)
+    monkeypatch.setattr(cd, "fetch_cninfo_announcements", MagicMock(return_value=[]))
+    monkeypatch.setattr(cd, "fetch_news_backfill_page", MagicMock(return_value=[]))
+    monkeypatch.setattr(cd, "fetch_article_html", MagicMock())
+    monkeypatch.setattr(cd, "download_pdf", boom)
+    rc = cd.run_lane("backfill")
+    assert rc == 0                                    # missing 不算失败
+    downloads = _wire_pdf_backfill(monkeypatch)
+    cd.run_lane("backfill")
+    assert downloads == []                            # 墓碑不重试
+
+
+def test_capacity_gate_pauses_pdf_only(root, monkeypatch):
+    """free < 阈值 → PDF 段暂停(其余段照跑), 留 capacity_paused.json 告警。"""
+    import pandas as pd
+    _seed_ann_meta(root, [_ann_norm_row("p1", "2025年年度报告",
+                                        "2025-04-30 10:00:00")])
+    downloads = _wire_pdf_backfill(monkeypatch)
+    monkeypatch.setattr(cd, "_disk_free_gb", lambda path: 100.0)
+    rc = cd.run_lane("backfill")
+    assert rc == 0                                    # 暂停≠失败,其余段不受影响
+    assert downloads == []                            # PDF 未下
+    flag = root / "state" / "capacity_paused.json"
+    assert flag.exists()
+    assert "free_gb" in json.loads(flag.read_text())
+    # 恢复(空间回升) → 复跑即补
+    downloads2 = _wire_pdf_backfill(monkeypatch)
+    monkeypatch.setattr(cd, "_disk_free_gb", lambda path: 645.0)
+    cd.run_lane("backfill")
+    assert len(downloads2) == 1