feat(data): P1+P2+P3季频财务+业绩报告——bs_fundamentals新脚本(sanguo-bs-fund schtask 23:05,VPS)——①P2回灌:profit利润+dupont杜邦2015Q1+按季分片,state文件推进日预算自适应续跑(约2季/天≈3周自动跑完;预算不够整季不拉半季,半段落库也幂等)②P1增量:回灌完成后拉当季+上一季(兜住8-31晚披季报)③P3业绩报告:forecast预告2003+/express快报,每股一次调用与区间无关→月首周日全区间恒定2N query幂等拉齐(--reports-now冒烟)④落parquet data/fundamentals_baostock/保留pubDate列(读侧pubDate<=date过滤防前视,同fundamentals-lookahead-bias-fix模式);去重(code,pubDate,statDate)restated后写者胜,键不足2列退全列绝不按单列丢历史⑤预算铁律(48000/天):bs_eod~11k+本脚本DAILY_CAP=30000串行=≤41k,月首周日叠reports 44.3k<48k余3.7k;开跑前schtasks探sanguo-bs-eod(极端尾22:41)在跑则exit0次日幂等补——同IP永不双连接⑥复用bs_eod四件套(login_with_retry/relogin/_with_timeout/全A清单)+周期relogin+失败relogin兜底;字段名读rs.fields不硬编码;+14测试(季度边界/parquet幂等+防呆/state损坏重头/动态字段+计数/回灌2季cap推进/state超前零拉/增量当季+上季/报告月首周日窗口/守卫文本/main跳过+exit2);data_platform 140绿 [vps]

2026-08-20 09:53:33 +08:00
parent dbb5ed5aa8
commit d0a538ced3
2 changed files with 592 additions and 0 deletions
@@ -0,0 +1,245 @@
+# -*- coding: utf-8 -*-
+"""TDD for bs_fundamentals.py (P1 季频增量 + P2 历史回灌 + P3 业绩预告/快报).
+
+设计 (2026-08-20 数据补全 P0-P3):
+- profit/dupont: 回灌 2015Q1+ 按季分片日预算自适应续跑(state 文件推进); 完成后转
+  增量(当季+上一季, 兜住晚披季报)
+- forecast/express: 每股一次调用与区间无关 → 月首周日全区间恒定 2N query 幂等拉
+- 落 parquet 保留 pubDate (读侧按 pubDate<=date 过滤防前视, 同
+  fundamentals-lookahead-bias-fix 模式)
+- 预算(单 IP 48000/天): bs_eod ~11k(18:05) + 本脚本 DAILY_CAP=30000(23:05 串行,
+  且开跑前探 sanguo-bs-eod 未在跑防同 IP 双连接) = ≤41k; 月首周日叠 reports 11k
+  → 44.3k < 48k 余 3.7k
+"""
+import datetime as dt
+import sys
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+if "baostock" not in sys.modules:
+    sys.modules["baostock"] = MagicMock()
+
+from scripts.data_platform import bs_fundamentals as bf  # noqa: E402
+
+
+# ---------- Fixtures ----------
+
+@pytest.fixture(autouse=True)
+def _fast(monkeypatch):
+    monkeypatch.setattr(bf, "BS_INTERVAL", 0.0)
+
+
+@pytest.fixture
+def tmp_out(tmp_path, monkeypatch):
+    monkeypatch.setattr(bf, "OUT_DIR", tmp_path)
+    return tmp_path
+
+
+@pytest.fixture
+def small_stocks():
+    return [("600001", "sh"), ("000002", "sz")]
+
+
+@pytest.fixture
+def reset_qc():
+    orig = bf.QUERY_COUNT
+    bf.QUERY_COUNT = 0
+    yield
+    bf.QUERY_COUNT = orig
+
+
+def _qrow(bs_code, year, quarter):
+    """fake 季频行: pubDate/statDate 随季变化(否则跨季被去重合并)."""
+    m = quarter * 3
+    return {"code": bs_code, "pubDate": f"{year}-{m:02d}-28",
+            "statDate": f"{year}-{m:02d}-30", "v": "1"}
+
+
+# ---------- 季度工具 ----------
+
+def test_quarter_math_boundaries():
+    assert bf.prev_quarter(2026, 1) == (2025, 4)
+    assert bf.next_quarter_of(2025, 4) == (2026, 1)
+    assert bf.quarter_list(2015, (2015, 2)) == [(2015, 1), (2015, 2)]
+    assert bf.quarter_list(2015, (2016, 1))[-1] == (2016, 1)
+
+
+# ---------- parquet 幂等追加 ----------
+
+def test_append_parquet_dedup_idempotent(tmp_out):
+    """同 (code,pubDate,statDate) 重写 → 后值胜, 行数不涨(幂等)."""
+    bf.append_parquet("profit", [
+        {"code": "sh.600519", "pubDate": "2026-04-20",
+         "statDate": "2026-03-31", "roeAvg": "0.3"}])
+    bf.append_parquet("profit", [
+        {"code": "sh.600519", "pubDate": "2026-04-20",
+         "statDate": "2026-03-31", "roeAvg": "0.31"}])  # restated
+    df = pd.read_parquet(tmp_out / "profit.parquet")
+    assert len(df) == 1
+    assert df.iloc[0]["roeAvg"] == "0.31"
+
+
+def test_append_parquet_guard_when_keys_missing(tmp_out):
+    """防呆: 去重键不足 2 列(如只剩 code) → 退全列去重, 绝不按单列丢历史."""
+    bf.append_parquet("profit", [{"code": "a", "v": "1"}])
+    bf.append_parquet("profit", [{"code": "a", "v": "2"}])
+    df = pd.read_parquet(tmp_out / "profit.parquet")
+    assert len(df) == 2  # 两行都在
+
+
+# ---------- 回灌 state ----------
+
+def test_state_roundtrip_and_corrupt_fallback(tmp_out):
+    assert bf.load_state()["next_quarter"] == "2015Q1"  # 缺省从 P2 起点开始
+    st = {"next_quarter": "2016Q3"}
+    bf.save_state(st)
+    assert bf.load_state()["next_quarter"] == "2016Q3"
+    (tmp_out / "backfill_state.json").write_text("{broken", encoding="utf-8")
+    assert bf.load_state()["next_quarter"] == "2015Q1"  # 损坏→从头(parquet 幂等)
+
+
+# ---------- fetch_table (动态字段 + 计数) ----------
+
+def _fake_rs(fields, rows):
+    rs = MagicMock()
+    rs.error_code = "0"
+    rs.fields = fields
+    rs.next.side_effect = [True] * len(rows) + [False]
+    rs.get_row_data.side_effect = rows
+    return rs
+
+
+def test_fetch_table_reads_dynamic_fields_and_counts(monkeypatch, reset_qc):
+    """字段名取自 rs.fields(不硬编码防 API 变动), QUERY_COUNT +1."""
+    mock_bs = MagicMock()
+    mock_bs.query_profit_data.return_value = _fake_rs(
+        ["code", "roeAvg", "npMargin"], [["sh.600519", "0.3", "0.4"]])
+    monkeypatch.setattr(bf, "bs", mock_bs)
+    rows = bf.fetch_table("profit", "sh.600519", year=2026, quarter=2)
+    assert rows == [{"code": "sh.600519", "roeAvg": "0.3", "npMargin": "0.4"}]
+    assert bf.QUERY_COUNT == 1
+
+
+def test_fetch_table_raises_on_error_code(monkeypatch, reset_qc):
+    mock_bs = MagicMock()
+    rs = MagicMock()
+    rs.error_code = "10002007"
+    rs.error_msg = "网络接收错误"
+    mock_bs.query_dupont_data.return_value = rs
+    monkeypatch.setattr(bf, "bs", mock_bs)
+    with pytest.raises(RuntimeError, match="10002007"):
+        bf.fetch_table("dupont", "sh.600519", year=2026, quarter=2)
+
+
+# ---------- P2 回灌: 按季分片 + 日预算自适应 ----------
+
+def test_backfill_two_quarters_then_cap(tmp_out, small_stocks, monkeypatch, reset_qc):
+    """cap=9: 每季 2表×2股=4q → Q1(4)+Q2(4)=8, Q3 预检 8+4>9 → capped, state 推进两季."""
+    monkeypatch.setattr(bf, "DAILY_CAP", 9)
+    calls = []
+
+    def fake_fetch(table, bs_code, year=None, quarter=None, start=None, end=None):
+        bf.QUERY_COUNT += 1
+        calls.append((table, year, quarter))
+        return [_qrow(bs_code, year, quarter)]
+
+    monkeypatch.setattr(bf, "fetch_table", fake_fetch)
+    status = bf.run_quarter_backfill(small_stocks, dt.date(2026, 8, 20))
+    assert status == "capped"
+    assert len(calls) == 8  # 2季 × 2表 × 2股 = 8 query, Q3 预检 8+4>9 不拉半季
+    assert bf.load_state()["next_quarter"] == "2015Q3"  # 推进两季(Q1,Q2)
+    # 两季都已落 parquet
+    df = pd.read_parquet(tmp_out / "profit.parquet")
+    assert df["statDate"].nunique() == 2
+
+
+def test_backfill_done_when_state_ahead_of_target(tmp_out, small_stocks, monkeypatch):
+    """state 已越过目标季 → done, 零 fetch."""
+    bf.save_state({"next_quarter": "2027Q1"})
+    monkeypatch.setattr(bf, "fetch_table",
+                        MagicMock(side_effect=AssertionError("不应再拉")))
+    assert bf.run_quarter_backfill(small_stocks,
+                                   dt.date(2026, 8, 20)) == "done"
+
+
+# ---------- P1 增量: 当季 + 上一季 ----------
+
+def test_incremental_pulls_current_and_prev_quarter(tmp_out, small_stocks,
+                                                    monkeypatch, reset_qc):
+    """2026-08 → 拉 2026Q3(当季) + 2026Q2(上一季晚披季报兜住)."""
+    seen = []
+
+    def fake_fetch(table, bs_code, year=None, quarter=None, start=None, end=None):
+        seen.append((table, year, quarter))
+        return [_qrow(bs_code, year, quarter)]
+
+    monkeypatch.setattr(bf, "fetch_table", fake_fetch)
+    bf.run_quarter_incremental(small_stocks, dt.date(2026, 8, 20))
+    assert {(y, q) for _, y, q in seen} == {(2026, 2), (2026, 3)}
+    assert {t for t, _, _ in seen} == {"profit", "dupont"}
+    assert (tmp_out / "profit.parquet").exists()
+    assert (tmp_out / "dupont.parquet").exists()
+
+
+# ---------- P3 业绩报告: 月首周日 ----------
+
+def _first_sunday(year, month):
+    d = dt.date(year, month, 1)
+    while d.weekday() != 6:
+        d += dt.timedelta(days=1)
+    return d
+
+
+def test_reports_run_on_first_sunday(tmp_out, small_stocks, monkeypatch, reset_qc):
+    sun = _first_sunday(2026, 8)  # 必 ≤7 号
+    assert sun.day <= 7
+    ranges = []
+
+    def fake_fetch(table, bs_code, year=None, quarter=None, start=None, end=None):
+        ranges.append((table, start, end))
+        return [{"code": bs_code, "pubDate": "2026-04-01", "v": "1"}]
+
+    monkeypatch.setattr(bf, "fetch_table", fake_fetch)
+    bf.run_reports(small_stocks, sun, force=False)
+    assert {t for t, _, _ in ranges} == {"forecast", "express"}
+    assert all(s == "2003-01-01" for _, s, _ in ranges)  # 全区间一次拉齐
+
+
+def test_reports_skip_non_sunday_and_second_sunday(tmp_out, small_stocks,
+                                                    monkeypatch):
+    monkeypatch.setattr(bf, "fetch_table",
+                        MagicMock(side_effect=AssertionError("不应拉")))
+    sun = _first_sunday(2026, 8)
+    bf.run_reports(small_stocks, sun + dt.timedelta(days=1), force=False)  # 周一
+    bf.run_reports(small_stocks, sun + dt.timedelta(days=7), force=False)  # 第二周日>7号
+
+
+# ---------- bs_eod 在跑守卫(同 IP 双连接红线) ----------
+
+def test_is_running_text_variants():
+    assert bf._is_running_text("Status: Running") is True
+    assert bf._is_running_text("状态: 运行中") is True
+    assert bf._is_running_text("Status: Ready") is False
+    assert bf._is_running_text("模式: 就绪") is False
+
+
+def test_main_skips_when_bs_eod_in_flight(monkeypatch, tmp_out):
+    """sanguo-bs-eod 仍在跑 → 直接 exit 0(次日幂等补), 绝不同 IP 双登录."""
+    monkeypatch.setattr(bf, "_bs_eod_running", lambda: True)
+    monkeypatch.setattr(bf, "login_with_retry",
+                        MagicMock(side_effect=AssertionError("不应登录")))
+    monkeypatch.setattr(sys, "argv", ["bs_fundamentals.py"])
+    with pytest.raises(SystemExit) as e:
+        bf.main()
+    assert e.value.code == 0
+
+
+def test_main_exit2_when_login_fails(monkeypatch, tmp_out):
+    monkeypatch.setattr(bf, "_bs_eod_running", lambda: False)
+    monkeypatch.setattr(bf, "login_with_retry", MagicMock(return_value=False))
+    monkeypatch.setattr(sys, "argv", ["bs_fundamentals.py"])
+    with pytest.raises(SystemExit) as e:
+        bf.main()
+    assert e.value.code == 2