claude_dev
67fae2fef2
根因: 全市场grid(5555股×2670日×33列≈4G)单次join_asof + batch_eval侧 特征帧+alpha_df双全量副本,峰值10G+,7.9G NAS必爆(合成3股测不出)。 - adapter重构: iter_fundamental_feature_chunks生成器——grid按股分批构建 (BATCH_CODES=500,批间无全量grid副本),事件右表(报告期+forecast)全批共用一份; build_fundamental_features改为chunks concat(单一代码路径) - batch_eval: 特征join移到del bars之后(省1G bars常驻),逐块filter→join alpha_df分片→concat,不再持有特征帧全量副本;断点续跑已完成的财务因子不再触发join - 消两处join_asof UserWarning: 显式按键sort后抑制polars 1.42 by分组无法 校验sortedness的无信息提示(sort即正确性保险;set_sorted实测压不住) - 等值测试: 6股合成域 batch=1/2/6 逐值等值(分块不改变结果) - NAS真数据探针(600真股×2018-2026×batch500): roe_ttm覆盖0.904, 峰值RSS 1017MB(含全量statement加载),零OOM;生产规模外推~2G内 [nas] Co-Authored-By: Claude Code <noreply@anthropic.com>
219 lines
10 KiB
Python
219 lines
10 KiB
Python
# tests/factor/test_fundamental_adapter.py
|
||
"""财务因子适配层:合成三表+forecast parquet → PIT 日频特征列.
|
||
|
||
核心口径红线(docs/fundamental_factor_survey_20260907.md §7):
|
||
- 单季差分缺上期 → NaN 不填 0
|
||
- PIT = NOTICE_DATE ≤ 决策日,NOTICE_DATE 缺失行整报告期跳过
|
||
- TTM 不足连续 4 季 → NaN
|
||
"""
|
||
import sys, os
|
||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "vnpy_v4.4.0")))
|
||
|
||
import pytest
|
||
|
||
from sanguo_factor.fundamental_adapter import build_fundamental_features, FEATURE_COLUMNS
|
||
|
||
A, B, C = "600000.SSE", "000001.SZSE", "300001.SZSE"
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def feat(synthetic_static):
|
||
return build_fundamental_features(
|
||
[A, B, C], "2023-01-01", "2023-12-31", data_dir=synthetic_static)
|
||
|
||
|
||
def _dt(day: str):
|
||
import datetime as _d
|
||
return _d.datetime.strptime(day, "%Y-%m-%d")
|
||
|
||
|
||
def val(df, vt: str, day: str, col: str):
|
||
row = df.filter((df["vt_symbol"] == vt) & (df["datetime"] == _dt(day)))
|
||
assert row.height == 1, f"grid 缺行 {vt} {day}"
|
||
v = row[col][0]
|
||
return None if v is None else float(v)
|
||
|
||
|
||
# ---------- 单季差分(R1) ----------
|
||
|
||
def test_single_quarter_diff_and_q1_direct(feat):
|
||
# 报告期级断言走 equity 之外的特征:用 grid 在披露日后取值
|
||
# 2022H1 披露于 2022-08-29,2023 年窗口看不到;单季值通过 rev_q_yoy 间接锁
|
||
# 直接锁:2023Q3 (i=10) rev_q_yoy = 170/150 - 1(单季差分链正确才可得)
|
||
assert val(feat, A, "2023-10-27", "rev_q_yoy") == pytest.approx(170 / 150 - 1)
|
||
|
||
|
||
def test_missing_prev_quarter_is_nan_not_zero(synthetic_static):
|
||
# B 缺 2022Q1 → 2022Q2 单季差分 NaN;传染:TTM 至 2023Q1、yoy 至 2023H1 均 NaN
|
||
df = build_fundamental_features([B], "2023-01-01", "2023-12-31", data_dir=synthetic_static)
|
||
# 2023Q1(i=8) 报告期 TTM 窗含 2022Q2(差分 NaN) → np_ttm NaN
|
||
assert val(df, B, "2023-04-28", "np_ttm") is None
|
||
# 2023H1(i=9) yoy 基期 = 2022H1 单季 NaN → NaN
|
||
assert val(df, B, "2023-08-29", "rev_q_yoy") is None
|
||
# 窗口滑出坏点后恢复:2023Q3(i=10) 基期 2022Q3 单季=15 有效
|
||
assert val(df, B, "2023-10-27", "rev_q_yoy") == pytest.approx(170 / 150 - 1)
|
||
assert val(df, B, "2023-08-29", "np_ttm") == pytest.approx(60.0)
|
||
|
||
|
||
def test_ttm_rolling_four_quarters(feat):
|
||
# A 2023Q2 报告期(i=9): NP_TTM = 15+18+13+14 = 60(=上年年报580+H1−上年H1 交叉验证同值)
|
||
assert val(feat, A, "2023-08-29", "np_ttm") == pytest.approx(60.0)
|
||
# 年报 NOTICE=2024-04-25,2023 窗末 np_ttm 停留在 Q3 报告期(i=10): 18+13+14+17=62
|
||
assert val(feat, A, "2023-12-31", "np_ttm") == pytest.approx(62.0)
|
||
|
||
|
||
# ---------- PIT(R3):NOTICE_DATE 前不可见 ----------
|
||
|
||
def test_pit_no_lookahead(feat):
|
||
# A 2023H1(报告期 06-30) NOTICE=08-29;equity 平衡表时点值 i=9→680, i=8→660
|
||
assert val(feat, A, "2023-08-28", "equity") == pytest.approx(660.0)
|
||
assert val(feat, A, "2023-08-29", "equity") == pytest.approx(680.0)
|
||
|
||
|
||
def test_notice_date_null_period_skipped(feat):
|
||
# B 2022Q3 NOTICE 缺失 → 整期跳过:2023-01-01 可见的最新披露 = 2022H1(i=5, EQ=600)
|
||
assert val(feat, B, "2023-01-01", "equity") == pytest.approx(600.0)
|
||
# 恢复:2022 年报(i=7) 2023-04-25 可见
|
||
assert val(feat, B, "2023-04-25", "equity") == pytest.approx(640.0)
|
||
|
||
|
||
def test_forward_fill_between_notices(feat):
|
||
assert val(feat, A, "2023-09-15", "equity") == pytest.approx(680.0)
|
||
assert val(feat, A, "2023-10-26", "equity") == pytest.approx(680.0)
|
||
assert val(feat, A, "2023-10-27", "equity") == pytest.approx(700.0)
|
||
# 年报 NOTICE=次年 04-25,2023 窗末仍是 Q3 值
|
||
assert val(feat, A, "2023-12-31", "equity") == pytest.approx(700.0)
|
||
|
||
|
||
# ---------- 报告期级指标公式 ----------
|
||
|
||
def test_profitability_ratios(feat):
|
||
# A 2023H1(i=9): NP_TTM=60, EQ=680, TA=1450, CFO_TTM=72, GP_TTM=0.4*REV_TTM
|
||
assert val(feat, A, "2023-08-29", "roe_ttm") == pytest.approx(60 / 680)
|
||
assert val(feat, A, "2023-08-29", "roa_ttm") == pytest.approx(60 / 1450)
|
||
assert val(feat, A, "2023-08-29", "cfo_over_assets") == pytest.approx(72 / 1450)
|
||
assert val(feat, A, "2023-08-29", "gp_over_assets") == pytest.approx(240 / 1450)
|
||
assert val(feat, A, "2023-08-29", "gross_margin") == pytest.approx(0.4, abs=1e-9)
|
||
assert val(feat, A, "2023-08-29", "net_margin") == pytest.approx(60 / 600, rel=1e-6)
|
||
assert val(feat, A, "2023-08-29", "roe_deduct_ttm") == pytest.approx(0.9 * 60 / 680)
|
||
|
||
|
||
def test_quality_ratios(feat):
|
||
assert val(feat, A, "2023-08-29", "tacc") == pytest.approx((60 - 72) / 1450)
|
||
assert val(feat, A, "2023-08-29", "nonrec_ratio") == pytest.approx(6 / 60)
|
||
# 减值 = abs(0.02+0.01)*NP_TTM / TA
|
||
assert val(feat, A, "2023-08-29", "impairment_ratio") == pytest.approx(1.8 / 1450)
|
||
# 投资收益依赖 = (0.05+0.01)*NP_TTM / abs(1.1*NP_TTM)
|
||
assert val(feat, A, "2023-08-29", "invest_income_dep") == pytest.approx(0.06 / 1.1)
|
||
assert val(feat, A, "2023-08-29", "sales_cash_ratio") == pytest.approx(1.05 * 600 / 600, rel=1e-6)
|
||
assert val(feat, A, "2023-08-29", "other_rece_ratio") == pytest.approx((5 + 9) / 1450)
|
||
# 应收异常 = AR同比 − REV_TTM 同比(2023H1 vs 2022H1; TTM 窗含 2022Q2..2023H1)
|
||
ar_yoy = (100 + 90) / (100 + 50) - 1
|
||
rev_ttm_yoy = 600 / (150 + 160 + 110 + 140) - 1
|
||
assert val(feat, A, "2023-08-29", "receivables_anomaly") == pytest.approx(ar_yoy - rev_ttm_yoy)
|
||
|
||
|
||
def test_growth_and_capital_features(feat):
|
||
assert val(feat, A, "2023-08-29", "np_q_yoy") == pytest.approx(14 / 14 - 1)
|
||
assert val(feat, A, "2023-10-27", "np_q_yoy") == pytest.approx(17 / 15 - 1)
|
||
assert val(feat, A, "2023-08-29", "growth_scissors") == pytest.approx(0.0, abs=1e-9)
|
||
assert val(feat, A, "2023-08-29", "gm_delta") == pytest.approx(0.0, abs=1e-9)
|
||
# roe_delta = ROE(2023H1) − ROE(2022H1) = 60/680 − 56/600
|
||
assert val(feat, A, "2023-08-29", "roe_delta") == pytest.approx(60 / 680 - 56 / 600)
|
||
# 2023Q1(i=8): 资产增速 = 1400/1200−1; NSI = (110−100)/100
|
||
assert val(feat, A, "2023-04-28", "asset_growth") == pytest.approx(1400 / 1200 - 1)
|
||
assert val(feat, A, "2023-04-28", "nsi") == pytest.approx(0.1)
|
||
# IBD = SHORT_LOAN 唯一组件(其余列缺失按 0) = 109 (i=9)
|
||
assert val(feat, A, "2023-08-29", "ibd_ratio") == pytest.approx(109 / 1450)
|
||
assert val(feat, A, "2023-08-29", "goodwill_ratio") == pytest.approx(50 / 1450)
|
||
|
||
|
||
def test_sue_foster_standardization(feat, np_q_series):
|
||
# 独立重算: diff4 = Q_t − Q_{t-4}, SUE = diff4 / std(过去 8 期 diff4, ddof=1)
|
||
import statistics
|
||
q = [float(x) for x in np_q_series] # 16 期序列,2020 为 SUE 滚动窗预热
|
||
diff4 = [q[i] - q[i - 4] for i in range(4, 16)] # diff4[r] ↔ 报告期 r+4
|
||
|
||
def sue_at(rep_idx: int):
|
||
r = rep_idx - 4
|
||
if r < 7:
|
||
return None
|
||
win = diff4[r - 7:r + 1]
|
||
return diff4[r] / statistics.stdev(win)
|
||
|
||
# 2023Q3(报告期 i=14, 披露 2023-10-27): 窗 diff4[3..10]
|
||
assert val(feat, A, "2023-10-27", "sue_np") == pytest.approx(sue_at(14))
|
||
# 前一日仍见 2023H1(i=13)
|
||
assert val(feat, A, "2023-10-26", "sue_np") == pytest.approx(sue_at(13))
|
||
# 2023Q4 披露在 2024-04-25,窗末仍是 Q3 值
|
||
assert val(feat, A, "2023-12-31", "sue_np") == pytest.approx(sue_at(14))
|
||
# 合成史 16 期 → i=11(2022Q4)起才有完整 8 期窗,更早报告期 SUE=NaN(间接受 PIT 保护)
|
||
|
||
|
||
def test_forecast_event_features(feat):
|
||
# A: 2023-07-15 预增(+3, 56.79);2023-10-15 扭亏(+2, 100.0)覆盖
|
||
assert val(feat, A, "2023-07-14", "forecast_type_score") is None
|
||
assert val(feat, A, "2023-07-15", "forecast_type_score") == 3.0
|
||
assert val(feat, A, "2023-07-15", "forecast_change_pct") == pytest.approx(56.79)
|
||
assert val(feat, A, "2023-10-14", "forecast_type_score") == 3.0
|
||
assert val(feat, A, "2023-10-15", "forecast_type_score") == 2.0
|
||
# B 预减(−3); C 无净利润行 fallback 营业收入行 略增(+2)
|
||
assert val(feat, B, "2023-07-20", "forecast_type_score") == -3.0
|
||
assert val(feat, B, "2023-07-20", "forecast_change_pct") == pytest.approx(-30.0)
|
||
assert val(feat, C, "2023-07-10", "forecast_type_score") == 2.0
|
||
|
||
|
||
# ---------- 结构与容错 ----------
|
||
|
||
def test_feature_columns_complete(feat):
|
||
out_cols = set(feat.columns) - {"vt_symbol", "datetime"}
|
||
assert out_cols == set(FEATURE_COLUMNS)
|
||
assert feat.height > 0
|
||
|
||
|
||
def test_missing_file_and_empty_rows_tolerated(synthetic_static):
|
||
# 600999 无文件(退市股形态) → 行存在但特征全 null,不炸
|
||
df = build_fundamental_features(["600999.SSE"], "2023-06-01", "2023-06-10",
|
||
data_dir=synthetic_static)
|
||
assert df.height > 0
|
||
for col in FEATURE_COLUMNS:
|
||
assert df[col].null_count() == df.height, f"{col} 应全 null"
|
||
|
||
|
||
def test_trading_dates_grid(feat, synthetic_static):
|
||
# 传入交易日子集 → grid 只含这些日期
|
||
df = build_fundamental_features(
|
||
[A], "2023-08-01", "2023-08-31", data_dir=synthetic_static,
|
||
trading_dates=[_dt("2023-08-14"), _dt("2023-08-29")])
|
||
assert df.height == 2
|
||
assert val(df, A, "2023-08-14", "equity") == pytest.approx(660.0)
|
||
assert val(df, A, "2023-08-29", "equity") == pytest.approx(680.0)
|
||
|
||
|
||
def test_scale_two_stock(feat):
|
||
# C(scale=2): TA=2*1450, NP_TTM=2*60 → roa 同 A(比率不变), np_ttm 翻倍
|
||
assert val(feat, C, "2023-08-29", "roa_ttm") == pytest.approx(60 / 1450)
|
||
assert val(feat, C, "2023-08-29", "np_ttm") == pytest.approx(120.0)
|
||
|
||
|
||
# ---------- 分块等值(NAS 全量防 OOM 路径) ----------
|
||
|
||
_SIX = ["600000.SSE", "000001.SZSE", "300001.SZSE",
|
||
"600004.SSE", "000333.SZSE", "300124.SZSE"]
|
||
|
||
|
||
def test_chunked_equals_full(synthetic_static):
|
||
"""batch_codes=2(3 批) 与 batch_codes=6(单批) 逐值等值——分块不改变结果."""
|
||
full = build_fundamental_features(
|
||
_SIX, "2023-01-01", "2023-12-31", data_dir=synthetic_static, batch_codes=6)
|
||
chunked = build_fundamental_features(
|
||
_SIX, "2023-01-01", "2023-12-31", data_dir=synthetic_static, batch_codes=2)
|
||
assert full.height == chunked.height == 6 * 365
|
||
key = ["vt_symbol", "datetime"]
|
||
assert full.sort(key).equals(chunked.sort(key))
|
||
# 批大小 1(极端) 与 trading_dates 路径同款等值
|
||
by_one = build_fundamental_features(
|
||
_SIX, "2023-01-01", "2023-12-31", data_dir=synthetic_static, batch_codes=1)
|
||
assert by_one.sort(key).equals(full.sort(key))
|