feat(factor): P1批library注册37因子(35+2变体)+batch_eval引用列瘦身防OOM [nas]
This commit is contained in:
@@ -135,15 +135,23 @@ def run_batch_eval(
|
||||
buffer: list[dict] = []
|
||||
|
||||
# 财务特征分块 join: 逐块 filter→join alpha_df 分片→concat(bars 已释放;
|
||||
# 单块特征帧用完即弃,峰值 ≈ alpha_df + 已 join 分片累积,无全量特征副本)
|
||||
# 单块特征帧用完即弃,峰值 ≈ alpha_df + 已 join 分片累积,无全量特征副本)。
|
||||
# 引用瘦身: 只 join 本批 fundamental 表达式实际引用的特征列——
|
||||
# FEATURE_COLUMNS 全量 68 列 × 1480 万行 ≈ 8G 超 NAS 7.9G 内存,
|
||||
# 按引用裁列后单族批 ≈ 1-2G(P0 33 列全量 4G 基线的必要减负)
|
||||
has_fund = any((get_factor(n) or {}).get("category") == "fundamental"
|
||||
for n in factor_names)
|
||||
if has_fund:
|
||||
from .fundamental_adapter import iter_fundamental_feature_chunks
|
||||
import re as _re
|
||||
from .fundamental_adapter import iter_fundamental_feature_chunks, FEATURE_COLUMNS
|
||||
needed = set()
|
||||
for n in fund_names:
|
||||
expr = (get_factor(n) or {}).get("expression", "")
|
||||
needed |= set(_re.findall(r"[A-Za-z_][A-Za-z0-9_]*", expr)) & set(FEATURE_COLUMNS)
|
||||
parts = []
|
||||
for feat_chunk in iter_fundamental_feature_chunks(
|
||||
fund_codes, start, end, data_dir=fund_static_dir,
|
||||
trading_dates=fund_days):
|
||||
trading_dates=fund_days, columns=sorted(needed)):
|
||||
syms = feat_chunk["vt_symbol"].unique().to_list()
|
||||
parts.append(
|
||||
alpha_df.filter(pl.col("vt_symbol").is_in(syms))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""财务因子批 P0 表达式库: 32 个因子注册(category="fundamental").
|
||||
"""财务因子批表达式库: P0 32 + P1 37(35 因子 + 2 互评变体)注册(category="fundamental").
|
||||
|
||||
选型来源 docs/fundamental_factor_survey_20260907.md(§3 候选池 96 个 + §4.1 去重),
|
||||
首批 32 = P0 50 个按族去重后的主代表;全部「因子 = cs_rank(基础指标)」一层截面,
|
||||
@@ -68,10 +68,65 @@ FUNDAMENTAL_FACTORS: list[tuple[str, str, str, str]] = [
|
||||
("fund_forecast_change", "cs_rank(forecast_change_pct)", "F05", "+"),
|
||||
]
|
||||
|
||||
# P1 批第二批(35 因子 + 2 互评变体;编号 = 调研文档 §3,IC 列为文档预期原始方向,
|
||||
# 负 IC 表达式取负统一「高=好」;IC「不定」按正向注册待实证)。
|
||||
# 口径登记: EBIT=TP+利息费用(未披露按 0) / DA=四件折旧摊销 / τ 截断[0,0.5]缺 0.25 /
|
||||
# IBD 含 LEASE_LIAB / VSIG 16 季 sample std / B12 为哑变量连续近似乘积 /
|
||||
# F07 锚 = max(披露日, 预告公告日)——详见 fundamental_adapter 注释。
|
||||
FUNDAMENTAL_P1_FACTORS: list[tuple[str, str, str, str]] = [
|
||||
# ---- A 盈利能力 P1(7) ----
|
||||
("fund_roe_avg", "cs_rank(roe_avg)", "A03", "+"),
|
||||
("fund_roa_pretax", "cs_rank(roa_pretax)", "A09", "+"),
|
||||
("fund_ebit_over_assets", "cs_rank(ebit_over_assets)", "A10", "+"),
|
||||
("fund_ebitda_margin", "cs_rank(ebitda_margin)", "A11", "+"),
|
||||
("fund_roic", "cs_rank(roic)", "A12", "+"),
|
||||
("fund_rd_intensity", "cs_rank(rd_intensity)", "A15", "不定"),
|
||||
("fund_sale_expense_ratio", "cs_rank(-sale_expense_ratio)", "A16", "-(弱)"),
|
||||
# ---- B 盈利质量 P1(8) ----
|
||||
("fund_inventory_anomaly", "cs_rank(-inventory_anomaly)", "B09", "-(弱)"),
|
||||
("fund_cash_ibd_double", "cs_rank(-cash_ibd_product)", "B12", "-"),
|
||||
("fund_vsig", "cs_rank(-vsig)", "B13", "-"),
|
||||
("fund_vsig_acc", "cs_rank(-vsig_acc)", "B14", "-"),
|
||||
("fund_vsig_cfo", "cs_rank(-vsig_cfo)", "B15", "-"),
|
||||
("fund_da_intensity", "cs_rank(da_intensity)", "B16", "不定"),
|
||||
("fund_gm_nm_scissors", "cs_rank(-gm_nm_scissors)", "B18", "-(弱)"),
|
||||
("fund_profit_streak", "cs_rank(profit_streak)", "B19", "+"),
|
||||
# ---- C 成长 P1(8) ----
|
||||
("fund_np_accel", "cs_rank(np_accel)", "C07", "+"),
|
||||
("fund_rev_accel", "cs_rank(rev_accel)", "C08", "+"),
|
||||
("fund_nm_delta", "cs_rank(nm_delta)", "C11", "+"),
|
||||
("fund_rev_cagr5", "cs_rank(rev_cagr5)", "C13", "+"),
|
||||
("fund_np_cagr5", "cs_rank(np_cagr5)", "C14", "+"),
|
||||
("fund_nwc_growth", "cs_rank(-nwc_growth)", "C16", "-"),
|
||||
("fund_invest_growth", "cs_rank(-invest_growth)", "C18", "-"),
|
||||
("fund_equity_growth", "cs_rank(equity_growth)", "C19", "不定"),
|
||||
# ---- D 估值 P1(6;EV = close×share_capital + ev_ex_mv(=IBD−MON)) ----
|
||||
("fund_ebit_over_ev", "cs_rank(ebit_ttm / (close * share_capital + ev_ex_mv))", "D07", "+"),
|
||||
("fund_ebitda_over_ev", "cs_rank(ebitda_ttm / (close * share_capital + ev_ex_mv))", "D08", "+"),
|
||||
("fund_gp_over_m", "cs_rank(gp_ttm / (close * share_capital))", "D09", "+"),
|
||||
# D12 非 alpha: 中性化控制变量(§3.4——财务估值因子上线必配 size 中性化)
|
||||
("fund_ln_mv", "cs_rank(log(close * share_capital))", "D12", "控制变量"),
|
||||
("fund_forward_ep", "cs_rank(forecast_np_annualized / (close * share_capital))", "D13", "+"),
|
||||
("fund_fcf_over_ev", "cs_rank(fcf_ttm / (close * share_capital + ev_ex_mv))", "D15", "+"),
|
||||
# ---- E 资本结构与行为 P1(3) ----
|
||||
("fund_debt_issue", "cs_rank(-(debt_issue_ttm / (close * share_capital)))", "E03", "-"),
|
||||
("fund_interest_cover", "cs_rank(interest_cover)", "E07", "+"),
|
||||
("fund_goodwill_growth", "cs_rank(-goodwill_growth)", "E10", "-(弱)"),
|
||||
# ---- F 预期事件 P1(3) ----
|
||||
("fund_sue_eps", "cs_rank(sue_eps)", "F03", "+"),
|
||||
("fund_disclosure_speed", "cs_rank(disclosure_speed)", "F06", "早=+"),
|
||||
("fund_forecast_beat", "cs_rank(forecast_beat)", "F07", "高兑现=+"),
|
||||
# ---- 互评变体(2) ----
|
||||
# SUE 严窗: σ 只用 t−1 及更早差分(不含当期),与 F01 对照
|
||||
("fund_sue_np_strict", "cs_rank(sue_np_strict)", "F01v", "+"),
|
||||
# 资产增长负向对照: 原始方向(Cooper 2008 学术预期负 IC),与 C15 取负版互证
|
||||
("fund_asset_growth_neg", "cs_rank(asset_growth)", "C15v", "学术负IC对照"),
|
||||
]
|
||||
|
||||
|
||||
def _register_all() -> None:
|
||||
"""注册全部财务因子(已存在同名跳过,幂等;同 library.py 模式)."""
|
||||
for name, expression, _doc_id, _ic in FUNDAMENTAL_FACTORS:
|
||||
for name, expression, _doc_id, _ic in [*FUNDAMENTAL_FACTORS, *FUNDAMENTAL_P1_FACTORS]:
|
||||
if name not in _REGISTRY:
|
||||
register_factor(name, expression, category="fundamental")
|
||||
|
||||
|
||||
@@ -90,6 +90,29 @@ def test_mixed_batch_price_plus_fundamental(db, synthetic_static, tmp_path):
|
||||
assert "error" not in m
|
||||
|
||||
|
||||
def test_fundamental_p1_end_to_end(db, synthetic_static, tmp_path):
|
||||
"""P1 批端到端: VSIG(16 季窗)+EV 群(分母含 ev_ex_mv)+兑现差走引用瘦身 join."""
|
||||
eval_db = str(tmp_path / "fund_p1_eval.db")
|
||||
out = run_batch_eval(
|
||||
factor_names=["fund_vsig", "fund_ebit_over_ev", "fund_forecast_beat",
|
||||
"fund_profit_streak"],
|
||||
start="2023-02-01", end="2023-12-31",
|
||||
eval_db=eval_db, label="fund_p1", cfg=None, vnpy_db_override=db,
|
||||
fund_data_dir=synthetic_static,
|
||||
)
|
||||
assert out["factors_done"] == 4
|
||||
assert out["errors"] == []
|
||||
for name in ("fund_vsig", "fund_ebit_over_ev", "fund_forecast_beat",
|
||||
"fund_profit_streak"):
|
||||
m = eval_store.get_detail(eval_db, out["run_id"], name)["metrics"]
|
||||
assert "error" not in m, f"{name}: {m.get('error')}"
|
||||
# EV 群/持续盈利全截面有值 → IC 样本点非零;vsig 仅 2/3 股非空(B 窗含
|
||||
# NaN)→ <3 截面 IC 记 0(n>=3 引擎下限),路径本身已被无错误评估锁死
|
||||
for name in ("fund_ebit_over_ev", "fund_profit_streak"):
|
||||
m = eval_store.get_detail(eval_db, out["run_id"], name)["metrics"]
|
||||
assert m["1"]["count"] > 0, name
|
||||
|
||||
|
||||
def test_pure_price_batch_untouched(db, tmp_path):
|
||||
"""纯量价批不传静态域照常跑(回归: 未新增强依赖)."""
|
||||
eval_db = str(tmp_path / "px_eval.db")
|
||||
|
||||
@@ -19,7 +19,8 @@ def _ensure_fundamental_registered():
|
||||
|
||||
|
||||
# 表达式可引用的列 = adapter 特征列 + 行情列 close(估值类 ÷ close×share_capital)
|
||||
_ALLOWED = set(FEATURE_COLUMNS) | {"close", "cs_rank"}
|
||||
# + 表达式函数名(P1 起用到 log)
|
||||
_ALLOWED = set(FEATURE_COLUMNS) | {"close", "cs_rank", "log"}
|
||||
|
||||
|
||||
def _fundamental_factors() -> list[dict]:
|
||||
@@ -29,7 +30,8 @@ def _fundamental_factors() -> list[dict]:
|
||||
def test_p0_32_factors_registered():
|
||||
facs = _fundamental_factors()
|
||||
names = {f["name"] for f in facs}
|
||||
assert len(facs) == 32, f"P0 首批应为 32 个,实际 {len(facs)}"
|
||||
# P0 32 + P1 37(35 因子 + 2 互评变体;P1 名单见 test_fundamental_p1_library)
|
||||
assert len(facs) == 69, f"P0+P1 应为 69 个,实际 {len(facs)}"
|
||||
# 六族代表抽查(全部名单见 fundamental_library 注释)
|
||||
expect = {
|
||||
"fund_roe_ttm", "fund_gp_over_assets", # A
|
||||
@@ -81,4 +83,4 @@ def test_registration_idempotent():
|
||||
"""重复 import 不炸(注册表防重入,同 library.py 模式)."""
|
||||
import importlib
|
||||
importlib.reload(fundamental_library)
|
||||
assert len(_fundamental_factors()) == 32
|
||||
assert len(_fundamental_factors()) == 69
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# tests/factor/test_fundamental_p1_library.py
|
||||
"""P1 批财务因子表达式库: 37 个注册(35 因子 + 2 互评变体)与契约锁定.
|
||||
|
||||
契约:
|
||||
- 表达式裸标识符 ⊆ adapter 特征列 + close/log/cs_rank(漏加列 = 拼写错)
|
||||
- 全部 cs_rank 一层截面;文档负 IC 因子表达式取负(高=好统一)
|
||||
- EV 群分母 = close×share_capital + ev_ex_mv;D12 是中性化控制变量非 alpha
|
||||
"""
|
||||
import re
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "vnpy_v4.4.0")))
|
||||
|
||||
import pytest
|
||||
|
||||
from sanguo_factor import fundamental_library # noqa: F401 import 即注册
|
||||
from sanguo_factor.fundamental_adapter import FEATURE_COLUMNS
|
||||
from sanguo_factor.fundamental_library import FUNDAMENTAL_P1_FACTORS
|
||||
from sanguo_factor.registry import list_factors, get_factor
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _ensure_fundamental_registered():
|
||||
"""其它测试模块清空 _REGISTRY 后只重挂 alpha/builtin(顺序依赖前科),
|
||||
这里逐测试幂等重注册财务因子,保证本模块与顺序无关."""
|
||||
fundamental_library._register_all()
|
||||
|
||||
|
||||
_ALLOWED = set(FEATURE_COLUMNS) | {"close", "cs_rank", "log"}
|
||||
|
||||
|
||||
def _p1_names() -> set[str]:
|
||||
return {name for name, _e, _d, _ic in FUNDAMENTAL_P1_FACTORS}
|
||||
|
||||
|
||||
def test_p1_37_factors_registered():
|
||||
facs = {f["name"]: f for f in list_factors("fundamental")}
|
||||
p1 = _p1_names()
|
||||
assert len(p1) == 37, f"P1 应 37 个(35+2 变体),实际 {len(p1)}"
|
||||
for name in p1:
|
||||
assert name in facs, f"{name} 未注册"
|
||||
assert facs[name]["category"] == "fundamental"
|
||||
|
||||
|
||||
def test_p1_expressions_only_reference_feature_columns():
|
||||
"""契约: 表达式裸标识符 ⊆ adapter 特征列 + close/log/cs_rank."""
|
||||
for name, expression, _d, _ic in FUNDAMENTAL_P1_FACTORS:
|
||||
idents = set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", expression))
|
||||
bad = idents - _ALLOWED
|
||||
assert not bad, f"{name} 引用了未产出列: {bad} in {expression}"
|
||||
|
||||
|
||||
def test_p1_all_cross_sectional_rank_one_layer():
|
||||
for name, expression, _d, _ic in FUNDAMENTAL_P1_FACTORS:
|
||||
assert expression.startswith("cs_rank("), f"{name}: {expression}"
|
||||
assert expression.endswith(")")
|
||||
|
||||
|
||||
def test_p1_negative_ic_factors_flipped():
|
||||
"""文档负 IC 的 P1 因子(B09/B12/B13/B14/B15/B18/C16/C18/E03/E10/A16)取负."""
|
||||
flipped = {"fund_sale_expense_ratio", "fund_inventory_anomaly",
|
||||
"fund_cash_ibd_double", "fund_vsig", "fund_vsig_acc",
|
||||
"fund_vsig_cfo", "fund_gm_nm_scissors", "fund_nwc_growth",
|
||||
"fund_invest_growth", "fund_debt_issue", "fund_goodwill_growth"}
|
||||
for name in flipped:
|
||||
expr = get_factor(name)["expression"]
|
||||
assert "(-" in expr, f"{name} 应翻转: {expr}"
|
||||
|
||||
|
||||
def test_p1_ev_group_denominator():
|
||||
"""EV 群(D07/D08/D15)分母 = close×share_capital + ev_ex_mv(IBD−MON)."""
|
||||
for name in ("fund_ebit_over_ev", "fund_ebitda_over_ev", "fund_fcf_over_ev"):
|
||||
expr = get_factor(name)["expression"]
|
||||
assert "close * share_capital + ev_ex_mv" in expr, f"{name}: {expr}"
|
||||
|
||||
|
||||
def test_p1_valuation_uses_self_computed_mv():
|
||||
"""市值类因子(D09/D13/E03)统一 close×share_capital 自算(不读 valuation)."""
|
||||
for name in ("fund_gp_over_m", "fund_forward_ep", "fund_debt_issue"):
|
||||
expr = get_factor(name)["expression"]
|
||||
assert "close * share_capital" in expr, f"{name}: {expr}"
|
||||
|
||||
|
||||
def test_p1_ln_mv_is_control_variable():
|
||||
"""D12 = ln(MV) 中性化控制变量(非 alpha): log(close×share_capital)."""
|
||||
expr = get_factor("fund_ln_mv")["expression"]
|
||||
assert expr == "cs_rank(log(close * share_capital))"
|
||||
|
||||
|
||||
def test_p1_two_variants():
|
||||
"""互评变体: SUE 严窗列 + 资产增长负向对照(原始方向,不取负)."""
|
||||
assert get_factor("fund_sue_np_strict")["expression"] == "cs_rank(sue_np_strict)"
|
||||
# 负向对照 = 现有 fund_asset_growth(cs_rank(-asset_growth))的方向反转
|
||||
assert get_factor("fund_asset_growth_neg")["expression"] == "cs_rank(asset_growth)"
|
||||
assert "-" not in get_factor("fund_asset_growth_neg")["expression"].replace(
|
||||
"cs_rank(", "").replace("asset_growth)", "")
|
||||
|
||||
|
||||
def test_p1_registration_idempotent():
|
||||
"""重复注册幂等(防重入)."""
|
||||
fundamental_library._register_all()
|
||||
fundamental_library._register_all()
|
||||
assert len(list_factors("fundamental")) == 69
|
||||
Reference in New Issue
Block a user