206 lines
9.2 KiB
Python
206 lines
9.2 KiB
Python
# sanguo_factor/challenger_lgbm.py
|
|
"""LGBM challenger 三路加权对拍(2026-10-10 spec §4.4 定案).
|
|
|
|
三路=等权(方向调整)/ICIR(quant12_icirfit_v1 档案)/LightGBM(月度重训),
|
|
同池同窗 walk-forward 影子对拍;challenger 永不进生产,对拍件落
|
|
reports/factor_monthly/challenger_lgbm/ 子目录(判定层端点只读根层).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
_PROFILE = Path(__file__).parent / "weight_profiles" / "quant12_icirfit_v1.json"
|
|
|
|
|
|
def load_pool() -> dict[str, dict]:
|
|
"""v1 对拍池=在役 12 源(ICIR 档案现成,三路同池才可比;扩池挂后续)."""
|
|
with open(_PROFILE, encoding="utf-8") as f:
|
|
return json.load(f)["sources"]
|
|
|
|
|
|
def build_label(close_wide: pd.DataFrame) -> pd.DataFrame:
|
|
"""防缺口 label=T+1 收盘→T+2 收盘(信号 T 收盘出,T+1 全天可成交)."""
|
|
return close_wide.shift(-2) / close_wide.shift(-1) - 1.0
|
|
|
|
|
|
def cs_rank_norm(df: pd.DataFrame) -> pd.DataFrame:
|
|
"""截面秩归一 (rank_pct-0.5);NaN 透传不占当日截面."""
|
|
return df.rank(axis=1, pct=True) - 0.5
|
|
|
|
|
|
LGBM_PARAMS = { # qlib Alpha158 基准超参起步(spec §4.4;无独立 valid 段,v1 固定轮数)
|
|
"objective": "mse", "learning_rate": 0.1, "max_depth": 8, "num_leaves": 210,
|
|
"colsample_bytree": 0.8879, "subsample": 0.8789,
|
|
"lambda_l1": 205.6999, "lambda_l2": 580.9768,
|
|
"min_child_samples": 100, "feature_fraction_bynode": 0.8,
|
|
"seed": 42, "num_threads": 4, "verbose": -1,
|
|
}
|
|
NUM_BOOST_ROUND = 500
|
|
|
|
|
|
def _aligned(values: dict[str, pd.DataFrame]) -> pd.DataFrame:
|
|
"""多因子宽表纵向拼接成特征长表(index 对齐,缺失源列 NaN)."""
|
|
return pd.concat({n: cs_rank_norm(v) for n, v in values.items()}, axis=1)
|
|
|
|
|
|
def _dir_sign(pool: dict, name: str) -> float:
|
|
return -1.0 if pool.get(name, {}).get("direction") == "-" else 1.0
|
|
|
|
|
|
def equal_weight_signal(values: dict[str, pd.DataFrame], pool: dict) -> pd.DataFrame:
|
|
stack = pd.concat([_dir_sign(pool, n) * cs_rank_norm(v) for n, v in values.items()])
|
|
return stack.groupby(level=0).mean()
|
|
|
|
|
|
def icir_signal(values: dict[str, pd.DataFrame], pool: dict) -> pd.DataFrame:
|
|
total = sum(pool[n].get("weight", 0.0) for n in values)
|
|
if total <= 0:
|
|
raise ValueError("ICIR 权重和为零,档案异常")
|
|
out = None
|
|
for n, v in values.items():
|
|
w = pool[n].get("weight", 0.0) * _dir_sign(pool, n)
|
|
part = w * cs_rank_norm(v)
|
|
out = part if out is None else out.add(part, fill_value=0.0)
|
|
return out / total
|
|
|
|
|
|
def lgbm_walk_forward(values: dict[str, pd.DataFrame], label: pd.DataFrame,
|
|
last_month: str, params: dict | None = None) -> tuple[pd.DataFrame, dict]:
|
|
"""训练窗=last_month 之前全部;样外=last_month 当月(test 隔离铁律).
|
|
|
|
返回 (样外日×股票预测宽表, {feature: gain}).特征=CSRankNorm 后各源,
|
|
label=CSRankNorm 后防缺口收益;日频截面样本(日期,股票)平铺训练.
|
|
"""
|
|
import lightgbm as lgb
|
|
|
|
feat = _aligned(values)
|
|
lab = cs_rank_norm(label)
|
|
common = feat.index.intersection(lab.index)
|
|
feat, lab = feat.loc[common], lab.loc[common]
|
|
|
|
oos_mask = feat.index.strftime("%Y-%m") == last_month
|
|
train_mask = ~oos_mask
|
|
X_tr = feat[train_mask].stack(future_stack=True).reset_index()
|
|
X_tr.columns = ["datetime", "vt_symbol", *feat.columns.levels[0]]
|
|
y_df = lab.stack(future_stack=True).rename("y").reset_index()
|
|
y_df.columns = ["datetime", "vt_symbol", "y"] # index 无名时 reset 生成 level_0/1,显式定名
|
|
tr = X_tr.merge(y_df, on=["datetime", "vt_symbol"]).dropna()
|
|
X = tr[list(feat.columns.levels[0])]
|
|
model = lgb.train(params or LGBM_PARAMS, lgb.Dataset(X, label=tr["y"]),
|
|
num_boost_round=NUM_BOOST_ROUND)
|
|
imp = dict(zip(X.columns, model.feature_importance("gain").tolist()))
|
|
|
|
X_oos = feat[oos_mask].stack(future_stack=True).reset_index()
|
|
X_oos.columns = X_tr.columns
|
|
X_oos = X_oos.dropna(subset=list(feat.columns.levels[0]))
|
|
X_oos = X_oos.merge(y_df.dropna()[["datetime", "vt_symbol"]],
|
|
on=["datetime", "vt_symbol"]) # 只留 label 有效对(可评分样外)
|
|
if X_oos.empty:
|
|
return pd.DataFrame(), imp
|
|
preds = model.predict(X_oos[list(feat.columns.levels[0])])
|
|
out = X_oos[["datetime", "vt_symbol"]].assign(p=preds)
|
|
return out.pivot(index="datetime", columns="vt_symbol", values="p"), imp
|
|
|
|
|
|
def _load_close_wide(vnpy_db: str, columns_ref: pd.Index, start: str, end: str) -> pd.DataFrame:
|
|
"""按因子宽表列(股票)拉收盘价.生产实现走 universe.load_universe_bars 轻量列.
|
|
|
|
真实 loader 必传 start/end(universe.py:45);+45d 前向缓冲覆盖 label 的 t+2.
|
|
"""
|
|
from sanguo_data.config import find_config_path, load_config
|
|
from .universe import load_universe_bars
|
|
|
|
cfg = load_config(find_config_path())
|
|
bars = load_universe_bars(vnpy_db or cfg.data_paths["vnpy_db"], start, end,
|
|
symbols=list(columns_ref))
|
|
_ = bars.select(["datetime", "vt_symbol", "close"]).to_pandas()
|
|
wide = _.pivot(index="datetime", columns="vt_symbol", values="close").sort_index()
|
|
wide.index = pd.to_datetime(wide.index)
|
|
return wide
|
|
|
|
|
|
def score_signal(signal: pd.DataFrame, label: pd.DataFrame) -> dict:
|
|
"""样外评分:日 IC 均值/ICIR/五分位多空累计/有效天数.
|
|
|
|
days=进评分的天数(截面≥5 对);IC 全 NaN(常数信号)时 ic_mean/icir=None.
|
|
"""
|
|
common = signal.index.intersection(label.index)
|
|
ics = []
|
|
ls_rets = []
|
|
for d in common:
|
|
s, y = signal.loc[d], label.loc[d]
|
|
pair = pd.concat([s, y], axis=1, keys=["s", "y"]).dropna()
|
|
if len(pair) < 5:
|
|
continue
|
|
ics.append(pair["s"].corr(pair["y"], method="spearman"))
|
|
q = pair["s"].quantile([0.2, 0.8])
|
|
lo, hi = pair[pair["s"] <= q[0.2]]["y"].mean(), pair[pair["s"] >= q[0.8]]["y"].mean()
|
|
ls_rets.append((hi - lo) if (lo is not None and hi is not None) else 0.0)
|
|
if not ics:
|
|
return {"ic_mean": None, "icir": None, "q5q1": None, "days": 0}
|
|
ser = pd.Series(ics).dropna()
|
|
icir = (ser.mean() / ser.std()) if len(ser) > 1 and ser.std() > 0 else None
|
|
return {"ic_mean": round(float(ser.mean()), 6) if len(ser) else None,
|
|
"icir": round(float(icir), 6) if icir is not None else None,
|
|
"q5q1": round(float(sum(ls_rets)), 6), "days": len(ics)}
|
|
|
|
|
|
def run_challenge(values_dir: str, vnpy_db: str, as_of: str, out_dir: str,
|
|
host: str = "nas") -> str:
|
|
"""月度链 stage6 入口:读导出宽表→三路→对拍件(append-only 子目录)."""
|
|
pool = load_pool()
|
|
names = sorted(pool)
|
|
values = {n: pd.read_parquet(os.path.join(values_dir, f"{n}.parquet"))
|
|
for n in names if os.path.exists(os.path.join(values_dir, f"{n}.parquet"))}
|
|
if not values:
|
|
raise FileNotFoundError(f"因子宽表目录无池内因子: {values_dir}")
|
|
cols_ref = next(iter(values.values())).columns
|
|
start = min(v.index.min() for v in values.values()).strftime("%Y-%m-%d")
|
|
end = max(v.index.max() for v in values.values()).strftime("%Y-%m-%d")
|
|
close = _load_close_wide(vnpy_db, cols_ref, start, end)
|
|
label = build_label(close)
|
|
last_month = as_of[:7]
|
|
|
|
signals = {"equal": equal_weight_signal(values, pool),
|
|
"icir": icir_signal(values, pool)}
|
|
pred, imp = lgbm_walk_forward(values, label, last_month)
|
|
if not pred.empty:
|
|
signals["lgbm"] = pred
|
|
|
|
oos_label = label[label.index.strftime("%Y-%m") == last_month]
|
|
scored = {k: score_signal(v, oos_label) for k, v in signals.items()}
|
|
|
|
# 池冗余注记:|corr|>0.7 只记录不剔除(首年观察期,剔除留人).
|
|
# 按日构造 股票×因子 矩阵→corr=因子×因子(源间相关,plan 原稿 groupby
|
|
# level=0 按因子分组会算成股票对,与下游 n1<n2 因子对语义矛盾,已重构)
|
|
by_date = pd.concat({n: cs_rank_norm(v).stack(future_stack=True)
|
|
for n, v in values.items()}, axis=1)
|
|
daily_corr = by_date.groupby(level=0).apply(
|
|
lambda g: g.corr(method="spearman") if g.shape[0] > 1 else None)
|
|
flagged = []
|
|
if daily_corr is not None and not daily_corr.empty:
|
|
means = daily_corr.groupby(level=1).mean()
|
|
for n1 in means.index:
|
|
for n2 in means.columns:
|
|
if n1 < n2 and abs(means.loc[n1, n2]) > 0.7:
|
|
flagged.append({"a": n1, "b": n2,
|
|
"corr": round(float(means.loc[n1, n2]), 4)})
|
|
|
|
doc = {"as_of": as_of, "generated_at": pd.Timestamp.now().isoformat(),
|
|
"host": host, "pool": {"names": sorted(values), "size": len(values)},
|
|
"split": {"oos_month": last_month,
|
|
"note": "训练=样外月前全部(test 隔离铁律);无独立 valid,固定轮数"},
|
|
"signals": scored, "feature_importance": imp,
|
|
"redundancy_flagged": flagged,
|
|
"lgbm_params": {**LGBM_PARAMS, "num_boost_round": NUM_BOOST_ROUND}}
|
|
sub = os.path.join(out_dir, "challenger_lgbm")
|
|
os.makedirs(sub, exist_ok=True)
|
|
path = os.path.join(sub, f"{host}_{as_of}.json")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump(doc, f, ensure_ascii=False, indent=2)
|
|
return path
|