183 lines
7.7 KiB
Python
183 lines
7.7 KiB
Python
# sanguo_factor/sentiment_adapter.py
|
||
"""情绪因子适配层: corpus news_meta → PIT 日频特征列(S 族,2026-09-22 P2 批).
|
||
|
||
数据面: sanguo_data.corpus_reader.get_news_meta(title+summary 打分面;核内
|
||
零去重→消费方按 art_code 幂等)。词表=sentiment_lexicon(短语优先+计数多数决).
|
||
|
||
口径红线:
|
||
- PIT=次日可见: eff = show_time 所在日 + 1 天(当日盘后新闻泄入当日收盘评估的
|
||
反例不可证,宁晚毋早——与重述可见性 notice=max 同族纪律);
|
||
- 覆盖门(宁缺毋假,UZI 完整性门思想): 窗内新闻量 < MIN_NEWS → NaN 不硬算;
|
||
- 前向积累型: corpus 采集 2026-09-06 起,历史窗无数据=全 NaN(诚实透出,
|
||
IC 验证随语料攒厚度,首个有意义窗=10 月下旬 Q3 披露潮);
|
||
- 窗口=网格行序(交易日网格≈月窗,与月度调仓对齐;日历网格=日历窗).
|
||
|
||
输出列(SENTIMENT_COLUMNS):
|
||
- sent_mom_20 近 20 行窗净情绪 = Σ(pos−neg)/Σn(窗内全新闻聚合,门 n≥5)
|
||
- sent_chg 近 30 行窗净情绪 − 前 30 行窗净情绪(两窗各自门 n≥5)
|
||
- sent_burst 当日新闻量 z 分(对含当日的 20 行窗;门 窗均量≥0.5 且 std>0)
|
||
- news_vol_chg 近 20 行窗量 / 前 20 行窗量 − 1(前窗 0 → NaN)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta
|
||
|
||
import polars as pl
|
||
|
||
from .sentiment_lexicon import score_text
|
||
|
||
SENTIMENT_COLUMNS = ["sent_mom_20", "sent_chg", "sent_burst", "news_vol_chg"]
|
||
|
||
# 回看天数(日历): sent_chg 需前 30 行窗,交易日网格 ≈ 42 日历日,留边
|
||
LOOKBACK_DAYS = 120
|
||
MIN_NEWS = 5 # 覆盖门: 窗内新闻量下限
|
||
|
||
|
||
def _vt_to_bare(vt: str) -> str:
|
||
return vt.partition(".")[0]
|
||
|
||
|
||
def _empty(columns: list[str]) -> pl.DataFrame:
|
||
schema = {"vt_symbol": pl.Utf8, "datetime": pl.Datetime("us"),
|
||
**{c: pl.Float64 for c in columns}}
|
||
return pl.DataFrame(schema=schema)
|
||
|
||
|
||
def _norm_day(v) -> str | None:
|
||
"""show_time 任意形态(str/datetime/NaT)→ 'YYYY-MM-DD';无效 None."""
|
||
if v is None:
|
||
return None
|
||
s = str(v)[:10]
|
||
return s if len(s) == 10 and s[4] == "-" and s[:4].isdigit() else None
|
||
|
||
|
||
def _series_or_blank(news, col: str):
|
||
if col in news.columns:
|
||
return news[col].fillna("")
|
||
import pandas as pd
|
||
return pd.Series([""] * len(news), index=news.index)
|
||
|
||
|
||
def _scored_events(news, bare_to_vt: dict) -> pl.DataFrame:
|
||
"""打分+次日可见 → (vt_symbol, eff, _s) 行集(无效行丢弃)."""
|
||
titles = _series_or_blank(news, "title")
|
||
summaries = _series_or_blank(news, "summary")
|
||
vts, effs, ss = [], [], []
|
||
for t, title, summ, code in zip(news["show_time"], titles, summaries,
|
||
news["stock_code"]):
|
||
day = _norm_day(t)
|
||
vt = bare_to_vt.get(str(code))
|
||
if day is None or vt is None:
|
||
continue
|
||
vts.append(vt)
|
||
effs.append((datetime.strptime(day, "%Y-%m-%d")
|
||
+ timedelta(days=1)).strftime("%Y-%m-%d"))
|
||
ss.append(score_text(f"{title} {summ}"))
|
||
if not vts:
|
||
return pl.DataFrame(schema={"vt_symbol": pl.Utf8, "eff": pl.Utf8,
|
||
"_s": pl.Int64})
|
||
return pl.DataFrame({"vt_symbol": vts, "eff": effs, "_s": ss})
|
||
|
||
|
||
def build_sentiment_features(
|
||
codes: list[str],
|
||
start: str,
|
||
end: str,
|
||
trading_dates: pl.Series | list | None = None,
|
||
columns: list[str] | None = None,
|
||
) -> pl.DataFrame:
|
||
"""构建 PIT 日频情绪特征: vt_symbol × datetime × columns.
|
||
|
||
corpus 根=SANGUO_CORPUS_ROOT(corpus_reader 同源);缺数据/无新闻 → 空 schema
|
||
帧不 raise(batch_eval join 后缺列=全 NaN,与财务域缺域降级同语义)。
|
||
"""
|
||
out_cols = list(columns) if columns is not None else list(SENTIMENT_COLUMNS)
|
||
if not codes:
|
||
return _empty(out_cols)
|
||
try:
|
||
from sanguo_data.corpus_reader import get_news_meta
|
||
except Exception:
|
||
return _empty(out_cols)
|
||
lb_start = (datetime.strptime(start, "%Y-%m-%d")
|
||
- timedelta(days=LOOKBACK_DAYS)).strftime("%Y-%m-%d")
|
||
try:
|
||
news = get_news_meta(start=lb_start, end=end,
|
||
symbols=[_vt_to_bare(c) for c in codes])
|
||
except Exception:
|
||
return _empty(out_cols)
|
||
if news is None or news.empty:
|
||
return _empty(out_cols)
|
||
news = news.drop_duplicates(subset=["art_code"])
|
||
news = news[news["show_time"].notna() & news["stock_code"].notna()]
|
||
if news.empty:
|
||
return _empty(out_cols)
|
||
|
||
scored = _scored_events(news, {_vt_to_bare(c): c for c in codes})
|
||
if scored.height == 0:
|
||
return _empty(out_cols)
|
||
daily = (scored.with_columns(
|
||
pl.col("eff").str.to_date("%Y-%m-%d").alias("_d"))
|
||
.group_by(["vt_symbol", "_d"]).agg(
|
||
# cast Int64: Boolean.sum() 产 UInt32,rolling_sum().over() 组合下
|
||
# 观测到 −1 环回成 2^32−1 的污染(值恰好多出 u32max+1,实测复现);
|
||
# 计数量级极小,Int64 零成本消除整族 u32 滚动边界
|
||
pl.len().cast(pl.Int64).alias("_n"),
|
||
(pl.col("_s") > 0).sum().cast(pl.Int64).alias("_pos"),
|
||
(pl.col("_s") < 0).sum().cast(pl.Int64).alias("_neg")))
|
||
|
||
# 日频 grid(codes × days);count 类缺日=0(事实性无新闻,非插补)
|
||
if trading_dates is not None:
|
||
days = sorted({str(d)[:10] for d in list(trading_dates)})
|
||
else:
|
||
s = datetime.strptime(start, "%Y-%m-%d")
|
||
e = datetime.strptime(end, "%Y-%m-%d")
|
||
days = []
|
||
while s <= e:
|
||
days.append(s.strftime("%Y-%m-%d"))
|
||
s += timedelta(days=1)
|
||
grid = pl.DataFrame({
|
||
"vt_symbol": [c for c in codes for _ in days],
|
||
"_d": days * len(codes),
|
||
}).with_columns(pl.col("_d").str.to_date("%Y-%m-%d"))
|
||
df = (grid.join(daily, on=["vt_symbol", "_d"], how="left")
|
||
.with_columns(
|
||
pl.col("_n").fill_null(0), pl.col("_pos").fill_null(0),
|
||
pl.col("_neg").fill_null(0))
|
||
.sort(["vt_symbol", "_d"]))
|
||
|
||
def _rsum(col: str, w: int, shift: int = 0) -> pl.Expr:
|
||
e = pl.col(col).shift(shift) if shift else pl.col(col)
|
||
return e.rolling_sum(w).over("vt_symbol")
|
||
|
||
n = pl.col("_n")
|
||
df = df.with_columns(
|
||
_rsum("_pos", 20).alias("_p20"), _rsum("_neg", 20).alias("_n20"),
|
||
_rsum("_n", 20).alias("_c20"),
|
||
_rsum("_pos", 30).alias("_p30"), _rsum("_neg", 30).alias("_n30"),
|
||
_rsum("_n", 30).alias("_c30"),
|
||
_rsum("_pos", 30, 30).alias("_p30p"), _rsum("_neg", 30, 30).alias("_n30p"),
|
||
_rsum("_n", 30, 30).alias("_c30p"),
|
||
_rsum("_n", 20, 20).alias("_c20p"),
|
||
n.rolling_mean(20).over("vt_symbol").alias("_m20"),
|
||
n.rolling_std(20).over("vt_symbol").alias("_sd20"),
|
||
)
|
||
net20 = pl.when(pl.col("_c20") >= MIN_NEWS).then(
|
||
(pl.col("_p20") - pl.col("_n20")) / pl.col("_c20")).otherwise(None)
|
||
net30 = pl.when(pl.col("_c30") >= MIN_NEWS).then(
|
||
(pl.col("_p30") - pl.col("_n30")) / pl.col("_c30")).otherwise(None)
|
||
net30p = pl.when(pl.col("_c30p") >= MIN_NEWS).then(
|
||
(pl.col("_p30p") - pl.col("_n30p")) / pl.col("_c30p")).otherwise(None)
|
||
df = df.with_columns(
|
||
net20.alias("sent_mom_20"),
|
||
(net30 - net30p).alias("sent_chg"),
|
||
pl.when((pl.col("_m20") >= 0.5) & (pl.col("_sd20") > 0) & (n > 0))
|
||
.then((n - pl.col("_m20")) / pl.col("_sd20")).otherwise(None)
|
||
.alias("sent_burst"),
|
||
pl.when(pl.col("_c20p") > 0)
|
||
.then(pl.col("_c20") / pl.col("_c20p") - 1.0).otherwise(None)
|
||
.alias("news_vol_chg"),
|
||
)
|
||
return (df.with_columns(pl.col("_d").cast(pl.Datetime("us")).alias("datetime"))
|
||
.sort(["vt_symbol", "datetime"])
|
||
.select(["vt_symbol", "datetime", *out_cols]))
|