From 9977ff6e2edce75de7d07a2ae937bfa876c2b661 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 26 Aug 2026 03:53:30 +0800 Subject: [PATCH] =?UTF-8?q?perf(factor):=20=E8=AF=84=E4=BC=B0=E5=86=85?= =?UTF-8?q?=E5=AD=98=E5=87=8F=E8=B4=9F=E2=80=94=E2=80=94bars=E5=8E=9F?= =?UTF-8?q?=E5=A7=8B=E5=B8=A7=E6=B4=BE=E7=94=9F=E5=90=8E=E5=8D=B3=E9=87=8A?= =?UTF-8?q?=E6=94=BE(~1G)+alpha=5Fdf=E5=BC=83=E6=97=A0=E5=BC=95=E7=94=A8op?= =?UTF-8?q?en=5Finterest=E5=88=97(80M),NAS=208G=E7=9B=92=E9=98=B2=E5=91=A8?= =?UTF-8?q?=E6=9C=9F=E6=80=A7OOM=20[vps]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sanguo_factor/batch_eval.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sanguo_factor/batch_eval.py b/sanguo_factor/batch_eval.py index d522c94..c0815ab 100644 --- a/sanguo_factor/batch_eval.py +++ b/sanguo_factor/batch_eval.py @@ -62,7 +62,7 @@ def run_batch_eval( raise ValueError(f"股票池为空: vnpy_db={vnpy_db} window={start}~{end}") alpha_df = bars.select(["vt_symbol", "datetime", "open", "high", "low", "close", - "volume", "turnover", "open_interest", "vwap"]) + "volume", "turnover", "vwap"]) # Pre-compute per-symbol warmup cutoff dates (bar_idx >= WARMUP_BARS 的首日) # 用于替代 per-factor hash join，改为 pivot 后 pandas 广播掩码置 NaN cutoffs = ( @@ -81,12 +81,20 @@ def run_batch_eval( close_wide.index = pd.to_datetime(close_wide.index) rets = _forward_return_matrices(close_wide) + # 保存symbols_count(删除bars前) + symbols_count = bars["vt_symbol"].n_unique() + + # 内存减负:bars(原始+bar_idx ~1G)派生完毕即释放——NAS 8G 盒防 OOM 周期性被杀 + import gc + del bars + gc.collect() + universe_label = universe if symbols is None else "custom" eval_store.init_db(eval_db) if run_id is None: run_id = eval_store.create_run( eval_db, label=label, universe=universe_label, - symbols_count=bars["vt_symbol"].n_unique(), factors_total=len(factor_names), + symbols_count=symbols_count, factors_total=len(factor_names), start=start, end=end, params={"limit": limit, "symbols": symbols[:20] if symbols else None}, # 截断防 4400 只全量塞 JSON ) done_before = set() @@ -104,7 +112,7 @@ def run_batch_eval( "factors_done": len(done_before), "errors": [], "elapsed_sec": round(time.time() - t0, 1), - "symbols_count": int(bars["vt_symbol"].n_unique()), + "symbols_count": int(symbols_count), } errors: list[str] = [] @@ -130,7 +138,7 @@ def run_batch_eval( "factors_done": len(done_before) + done, "errors": errors, "elapsed_sec": round(time.time() - t0, 1), - "symbols_count": int(bars["vt_symbol"].n_unique()), + "symbols_count": int(symbols_count), }