fix(factor): load段内存峰值根治——chunks在concat后即清+transform链lazy单次物化,RSS监控实锤6.6GB被kernel周期性OOM击杀,eager逐步链3-4倍峰值收敛到~1.5倍 [vps]
This commit is contained in:
@@ -84,8 +84,11 @@ def load_universe_bars(
|
||||
return _empty_alpha_df()
|
||||
|
||||
df = pl.concat(chunks)
|
||||
chunks.clear() # 立即释放逐symbol块(~1G),勿持有到函数尾
|
||||
# transform 链 lazy 单次物化:eager 逐步链每步全量拷贝,峰值3-4倍;lazy 融合后~1.5倍
|
||||
df = (
|
||||
df.with_columns(
|
||||
df.lazy()
|
||||
.with_columns(
|
||||
pl.col("dt").str.slice(0, 10).str.to_datetime("%Y-%m-%d").alias("datetime"),
|
||||
(pl.col("symbol") + "." + pl.col("exchange")).alias("vt_symbol"),
|
||||
pl.when(pl.col("volume") > 0)
|
||||
@@ -97,6 +100,7 @@ def load_universe_bars(
|
||||
.with_columns(pl.int_range(pl.len()).over("vt_symbol").alias("bar_idx"))
|
||||
.select(["vt_symbol", "datetime", "open", "high", "low", "close",
|
||||
"volume", "turnover", "open_interest", "vwap", "bar_idx"])
|
||||
.collect()
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
Reference in New Issue
Block a user