perf(factor): 股票池枚举改无过滤DISTINCT覆盖索引扫——带WHERE(interval/datetime/LIKE)版本在26G库退化全表扫NAS实测>5min不归,前缀python侧滤,周期/窗口由数据查询天然过滤(语义不变,15m-only股锁定测试) [vps]
This commit is contained in:
@@ -35,15 +35,14 @@ def load_universe_bars(
|
||||
).strftime("%Y-%m-%d")
|
||||
|
||||
if symbols is None:
|
||||
conn = sqlite3.connect(vnpy_db, timeout=30)
|
||||
conn = sqlite3.connect(vnpy_db, timeout=60)
|
||||
try:
|
||||
likes = " OR ".join(f"symbol LIKE '{p}%'" for p in STOCK_PREFIXES)
|
||||
cur = conn.execute(
|
||||
f"SELECT DISTINCT symbol FROM dbbardata "
|
||||
f"WHERE interval='d' AND datetime>=? AND datetime<=? AND ({likes})",
|
||||
(lookback_start, forward_end),
|
||||
)
|
||||
symbols = [r[0] for r in cur.fetchall()]
|
||||
# 无过滤 DISTINCT symbol 走 (symbol,...) 前导索引顺序流式扫——
|
||||
# 带 WHERE(interval/datetime/LIKE)的版本会退化为 26G 全表扫(NAS 实测>5min)。
|
||||
# 前缀在 Python 侧滤;interval='d'/窗口过滤由下方分块数据查询天然承担
|
||||
# (无日线数据的 symbol 返回 0 行,不进最终 df,语义不变)。
|
||||
cur = conn.execute("SELECT DISTINCT symbol FROM dbbardata")
|
||||
symbols = [r[0] for r in cur if str(r[0]).startswith(STOCK_PREFIXES)]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user