perf(factor): 股票池枚举改无过滤DISTINCT覆盖索引扫——带WHERE(interval/datetime/LIKE)版本在26G库退化全表扫NAS实测>5min不归,前缀python侧滤,周期/窗口由数据查询天然过滤(语义不变,15m-only股锁定测试) [vps]

This commit is contained in:
2026-08-25 01:01:48 +08:00
parent cdaa88fea1
commit 0d038548ae
2 changed files with 14 additions and 8 deletions
+7 -8
View File
@@ -35,15 +35,14 @@ def load_universe_bars(
).strftime("%Y-%m-%d")
if symbols is None:
conn = sqlite3.connect(vnpy_db, timeout=30)
conn = sqlite3.connect(vnpy_db, timeout=60)
try:
likes = " OR ".join(f"symbol LIKE '{p}%'" for p in STOCK_PREFIXES)
cur = conn.execute(
f"SELECT DISTINCT symbol FROM dbbardata "
f"WHERE interval='d' AND datetime>=? AND datetime<=? AND ({likes})",
(lookback_start, forward_end),
)
symbols = [r[0] for r in cur.fetchall()]
# 无过滤 DISTINCT symbol 走 (symbol,...) 前导索引顺序流式扫——
# 带 WHERE(interval/datetime/LIKE)的版本会退化为 26G 全表扫(NAS 实测>5min)。
# 前缀在 Python 侧滤;interval='d'/窗口过滤由下方分块数据查询天然承担
# (无日线数据的 symbol 返回 0 行,不进最终 df,语义不变)。
cur = conn.execute("SELECT DISTINCT symbol FROM dbbardata")
symbols = [r[0] for r in cur if str(r[0]).startswith(STOCK_PREFIXES)]
finally:
conn.close()