perf(factor): 股票池枚举改无过滤DISTINCT覆盖索引扫——带WHERE(interval/datetime/LIKE)版本在26G库退化全表扫NAS实测>5min不归,前缀python侧滤,周期/窗口由数据查询天然过滤(语义不变,15m-only股锁定测试) [vps]

sanguo_factor/universe.py

@@ -35,15 +35,14 @@ def load_universe_bars(
     ).strftime("%Y-%m-%d")
 
     if symbols is None:
-        conn = sqlite3.connect(vnpy_db, timeout=30)
+        conn = sqlite3.connect(vnpy_db, timeout=60)
         try:
-            likes = " OR ".join(f"symbol LIKE '{p}%'" for p in STOCK_PREFIXES)
+            # 无过滤 DISTINCT symbol 走 (symbol,...) 前导索引顺序流式扫——
-            cur = conn.execute(
+            # 带 WHERE(interval/datetime/LIKE)的版本会退化为 26G 全表扫(NAS 实测>5min)。
-                f"SELECT DISTINCT symbol FROM dbbardata "
+            # 前缀在 Python 侧滤;interval='d'/窗口过滤由下方分块数据查询天然承担
-                f"WHERE interval='d' AND datetime>=? AND datetime<=? AND ({likes})",
+            # (无日线数据的 symbol 返回 0 行,不进最终 df,语义不变)。
-                (lookback_start, forward_end),
+            cur = conn.execute("SELECT DISTINCT symbol FROM dbbardata")
-            )
+            symbols = [r[0] for r in cur if str(r[0]).startswith(STOCK_PREFIXES)]
-            symbols = [r[0] for r in cur.fetchall()]
         finally:
             conn.close()
 

tests/factor/test_universe.py

@@ -49,6 +49,8 @@ def db(tmp_path):
     rows.append(_row("510300", "SSE", "2018-01-02", 4.0))
     # 非日线 interval 应忽略
     rows.append(("600000", "SSE", "2018-01-02 09:35:00", "15m", 1, 1, 0, 1, 1, 1, 1))
+    # 只有 15m 数据、无日线的 symbol:枚举会带上但数据查询 0 行,不应出现在结果
+    rows.append(("159915", "SZSE", "2018-01-02 09:35:00", "15m", 100.0, 100000.0, 0, 1, 1, 1, 1))
     return _mk_db(tmp_path, rows)
 
 
@@ -90,3 +92,8 @@ def test_limit_deterministic(db):
     df1 = load_universe_bars(db, "2018-01-01", "2018-01-31", limit=1)
     df2 = load_universe_bars(db, "2018-01-01", "2018-01-31", limit=1)
     assert set(df1["vt_symbol"].unique()) == set(df2["vt_symbol"].unique())
+
+
+def test_minute_only_symbol_excluded(db):
+    df = load_universe_bars(db, "2018-01-01", "2018-01-31")
+    assert "159915.SZSE" not in set(df["vt_symbol"].unique().to_list())