diff --git a/sanguo_factor/universe.py b/sanguo_factor/universe.py index b52f00a..ce7fd13 100644 --- a/sanguo_factor/universe.py +++ b/sanguo_factor/universe.py @@ -35,15 +35,14 @@ def load_universe_bars( ).strftime("%Y-%m-%d") if symbols is None: - conn = sqlite3.connect(vnpy_db, timeout=30) + conn = sqlite3.connect(vnpy_db, timeout=60) try: - likes = " OR ".join(f"symbol LIKE '{p}%'" for p in STOCK_PREFIXES) - cur = conn.execute( - f"SELECT DISTINCT symbol FROM dbbardata " - f"WHERE interval='d' AND datetime>=? AND datetime<=? AND ({likes})", - (lookback_start, forward_end), - ) - symbols = [r[0] for r in cur.fetchall()] + # 无过滤 DISTINCT symbol 走 (symbol,...) 前导索引顺序流式扫—— + # 带 WHERE(interval/datetime/LIKE)的版本会退化为 26G 全表扫(NAS 实测>5min)。 + # 前缀在 Python 侧滤;interval='d'/窗口过滤由下方分块数据查询天然承担 + # (无日线数据的 symbol 返回 0 行,不进最终 df,语义不变)。 + cur = conn.execute("SELECT DISTINCT symbol FROM dbbardata") + symbols = [r[0] for r in cur if str(r[0]).startswith(STOCK_PREFIXES)] finally: conn.close() diff --git a/tests/factor/test_universe.py b/tests/factor/test_universe.py index 31defc1..7777a3a 100644 --- a/tests/factor/test_universe.py +++ b/tests/factor/test_universe.py @@ -49,6 +49,8 @@ def db(tmp_path): rows.append(_row("510300", "SSE", "2018-01-02", 4.0)) # 非日线 interval 应忽略 rows.append(("600000", "SSE", "2018-01-02 09:35:00", "15m", 1, 1, 0, 1, 1, 1, 1)) + # 只有 15m 数据、无日线的 symbol:枚举会带上但数据查询 0 行,不应出现在结果 + rows.append(("159915", "SZSE", "2018-01-02 09:35:00", "15m", 100.0, 100000.0, 0, 1, 1, 1, 1)) return _mk_db(tmp_path, rows) @@ -90,3 +92,8 @@ def test_limit_deterministic(db): df1 = load_universe_bars(db, "2018-01-01", "2018-01-31", limit=1) df2 = load_universe_bars(db, "2018-01-01", "2018-01-31", limit=1) assert set(df1["vt_symbol"].unique()) == set(df2["vt_symbol"].unique()) + + +def test_minute_only_symbol_excluded(db): + df = load_universe_bars(db, "2018-01-01", "2018-01-31") + assert "159915.SZSE" not in set(df["vt_symbol"].unique().to_list())