From 0d038548ae348afea67f6fe20807ef8b5ea2b9ce Mon Sep 17 00:00:00 2001 From: claude_dev Date: Tue, 25 Aug 2026 01:01:48 +0800 Subject: [PATCH] =?UTF-8?q?perf(factor):=20=E8=82=A1=E7=A5=A8=E6=B1=A0?= =?UTF-8?q?=E6=9E=9A=E4=B8=BE=E6=94=B9=E6=97=A0=E8=BF=87=E6=BB=A4DISTINCT?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E7=B4=A2=E5=BC=95=E6=89=AB=E2=80=94=E2=80=94?= =?UTF-8?q?=E5=B8=A6WHERE(interval/datetime/LIKE)=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=9C=A826G=E5=BA=93=E9=80=80=E5=8C=96=E5=85=A8=E8=A1=A8?= =?UTF-8?q?=E6=89=ABNAS=E5=AE=9E=E6=B5=8B>5min=E4=B8=8D=E5=BD=92,=E5=89=8D?= =?UTF-8?q?=E7=BC=80python=E4=BE=A7=E6=BB=A4,=E5=91=A8=E6=9C=9F/=E7=AA=97?= =?UTF-8?q?=E5=8F=A3=E7=94=B1=E6=95=B0=E6=8D=AE=E6=9F=A5=E8=AF=A2=E5=A4=A9?= =?UTF-8?q?=E7=84=B6=E8=BF=87=E6=BB=A4(=E8=AF=AD=E4=B9=89=E4=B8=8D?= =?UTF-8?q?=E5=8F=98,15m-only=E8=82=A1=E9=94=81=E5=AE=9A=E6=B5=8B=E8=AF=95?= =?UTF-8?q?)=20[vps]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sanguo_factor/universe.py | 15 +++++++-------- tests/factor/test_universe.py | 7 +++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/sanguo_factor/universe.py b/sanguo_factor/universe.py index b52f00a..ce7fd13 100644 --- a/sanguo_factor/universe.py +++ b/sanguo_factor/universe.py @@ -35,15 +35,14 @@ def load_universe_bars( ).strftime("%Y-%m-%d") if symbols is None: - conn = sqlite3.connect(vnpy_db, timeout=30) + conn = sqlite3.connect(vnpy_db, timeout=60) try: - likes = " OR ".join(f"symbol LIKE '{p}%'" for p in STOCK_PREFIXES) - cur = conn.execute( - f"SELECT DISTINCT symbol FROM dbbardata " - f"WHERE interval='d' AND datetime>=? AND datetime<=? AND ({likes})", - (lookback_start, forward_end), - ) - symbols = [r[0] for r in cur.fetchall()] + # 无过滤 DISTINCT symbol 走 (symbol,...) 前导索引顺序流式扫—— + # 带 WHERE(interval/datetime/LIKE)的版本会退化为 26G 全表扫(NAS 实测>5min)。 + # 前缀在 Python 侧滤;interval='d'/窗口过滤由下方分块数据查询天然承担 + # (无日线数据的 symbol 返回 0 行,不进最终 df,语义不变)。 + cur = conn.execute("SELECT DISTINCT symbol FROM dbbardata") + symbols = [r[0] for r in cur if str(r[0]).startswith(STOCK_PREFIXES)] finally: conn.close() diff --git a/tests/factor/test_universe.py b/tests/factor/test_universe.py index 31defc1..7777a3a 100644 --- a/tests/factor/test_universe.py +++ b/tests/factor/test_universe.py @@ -49,6 +49,8 @@ def db(tmp_path): rows.append(_row("510300", "SSE", "2018-01-02", 4.0)) # 非日线 interval 应忽略 rows.append(("600000", "SSE", "2018-01-02 09:35:00", "15m", 1, 1, 0, 1, 1, 1, 1)) + # 只有 15m 数据、无日线的 symbol:枚举会带上但数据查询 0 行,不应出现在结果 + rows.append(("159915", "SZSE", "2018-01-02 09:35:00", "15m", 100.0, 100000.0, 0, 1, 1, 1, 1)) return _mk_db(tmp_path, rows) @@ -90,3 +92,8 @@ def test_limit_deterministic(db): df1 = load_universe_bars(db, "2018-01-01", "2018-01-31", limit=1) df2 = load_universe_bars(db, "2018-01-01", "2018-01-31", limit=1) assert set(df1["vt_symbol"].unique()) == set(df2["vt_symbol"].unique()) + + +def test_minute_only_symbol_excluded(db): + df = load_universe_bars(db, "2018-01-01", "2018-01-31") + assert "159915.SZSE" not in set(df["vt_symbol"].unique().to_list())