From aafa5baa6728a958a53664f5f9a2d020902836e6 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Mon, 24 Aug 2026 14:14:11 +0800 Subject: [PATCH] =?UTF-8?q?perf(data):=20min/max=E6=94=B9=E9=80=90?= =?UTF-8?q?=E5=AF=B9=E5=8F=8Cseek=E2=80=94=E2=80=94INDEXED=20BY=E6=89=B9IN?= =?UTF-8?q?=E5=BD=A2=E6=80=81VPS=E5=AE=9E=E6=B5=8B=E4=BB=8D=E4=B8=8D?= =?UTF-8?q?=E8=BE=BE=E6=A0=87(30=E5=8F=AA=E5=86=B78.1s/=E7=83=AD0.25s,?= =?UTF-8?q?=E5=A4=A7=E6=B1=A0=E5=86=B7=E6=80=81=E5=88=86=E9=92=9F=E7=BA=A7?= =?UTF-8?q?),ORDER=20BY=20=C2=B1datetime=20LIMIT=201=E4=B8=89=E5=88=97?= =?UTF-8?q?=E5=85=A8=E7=AD=89=E5=89=8D=E7=BC=80=E6=AF=8F=E8=82=A1=E6=81=B0?= =?UTF-8?q?2=E6=AC=A1=E5=AF=BB=E4=BD=8D=E5=86=B7=E7=83=AD=E7=9A=86?= =?UTF-8?q?=E6=AF=AB=E7=A7=92=E7=BA=A7=E2=80=94=E2=80=94=E4=B8=8A=E4=B8=80?= =?UTF-8?q?=E5=8F=91(80fc517)=E7=94=9F=E4=BA=A7=E6=8E=A2=E9=92=88=E7=BB=AD?= =?UTF-8?q?=E6=B5=8B=E5=AE=9A=E5=BD=A2=E6=80=81:=E2=91=A0=E5=8E=9FIN+GROUP?= =?UTF-8?q?=20BY=E5=8A=A3=E5=8C=96=E8=AE=A1=E5=88=92101s=E5=86=B7;?= =?UTF-8?q?=E2=91=A1=E9=92=89=E5=A4=8D=E5=90=88=E7=B4=A2=E5=BC=95=E5=90=8E?= =?UTF-8?q?=E8=AE=A1=E5=88=92=E7=BF=BB=E8=BD=AC(TEMP=20B-TREE=E6=B6=88?= =?UTF-8?q?=E5=A4=B1)=E4=BD=86=E4=BB=8D=E9=80=90=E8=82=A1=E6=89=AB?= =?UTF-8?q?=E5=85=A8=E9=83=A8interval=E6=9D=A1=E7=9B=AE(d+15m+5m),30?= =?UTF-8?q?=E5=8F=AA=E5=86=B78.1s=E2=86=92momentum=E6=B1=A0~2000=E5=8F=AA?= =?UTF-8?q?=E5=A4=96=E6=8E=A8=E5=86=B7=E6=80=81=E2=89=889min=E8=BF=87?= =?UTF-8?q?=E4=B8=8D=E4=BA=86<60s=E9=AA=8C=E6=94=B6;=E2=91=A2=E8=A1=8C?= =?UTF-8?q?=E5=80=BCIN=E5=8F=98=E4=BD=93=E5=9C=A8=E6=9C=ACSQLite=E5=AF=B9(?= =?UTF-8?q?row-value=20IN+INDEXED=20BY)=E8=B5=B0=E5=85=A8=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E6=89=AB=E6=8F=8F7min=E4=B8=8D=E5=BD=92(=E5=BC=83);?= =?UTF-8?q?=E7=BB=88=E6=80=81=3D=E9=80=90=E5=AF=B9=E5=8F=8Cseek,symbol/exc?= =?UTF-8?q?hange/interval=E4=B8=89=E5=88=97=E5=85=A8=E7=AD=89=E7=9B=B4?= =?UTF-8?q?=E8=90=BDd=E5=AD=90=E5=8C=BA=E9=97=B4=E4=B8=A4=E7=AB=AF,?= =?UTF-8?q?=E8=BF=9E15m/5m=E6=9D=A1=E7=9B=AE=E9=83=BD=E4=B8=8D=E7=A2=B0;?= =?UTF-8?q?=E6=97=A0=E6=97=A5=E7=BA=BF=E5=AF=B9=E4=B8=8D=E8=90=BDminmax(?= =?UTF-8?q?=E4=B8=8B=E6=B8=B8.get=E9=BB=98=E8=AE=A4(None,None)=E5=90=8C?= =?UTF-8?q?=E6=97=A7GROUP=20BY=E8=AF=AD=E4=B9=89);=E6=B5=8B=E8=AF=955?= =?UTF-8?q?=E4=B8=AA=E9=87=8D=E5=86=99=E5=88=B0seek=20API(=E5=8F=8C?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E5=90=8D/=E5=9B=9E=E9=80=80/EXPLAIN=E8=AE=A1?= =?UTF-8?q?=E5=88=92=E7=AD=BE=E5=90=8Dinterval=3D=3F=E7=9B=B4=E8=90=BD+?= =?UTF-8?q?=E6=97=A0TEMP=20B-TREE/batch=3D=3D=E9=80=90=E5=8F=AA=E7=AD=89?= =?UTF-8?q?=E4=BB=B7);628=E7=BB=BF;=E5=BE=85VPS=E6=8E=A2=E9=92=88=E7=BB=88?= =?UTF-8?q?=E9=AA=8C=E5=8F=8Cseek=E8=AE=A1=E6=97=B6=20[vps]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../providers/local_unified_provider.py | 46 +++++++++----- .../portfolio/test_local_unified_provider.py | 63 ++++++++++--------- 2 files changed, 65 insertions(+), 44 deletions(-) diff --git a/sanguo_portfolio/providers/local_unified_provider.py b/sanguo_portfolio/providers/local_unified_provider.py index 7d08d1b..5111fda 100644 --- a/sanguo_portfolio/providers/local_unified_provider.py +++ b/sanguo_portfolio/providers/local_unified_provider.py @@ -640,15 +640,25 @@ class LocalUnifiedProvider(DataProvider): # type: ignore[misc] self._cached_minmax_index = name return name - def _minmax_sql(self, conn: sqlite3.Connection, n_symbols: int) -> str: - """min/max 批量 SQL; 复合索引可用时 INDEXED BY 钉死(防计划劣化)。""" - ph = ",".join("?" * n_symbols) + def _minmax_seek_sql(self, conn: sqlite3.Connection) -> tuple: + """min/max 双 seek SQL(ORDER BY ±datetime LIMIT 1); 复合索引可用时钉死。 + + 2026-08-24 计划劣化治本(VPS 生产实测选形态): ①原 IN+GROUP BY 批量 SQL 被 + 选去单列 interval 索引整扫 ~21M 日线行(30 只冷 101s, 选股 38-51min 主根因); + ②INDEXED BY 钉复合索引后仍要逐股扫该股**全部 interval** 条目(30 只冷 + 8.1s/热 0.25s, 大池冷态分钟级); ③双 seek = 三列全等前缀直落 + (symbol,exchange,'d') 子区间两端, 每股恰 2 次索引寻位, 冷热皆毫秒级。 + 索引名跨环境不同(ensure_schema 长名/NAS 副本 uq_dbbardata)→运行时探测, + 探不到回退裸 SQL(三列全等下 planner 本就倾向复合索引)。 + """ idx = self._composite_minmax_index(conn) hint = f" INDEXED BY {idx}" if idx else "" - return ( - f"SELECT symbol, exchange, MIN(datetime), MAX(datetime) FROM dbbardata{hint} " - f"WHERE interval='d' AND symbol IN ({ph}) GROUP BY symbol, exchange" + base = ( + f"SELECT datetime FROM dbbardata{hint} " + f"WHERE symbol=? AND exchange=? AND interval='d' " + f"ORDER BY datetime" ) + return base + " LIMIT 1", base + " DESC LIMIT 1" def get_security_info_batch( self, @@ -670,17 +680,21 @@ class LocalUnifiedProvider(DataProvider): # type: ignore[misc] syms = list({sym for sym, _ in (p[1] for p in pairs)}) conn = self._connect() - # 1. dbbardata min/max per (symbol, exchange); chunk 防 >999 参数; - # INDEXED BY 钉复合索引(2026-08-24 计划劣化治本, 见 _minmax_sql) + # 1. dbbardata min/max per (symbol, exchange): 逐对双 seek(ORDER BY + # ±datetime LIMIT 1, 见 _minmax_seek_sql); 无日线对不落 minmax, + # 下游 .get((sym,exc),(None,None)) 同旧 GROUP BY 语义 minmax: Dict[tuple, tuple] = {} - for i in range(0, len(syms), 400): - chunk = syms[i:i + 400] - sql = self._minmax_sql(conn, len(chunk)) - for r in conn.execute(sql, chunk): - minmax[(r[0], r[1])] = ( - r[2][:10] if r[2] else None, - r[3][:10] if r[3] else None, - ) + sym_exc = list(dict.fromkeys(pe for _, pe in pairs)) # 去重保序 + sql_lo, sql_hi = self._minmax_seek_sql(conn) + for sym, exc in sym_exc: + lo = conn.execute(sql_lo, (sym, exc)).fetchone() + if lo is None or not lo[0]: + continue + hi = conn.execute(sql_hi, (sym, exc)).fetchone() + minmax[(sym, exc)] = ( + lo[0][:10], + hi[0][:10] if hi and hi[0] else None, + ) # 2. constituent_unified code_name(碰撞股名; 缺则回退 jq_code) names: Dict[str, str] = {} diff --git a/tests/portfolio/test_local_unified_provider.py b/tests/portfolio/test_local_unified_provider.py index 8fa14b7..25edbab 100644 --- a/tests/portfolio/test_local_unified_provider.py +++ b/tests/portfolio/test_local_unified_provider.py @@ -603,11 +603,12 @@ class TestGetSecurityInfoBatch: class TestSecurityInfoBatchIndexHint: - """2026-08-24 P0 计划劣化治本: VPS 生产 EXPLAIN 实锤 min/max 批量 SQL 被选去 - 单列 interval 索引, 整扫 ~21M 日线行 + USE TEMP B-TREE(30 只冷跑 101s, - 开盘选股 38-51min 主根因)。修 = INDEXED BY 钉死 (symbol,exchange,interval, - datetime) 复合唯一索引; 索引名跨环境不同(ensure_schema 长名 / NAS 副本 - merge_increment 建 uq_dbbardata)→运行时探测+实例缓存, 探不到回退裸查询。""" + """2026-08-24 P0 计划劣化治本(VPS 生产实测定形态): ①原 IN+GROUP BY 批量 SQL + 被选去单列 interval 索引整扫 ~21M 日线行(30 只冷 101s, 选股 38-51min 主根因); + ②INDEXED BY 钉复合索引后仍逐股扫全部 interval 条目(30 只冷 8.1s/热 0.25s); + ③终态 = 逐对双 seek(ORDER BY ±datetime LIMIT 1), 三列全等前缀直落 d 子区间 + 两端, 每股恰 2 次索引寻位。复合索引名跨环境不同(ensure_schema 长名 / NAS + 副本 merge_increment 建 uq_dbbardata)→运行时探测+实例缓存, 探不到回退裸 SQL。""" @staticmethod def _db_with_index(tmp_path, index_name): @@ -621,34 +622,38 @@ class TestSecurityInfoBatchIndexHint: c.close() return db - def test_minmax_sql_pins_composite_index(self, tmp_path): + def test_seek_sql_pins_composite_index(self, tmp_path): db = self._db_with_index(tmp_path, "dbbardata_symbol_exchange_interval_datetime") p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)}) conn = p._connect() - sql = p._minmax_sql(conn, 3) - assert "INDEXED BY dbbardata_symbol_exchange_interval_datetime" in sql + sql_lo, sql_hi = p._minmax_seek_sql(conn) + assert "INDEXED BY dbbardata_symbol_exchange_interval_datetime" in sql_lo + assert "INDEXED BY dbbardata_symbol_exchange_interval_datetime" in sql_hi + assert sql_lo.endswith("ORDER BY datetime LIMIT 1") + assert sql_hi.endswith("ORDER BY datetime DESC LIMIT 1") # 探测结果实例缓存(None 与未探测用哨兵区分, 见 _composite_minmax_index) - assert p._composite_minmax_index(conn) == "dbbardata_symbol_exchange_interval_datetime" + assert p._composite_minmax_index(conn) == \ + "dbbardata_symbol_exchange_interval_datetime" - def test_minmax_sql_detects_nas_replica_index_name(self, tmp_path): + def test_seek_sql_detects_nas_replica_index_name(self, tmp_path): """NAS 副本复合索引叫 uq_dbbardata(硬编码索引名在此环境必炸)。""" db = self._db_with_index(tmp_path, "uq_dbbardata") p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)}) - sql = p._minmax_sql(p._connect(), 3) - assert "INDEXED BY uq_dbbardata" in sql + sql_lo, _ = p._minmax_seek_sql(p._connect()) + assert "INDEXED BY uq_dbbardata" in sql_lo - def test_minmax_sql_fallback_without_composite_index(self, tmp_path): + def test_seek_sql_fallback_without_composite_index(self, tmp_path): db = _make_security_info_fixture(tmp_path) p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)}) conn = p._connect() - sql = p._minmax_sql(conn, 3) - assert "INDEXED BY" not in sql # 探不到回退裸查询(不劣于现状) - rows = list(conn.execute(sql, ("600519", "000001", "000002"))) - assert rows # 裸查询仍可执行 + sql_lo, sql_hi = p._minmax_seek_sql(conn) + assert "INDEXED BY" not in sql_lo # 探不到回退裸 SQL(三列全等仍高效) + assert conn.execute(sql_lo, ("600519", "SSE")).fetchone()[0][:10] == "2024-06-18" + assert conn.execute(sql_hi, ("600519", "SSE")).fetchone()[0][:10] == "2024-06-20" - def test_pinned_plan_avoids_temp_btree(self, tmp_path): - """钉死复合索引后 GROUP BY (symbol,exchange) 走索引序, 无 TEMP B-TREE - (劣化计划的特征签名); 单列 interval 索引同库共存也不受其扰。""" + def test_seek_plan_hits_d_subrange_directly(self, tmp_path): + """三列全等前缀 + 索引序 ORDER BY: 无 TEMP B-TREE、直落复合索引 d 子区间; + 单列 interval 索引同库共存也不受其扰。""" db = self._db_with_index(tmp_path, "dbbardata_symbol_exchange_interval_datetime") c = sqlite3.connect(str(db)) c.execute("CREATE INDEX idx_dbbardata_interval ON dbbardata (interval)") @@ -656,16 +661,18 @@ class TestSecurityInfoBatchIndexHint: c.close() p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)}) conn = p._connect() - sql = p._minmax_sql(conn, 3) - plan = "\n".join( - r[3] for r in conn.execute( - "EXPLAIN QUERY PLAN " + sql, ("600519", "000001", "000002")) - ) - assert "TEMP B-TREE" not in plan - assert "dbbardata_symbol_exchange_interval_datetime" in plan + sql_lo, sql_hi = p._minmax_seek_sql(conn) + for sql in (sql_lo, sql_hi): + plan = "\n".join( + r[3] for r in conn.execute( + "EXPLAIN QUERY PLAN " + sql, ("600519", "SSE")) + ) + assert "TEMP B-TREE" not in plan + assert "dbbardata_symbol_exchange_interval_datetime" in plan + assert "interval=?" in plan # 三列全等(d 子区间直落, 不扫 15m/5m) def test_batch_equivalence_with_index_hint(self, tmp_path): - """带复合索引(=生产形态)下 batch 与逐只结果一致(hint 不改语义)。""" + """带复合索引(=生产形态)下 batch 与逐只结果一致(seek 不改语义)。""" db = self._db_with_index(tmp_path, "uq_dbbardata") p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)}) codes = ["600519.XSHG", "000001.XSHE", "000002.XSHE", "999999.XSHG"]