perf(data): min/max改逐对双seek——INDEXED BY批IN形态VPS实测仍不达标(30只冷8.1s/热0.25s,大池冷态分钟级),ORDER BY ±datetime LIMIT 1三列全等前缀每股恰2次寻位冷热皆毫秒级——上一发(80fc517)生产探针续测定形态:①原IN+GROUP BY劣化计划101s冷;②钉复合索引后计划翻转(TEMP B-TREE消失)但仍逐股扫全部interval条目(d+15m+5m),30只冷8.1s→momentum池~2000只外推冷态≈9min过不了<60s验收;③行值IN变体在本SQLite对(row-value IN+INDEXED BY)走全索引扫描7min不归(弃);终态=逐对双seek,symbol/exchange/interval三列全等直落d子区间两端,连15m/5m条目都不碰;无日线对不落minmax(下游.get默认(None,None)同旧GROUP BY语义);测试5个重写到seek API(双索引名/回退/EXPLAIN计划签名interval=?直落+无TEMP B-TREE/batch==逐只等价);628绿;待VPS探针终验双seek计时 [vps]

2026-08-24 14:14:11 +08:00
parent 96a1e6b413
commit aafa5baa67
2 changed files with 65 additions and 44 deletions
@@ -640,15 +640,25 @@ class LocalUnifiedProvider(DataProvider):  # type: ignore[misc]
        self._cached_minmax_index = name
        return name

-    def _minmax_sql(self, conn: sqlite3.Connection, n_symbols: int) -> str:
-        """min/max 批量 SQL; 复合索引可用时 INDEXED BY 钉死(防计划劣化)。"""
-        ph = ",".join("?" * n_symbols)
+    def _minmax_seek_sql(self, conn: sqlite3.Connection) -> tuple:
+        """min/max 双 seek SQL(ORDER BY ±datetime LIMIT 1); 复合索引可用时钉死。
+
+        2026-08-24 计划劣化治本(VPS 生产实测选形态): ①原 IN+GROUP BY 批量 SQL 被
+        选去单列 interval 索引整扫 ~21M 日线行(30 只冷 101s, 选股 38-51min 主根因);
+        ②INDEXED BY 钉复合索引后仍要逐股扫该股**全部 interval** 条目(30 只冷
+        8.1s/热 0.25s, 大池冷态分钟级); ③双 seek = 三列全等前缀直落
+        (symbol,exchange,'d') 子区间两端, 每股恰 2 次索引寻位, 冷热皆毫秒级。
+        索引名跨环境不同(ensure_schema 长名/NAS 副本 uq_dbbardata)→运行时探测,
+        探不到回退裸 SQL(三列全等下 planner 本就倾向复合索引)。
+        """
        idx = self._composite_minmax_index(conn)
        hint = f" INDEXED BY {idx}" if idx else ""
-        return (
-            f"SELECT symbol, exchange, MIN(datetime), MAX(datetime) FROM dbbardata{hint} "
-            f"WHERE interval='d' AND symbol IN ({ph}) GROUP BY symbol, exchange"
+        base = (
+            f"SELECT datetime FROM dbbardata{hint} "
+            f"WHERE symbol=? AND exchange=? AND interval='d' "
+            f"ORDER BY datetime"
        )
+        return base + " LIMIT 1", base + " DESC LIMIT 1"

    def get_security_info_batch(
        self,
@@ -670,17 +680,21 @@ class LocalUnifiedProvider(DataProvider):  # type: ignore[misc]
        syms = list({sym for sym, _ in (p[1] for p in pairs)})
        conn = self._connect()

-        # 1. dbbardata min/max per (symbol, exchange); chunk 防 >999 参数;
-        #    INDEXED BY 钉复合索引(2026-08-24 计划劣化治本, 见 _minmax_sql)
+        # 1. dbbardata min/max per (symbol, exchange): 逐对双 seek(ORDER BY
+        #    ±datetime LIMIT 1, 见 _minmax_seek_sql); 无日线对不落 minmax,
+        #    下游 .get((sym,exc),(None,None)) 同旧 GROUP BY 语义
        minmax: Dict[tuple, tuple] = {}
-        for i in range(0, len(syms), 400):
-            chunk = syms[i:i + 400]
-            sql = self._minmax_sql(conn, len(chunk))
-            for r in conn.execute(sql, chunk):
-                minmax[(r[0], r[1])] = (
-                    r[2][:10] if r[2] else None,
-                    r[3][:10] if r[3] else None,
-                )
+        sym_exc = list(dict.fromkeys(pe for _, pe in pairs))  # 去重保序
+        sql_lo, sql_hi = self._minmax_seek_sql(conn)
+        for sym, exc in sym_exc:
+            lo = conn.execute(sql_lo, (sym, exc)).fetchone()
+            if lo is None or not lo[0]:
+                continue
+            hi = conn.execute(sql_hi, (sym, exc)).fetchone()
+            minmax[(sym, exc)] = (
+                lo[0][:10],
+                hi[0][:10] if hi and hi[0] else None,
+            )

        # 2. constituent_unified code_name(碰撞股名; 缺则回退 jq_code)
        names: Dict[str, str] = {}
@@ -603,11 +603,12 @@ class TestGetSecurityInfoBatch:


 class TestSecurityInfoBatchIndexHint:
-    """2026-08-24 P0 计划劣化治本: VPS 生产 EXPLAIN 实锤 min/max 批量 SQL 被选去
-    单列 interval 索引, 整扫 ~21M 日线行 + USE TEMP B-TREE(30 只冷跑 101s,
-    开盘选股 38-51min 主根因)。修 = INDEXED BY 钉死 (symbol,exchange,interval,
-    datetime) 复合唯一索引; 索引名跨环境不同(ensure_schema 长名 / NAS 副本
-    merge_increment 建 uq_dbbardata)→运行时探测+实例缓存, 探不到回退裸查询。"""
+    """2026-08-24 P0 计划劣化治本(VPS 生产实测定形态): ①原 IN+GROUP BY 批量 SQL
+    被选去单列 interval 索引整扫 ~21M 日线行(30 只冷 101s, 选股 38-51min 主根因);
+    ②INDEXED BY 钉复合索引后仍逐股扫全部 interval 条目(30 只冷 8.1s/热 0.25s);
+    ③终态 = 逐对双 seek(ORDER BY ±datetime LIMIT 1), 三列全等前缀直落 d 子区间
+    两端, 每股恰 2 次索引寻位。复合索引名跨环境不同(ensure_schema 长名 / NAS
+    副本 merge_increment 建 uq_dbbardata)→运行时探测+实例缓存, 探不到回退裸 SQL。"""

    @staticmethod
    def _db_with_index(tmp_path, index_name):
@@ -621,34 +622,38 @@ class TestSecurityInfoBatchIndexHint:
        c.close()
        return db

-    def test_minmax_sql_pins_composite_index(self, tmp_path):
+    def test_seek_sql_pins_composite_index(self, tmp_path):
        db = self._db_with_index(tmp_path, "dbbardata_symbol_exchange_interval_datetime")
        p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)})
        conn = p._connect()
-        sql = p._minmax_sql(conn, 3)
-        assert "INDEXED BY dbbardata_symbol_exchange_interval_datetime" in sql
+        sql_lo, sql_hi = p._minmax_seek_sql(conn)
+        assert "INDEXED BY dbbardata_symbol_exchange_interval_datetime" in sql_lo
+        assert "INDEXED BY dbbardata_symbol_exchange_interval_datetime" in sql_hi
+        assert sql_lo.endswith("ORDER BY datetime LIMIT 1")
+        assert sql_hi.endswith("ORDER BY datetime DESC LIMIT 1")
        # 探测结果实例缓存(None 与未探测用哨兵区分, 见 _composite_minmax_index)
-        assert p._composite_minmax_index(conn) == "dbbardata_symbol_exchange_interval_datetime"
+        assert p._composite_minmax_index(conn) == \
+            "dbbardata_symbol_exchange_interval_datetime"

-    def test_minmax_sql_detects_nas_replica_index_name(self, tmp_path):
+    def test_seek_sql_detects_nas_replica_index_name(self, tmp_path):
        """NAS 副本复合索引叫 uq_dbbardata(硬编码索引名在此环境必炸)。"""
        db = self._db_with_index(tmp_path, "uq_dbbardata")
        p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)})
-        sql = p._minmax_sql(p._connect(), 3)
-        assert "INDEXED BY uq_dbbardata" in sql
+        sql_lo, _ = p._minmax_seek_sql(p._connect())
+        assert "INDEXED BY uq_dbbardata" in sql_lo

-    def test_minmax_sql_fallback_without_composite_index(self, tmp_path):
+    def test_seek_sql_fallback_without_composite_index(self, tmp_path):
        db = _make_security_info_fixture(tmp_path)
        p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)})
        conn = p._connect()
-        sql = p._minmax_sql(conn, 3)
-        assert "INDEXED BY" not in sql  # 探不到回退裸查询(不劣于现状)
-        rows = list(conn.execute(sql, ("600519", "000001", "000002")))
-        assert rows  # 裸查询仍可执行
+        sql_lo, sql_hi = p._minmax_seek_sql(conn)
+        assert "INDEXED BY" not in sql_lo  # 探不到回退裸 SQL(三列全等仍高效)
+        assert conn.execute(sql_lo, ("600519", "SSE")).fetchone()[0][:10] == "2024-06-18"
+        assert conn.execute(sql_hi, ("600519", "SSE")).fetchone()[0][:10] == "2024-06-20"

-    def test_pinned_plan_avoids_temp_btree(self, tmp_path):
-        """钉死复合索引后 GROUP BY (symbol,exchange) 走索引序, 无 TEMP B-TREE
-        (劣化计划的特征签名); 单列 interval 索引同库共存也不受其扰。"""
+    def test_seek_plan_hits_d_subrange_directly(self, tmp_path):
+        """三列全等前缀 + 索引序 ORDER BY: 无 TEMP B-TREE、直落复合索引 d 子区间;
+        单列 interval 索引同库共存也不受其扰。"""
        db = self._db_with_index(tmp_path, "dbbardata_symbol_exchange_interval_datetime")
        c = sqlite3.connect(str(db))
        c.execute("CREATE INDEX idx_dbbardata_interval ON dbbardata (interval)")
@@ -656,16 +661,18 @@ class TestSecurityInfoBatchIndexHint:
        c.close()
        p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)})
        conn = p._connect()
-        sql = p._minmax_sql(conn, 3)
-        plan = "\n".join(
-            r[3] for r in conn.execute(
-                "EXPLAIN QUERY PLAN " + sql, ("600519", "000001", "000002"))
-        )
-        assert "TEMP B-TREE" not in plan
-        assert "dbbardata_symbol_exchange_interval_datetime" in plan
+        sql_lo, sql_hi = p._minmax_seek_sql(conn)
+        for sql in (sql_lo, sql_hi):
+            plan = "\n".join(
+                r[3] for r in conn.execute(
+                    "EXPLAIN QUERY PLAN " + sql, ("600519", "SSE"))
+            )
+            assert "TEMP B-TREE" not in plan
+            assert "dbbardata_symbol_exchange_interval_datetime" in plan
+            assert "interval=?" in plan  # 三列全等(d 子区间直落, 不扫 15m/5m)

    def test_batch_equivalence_with_index_hint(self, tmp_path):
-        """带复合索引(=生产形态)下 batch 与逐只结果一致(hint 不改语义)。"""
+        """带复合索引(=生产形态)下 batch 与逐只结果一致(seek 不改语义)。"""
        db = self._db_with_index(tmp_path, "uq_dbbardata")
        p = LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)})
        codes = ["600519.XSHG", "000001.XSHE", "000002.XSHE", "999999.XSHG"]