perf(data): 四热路径日期区间SARGable化——substr(datetime,1,10)对索引列套函数打不进复合索引datetime列,每股扫全量日线史取短窗——08-25晨9:30生产实锤:momentum选股1/2(RPS池)174s未达<60s验收,阶段日志精确定罪(过滤段含双seek仅7.8s);病根=PanelFetcher的30天窗SQL每只股扫~5000行日线史取~22行,3226只≈1600万行=174s量级吻合。修=裸列datetime>=start AND datetime<end+1天排他上界(与按日期前10位比较在d裸日期/15m·5m时间戳两格式下语义严格等价,含end当日全部行排除次日),区间打进复合索引第4列每股只扫窗口行。四处同病同修:PanelFetcher宽表/PriceFetcher逐只get_price/limit-status近2根90天窗(原substr版90天下界同样打不进索引)/datareader CTA回测K线。+3测试:防回潮扫描(三文件钉死禁substr谓词)+双格式边界行为(d裸日期与15m时间戳end当日含次日排)+get_price同语义;portfolio+data_platform 644绿;待VPS探针终验计时 [vps]
CI/CD / test (push) Successful in 3s
CI/CD / nas-deploy (push) Successful in 21s
CI/CD / nas-verify (push) Successful in 8s

This commit is contained in:
2026-08-25 20:54:40 +08:00
parent 841ea1536e
commit 3304ff46b2
4 changed files with 101 additions and 13 deletions
+8 -4
View File
@@ -9,7 +9,7 @@ if _VNPY_SRC not in sys.path:
sys.path.insert(0, _VNPY_SRC)
import pandas as pd
from datetime import datetime, date
from datetime import datetime, date, timedelta
from vnpy.trader.object import BarData
from vnpy.trader.constant import Exchange, Interval
from vnpy.trader.setting import SETTINGS
@@ -148,12 +148,16 @@ def read_index_daily(code: str, start, end, cfg) -> pd.DataFrame:
conn = sqlite3.connect(db_path, timeout=30)
conn.execute("PRAGMA busy_timeout = 30000")
try:
# substr(datetime,1,10) 比日期规避混合格式(有纯日期有带时间,同 provider 模式)
# SARGable 日期区间(2026-08-25 同 provider P0 治本): 裸列 datetime>=/<,
# 右端 end+1 天排他——d 裸日期与 15m/5m 时间戳混合格式下与按日期前 10 位
# 比较语义严格等价(含 end 当日全部行);substr 版打不进复合索引 datetime
# 列, 短窗读每股扫全量日线史
end_excl = (datetime.strptime(end_str, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
df = pd.read_sql(
"SELECT datetime, open_price, high_price, low_price, close_price, volume "
"FROM dbbardata WHERE symbol=? AND exchange=? AND interval='d' "
"AND substr(datetime,1,10)>=? AND substr(datetime,1,10)<=? ORDER BY datetime",
conn, params=(symbol, exchange, start_str, end_str),
"AND datetime>=? AND datetime<? ORDER BY datetime",
conn, params=(symbol, exchange, start_str, end_excl),
)
finally:
conn.close()
+27 -6
View File
@@ -7,7 +7,7 @@ get_closes_panel(e7f9426 生产版,UNION ALL 340× 索引优化保留),合法参
from __future__ import annotations
import re
from datetime import datetime
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
import pandas as pd
@@ -28,6 +28,20 @@ def _safe_date_literal(s: str) -> str:
return f"'{s}'"
def end_exclusive_str(end_str: str) -> str:
"""end 日期 → 排他上界(end+1 天)字符串。SARGable 日期区间右端。
2026-08-25 P0 治本: 取日期前 10 位再比较写法对索引列套函数,日期区间
打不进复合索引 datetime 每只股扫全量日线史取 30 天窗(momentum RPS
3226 只实测 174s)裸列 ``datetime<end+1`` 与旧写法语义严格等价:
d 裸日期('2026-08-22'<23')与 15m/5m 时间戳('2026-08-22 15:00:00'<…23')
均含 end 当日全部行排除次日
"""
return (
datetime.strptime(end_str[:10], "%Y-%m-%d").date() + timedelta(days=1)
).isoformat()
def _safe_interval_literal(s: str) -> str:
if not isinstance(s, str) or not _INTERVAL_RE.match(s) or len(s) > 16:
raise ValueError(f"Invalid interval: {s!r}")
@@ -55,10 +69,13 @@ class PriceFetcher:
conn = ctx._connect()
start_str = query.start_date or "1990-01-01"
end_str = query.end_date or datetime.now().strftime("%Y-%m-%d")
# SARGable 日期区间(2026-08-25 P0): 裸列 datetime>=/<,右端 end+1 天排他
# ——substr 版每只股扫全量日线史,详见 end_exclusive_str
end_excl = end_exclusive_str(end_str)
q = (
"SELECT datetime, open_price, high_price, low_price, close_price, "
"volume, turnover FROM dbbardata WHERE symbol=? AND exchange=? "
"AND interval='d' AND substr(datetime,1,10)>=? AND substr(datetime,1,10)<=? "
"AND interval='d' AND datetime>=? AND datetime<? "
"ORDER BY datetime"
)
need_qfq = query.fq in ("qfq", "pre", "前复权")
@@ -67,7 +84,7 @@ class PriceFetcher:
for jq_code in query.security:
sym, exc = jq_to_dbbardata(jq_code)
frames[jq_code] = pd.read_sql(
q, conn, params=(sym, exc, start_str, end_str)
q, conn, params=(sym, exc, start_str, end_excl)
)
if need_qfq:
qfq_rows[jq_code] = _read_qfq_rows(_jq_to_bs_code(jq_code), conn)
@@ -188,18 +205,22 @@ class PanelFetcher:
conn = ctx._connect()
start_lit = _safe_date_literal(query.start or "1990-01-01")
end_lit = _safe_date_literal(query.end or datetime.now().strftime("%Y-%m-%d"))
end_raw = query.end or datetime.now().strftime("%Y-%m-%d")
_safe_date_literal(end_raw) # 校验 YYYY-MM-DD(fail-fast 同老契约)
end_excl_lit = f"'{end_exclusive_str(end_raw)}'"
interval_lit = _safe_interval_literal(query.interval)
CHUNK_SIZE = 400
chunk_frames: List[pd.DataFrame] = []
for i in range(0, len(pairs), CHUNK_SIZE):
chunk = pairs[i:i + CHUNK_SIZE]
# SARGable 日期区间(2026-08-25 P0): 裸列区间打进复合索引 datetime 列,
# 每股只扫窗口行(30 天窗 ~22 行);substr 版扫全量日线史(3226 只=174s)
sub_template = (
"SELECT datetime, symbol, close_price FROM dbbardata "
f"WHERE symbol=? AND exchange=? AND interval={interval_lit} "
f"AND substr(datetime,1,10)>={start_lit} "
f"AND substr(datetime,1,10)<={end_lit}"
f"AND datetime>={start_lit} "
f"AND datetime<{end_excl_lit}"
)
q = " UNION ALL ".join([sub_template] * len(chunk))
params: List[Any] = []
@@ -782,14 +782,18 @@ class LocalUnifiedProvider(DataProvider): # type: ignore[misc]
from datetime import timedelta
start_lim = (datetime.strptime(date_str, "%Y-%m-%d") - timedelta(days=90)).strftime("%Y-%m-%d")
start_lit = _safe_date_literal(start_lim)
end_lit = _safe_date_literal(date_str)
# SARGable 日期区间(2026-08-25 P0): 裸列区间+end+1 天排他——substr 版的
# 90 天下界打不进索引, 每股仍扫全量日线史(语义等价, 见 fetchers.price)
end_excl_lit = _safe_date_literal(
(datetime.strptime(date_str, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
)
interval_lit = _safe_interval_literal("d")
# 纯 SELECT UNION ALL(子查询带 ORDER BY/LIMIT 触发 SQLite compound 限制);
# 90 天下界限定范围(全历史 → 近 90 天), 走复合索引, pandas 端取最近 2 根。
# 90 天下界 + 裸列区间走复合索引(每股只扫窗口行), pandas 端取最近 2 根。
sub = (
"SELECT symbol, exchange, close_price, high_price, low_price, volume, datetime "
f"FROM dbbardata WHERE symbol=? AND exchange=? AND interval={interval_lit} "
f"AND substr(datetime,1,10)>={start_lit} AND substr(datetime,1,10)<={end_lit}"
f"AND datetime>={start_lit} AND datetime<{end_excl_lit}"
)
bars: Dict[tuple, list] = {}
sym_exc = list({p[1] for p in pairs}) # 去重 (sym,exc)
+59
View File
@@ -242,3 +242,62 @@ class TestOptionalContract:
fields=["close", "acc_net_value"],
)
assert df["acc_net_value"].isna().all()
# ======================== SARGable 日期区间(2026-08-25 P0) ========================
class TestSargableDateRange:
"""substr(datetime,1,10) 对索引列套函数 → 日期区间打不进复合索引 datetime 列,
每股扫全量日线史取短窗(momentum RPS 3226 30 天窗 VPS 实测 174s;provider
/datareader 四热路径同病)裸列 ``datetime>=start AND datetime<end+1``
substr 语义严格等价(d 裸日期/带时间戳两格式, end 当日全部行排除次日)"""
def test_no_substr_datetime_regression(self):
"""钉死四个热路径文件不再回潮非 SARGable 谓词。"""
import pathlib
root = pathlib.Path(__file__).resolve().parents[2]
for rel in (
"sanguo_portfolio/providers/fetchers/price.py",
"sanguo_portfolio/providers/local_unified_provider.py",
"sanguo_data/datareader.py",
):
src = (root / rel).read_text(encoding="utf-8")
assert "substr(datetime" not in src, f"{rel} 回潮非 SARGable 谓词"
def test_panel_end_date_inclusive_bare_and_timestamp(self, unified_provider):
"""end 当日全部行含(d 裸日期 + 15m 时间戳两格式), 次日排除。"""
conn = sqlite3.connect(unified_provider.db_path)
conn.executemany("INSERT INTO dbbardata VALUES(?,?,?,?,?,?,?,?,?,?,?)", [
# d 裸日期: end 当日含 / 次日排
("600600", "SSE", "2024-06-19", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 10.0),
("600600", "SSE", "2024-06-21", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 12.0),
# 15m 时间戳: end 当日含 / 次日排
("600600", "SSE", "2024-06-20 14:35:00", "15m", 100, 1e3, 0, 1.0, 1.0, 1.0, 11.0),
("600600", "SSE", "2024-06-21 09:35:00", "15m", 100, 1e3, 0, 1.0, 1.0, 1.0, 13.0),
])
conn.commit()
conn.close()
# d 面: 窗口 06-18~06-20 → 只有裸日期 06-19 行
panel_d = unified_provider.get_closes_panel(
["600600.XSHG"], "2024-06-18", "2024-06-20", fq="raw")
assert [str(d)[:10] for d in panel_d.index] == ["2024-06-19"]
assert panel_d.iloc[0, 0] == 10.0
# 15m 面: 同窗 → 只有 06-20 14:35 行(end 当日带时间戳不被右界误伤)
panel_15 = unified_provider.get_closes_panel(
["600600.XSHG"], "2024-06-18", "2024-06-20", interval="15m", fq="raw")
assert [str(d)[:16] for d in panel_15.index] == ["2024-06-20 14:35"]
assert panel_15.iloc[0, 0] == 11.0
def test_get_price_end_date_inclusive_bare_date(self, unified_provider):
"""PriceFetcher 逐只路径同语义: 裸日期 end 当日含、次日排。"""
conn = sqlite3.connect(unified_provider.db_path)
conn.executemany("INSERT INTO dbbardata VALUES(?,?,?,?,?,?,?,?,?,?,?)", [
("600600", "SSE", "2024-06-19", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 10.0),
("600600", "SSE", "2024-06-21", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 12.0),
])
conn.commit()
conn.close()
df = unified_provider.get_price(
"600600.XSHG", start_date="2024-06-18", end_date="2024-06-20")
assert len(df) == 1
assert df.iloc[-1]["close"] == 10.0