perf(data): 四热路径日期区间SARGable化——substr(datetime,1,10)对索引列套函数打不进复合索引datetime列,每股扫全量日线史取短窗——08-25晨9:30生产实锤:momentum选股1/2(RPS池)174s未达<60s验收,阶段日志精确定罪(过滤段含双seek仅7.8s);病根=PanelFetcher的30天窗SQL每只股扫~5000行日线史取~22行,3226只≈1600万行=174s量级吻合。修=裸列datetime>=start AND datetime<end+1天排他上界(与按日期前10位比较在d裸日期/15m·5m时间戳两格式下语义严格等价,含end当日全部行排除次日),区间打进复合索引第4列每股只扫窗口行。四处同病同修:PanelFetcher宽表/PriceFetcher逐只get_price/limit-status近2根90天窗(原substr版90天下界同样打不进索引)/datareader CTA回测K线。+3测试:防回潮扫描(三文件钉死禁substr谓词)+双格式边界行为(d裸日期与15m时间戳end当日含次日排)+get_price同语义;portfolio+data_platform 644绿;待VPS探针终验计时 [vps]
This commit is contained in:
@@ -9,7 +9,7 @@ if _VNPY_SRC not in sys.path:
|
||||
sys.path.insert(0, _VNPY_SRC)
|
||||
|
||||
import pandas as pd
|
||||
from datetime import datetime, date
|
||||
from datetime import datetime, date, timedelta
|
||||
from vnpy.trader.object import BarData
|
||||
from vnpy.trader.constant import Exchange, Interval
|
||||
from vnpy.trader.setting import SETTINGS
|
||||
@@ -148,12 +148,16 @@ def read_index_daily(code: str, start, end, cfg) -> pd.DataFrame:
|
||||
conn = sqlite3.connect(db_path, timeout=30)
|
||||
conn.execute("PRAGMA busy_timeout = 30000")
|
||||
try:
|
||||
# substr(datetime,1,10) 比日期规避混合格式(有纯日期有带时间,同 provider 模式)
|
||||
# SARGable 日期区间(2026-08-25 同 provider P0 治本): 裸列 datetime>=/<,
|
||||
# 右端 end+1 天排他——d 裸日期与 15m/5m 时间戳混合格式下与按日期前 10 位
|
||||
# 比较语义严格等价(含 end 当日全部行);substr 版打不进复合索引 datetime
|
||||
# 列, 短窗读每股扫全量日线史
|
||||
end_excl = (datetime.strptime(end_str, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
df = pd.read_sql(
|
||||
"SELECT datetime, open_price, high_price, low_price, close_price, volume "
|
||||
"FROM dbbardata WHERE symbol=? AND exchange=? AND interval='d' "
|
||||
"AND substr(datetime,1,10)>=? AND substr(datetime,1,10)<=? ORDER BY datetime",
|
||||
conn, params=(symbol, exchange, start_str, end_str),
|
||||
"AND datetime>=? AND datetime<? ORDER BY datetime",
|
||||
conn, params=(symbol, exchange, start_str, end_excl),
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -7,7 +7,7 @@ get_closes_panel(e7f9426 生产版,UNION ALL 340× 索引优化保留),合法参
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
|
||||
|
||||
import pandas as pd
|
||||
@@ -28,6 +28,20 @@ def _safe_date_literal(s: str) -> str:
|
||||
return f"'{s}'"
|
||||
|
||||
|
||||
def end_exclusive_str(end_str: str) -> str:
|
||||
"""end 日期 → 排他上界(end+1 天)字符串。SARGable 日期区间右端。
|
||||
|
||||
2026-08-25 P0 治本: 旧「取日期前 10 位再比较」写法对索引列套函数,日期区间
|
||||
打不进复合索引 datetime 列 → 每只股扫全量日线史取 30 天窗(momentum RPS 池
|
||||
3226 只实测 174s)。裸列 ``datetime<end+1天`` 与旧写法语义严格等价:
|
||||
d 裸日期('2026-08-22'<…23')与 15m/5m 时间戳('2026-08-22 15:00:00'<…23')
|
||||
均含 end 当日全部行、排除次日。
|
||||
"""
|
||||
return (
|
||||
datetime.strptime(end_str[:10], "%Y-%m-%d").date() + timedelta(days=1)
|
||||
).isoformat()
|
||||
|
||||
|
||||
def _safe_interval_literal(s: str) -> str:
|
||||
if not isinstance(s, str) or not _INTERVAL_RE.match(s) or len(s) > 16:
|
||||
raise ValueError(f"Invalid interval: {s!r}")
|
||||
@@ -55,10 +69,13 @@ class PriceFetcher:
|
||||
conn = ctx._connect()
|
||||
start_str = query.start_date or "1990-01-01"
|
||||
end_str = query.end_date or datetime.now().strftime("%Y-%m-%d")
|
||||
# SARGable 日期区间(2026-08-25 P0): 裸列 datetime>=/<,右端 end+1 天排他
|
||||
# ——substr 版每只股扫全量日线史,详见 end_exclusive_str
|
||||
end_excl = end_exclusive_str(end_str)
|
||||
q = (
|
||||
"SELECT datetime, open_price, high_price, low_price, close_price, "
|
||||
"volume, turnover FROM dbbardata WHERE symbol=? AND exchange=? "
|
||||
"AND interval='d' AND substr(datetime,1,10)>=? AND substr(datetime,1,10)<=? "
|
||||
"AND interval='d' AND datetime>=? AND datetime<? "
|
||||
"ORDER BY datetime"
|
||||
)
|
||||
need_qfq = query.fq in ("qfq", "pre", "前复权")
|
||||
@@ -67,7 +84,7 @@ class PriceFetcher:
|
||||
for jq_code in query.security:
|
||||
sym, exc = jq_to_dbbardata(jq_code)
|
||||
frames[jq_code] = pd.read_sql(
|
||||
q, conn, params=(sym, exc, start_str, end_str)
|
||||
q, conn, params=(sym, exc, start_str, end_excl)
|
||||
)
|
||||
if need_qfq:
|
||||
qfq_rows[jq_code] = _read_qfq_rows(_jq_to_bs_code(jq_code), conn)
|
||||
@@ -188,18 +205,22 @@ class PanelFetcher:
|
||||
|
||||
conn = ctx._connect()
|
||||
start_lit = _safe_date_literal(query.start or "1990-01-01")
|
||||
end_lit = _safe_date_literal(query.end or datetime.now().strftime("%Y-%m-%d"))
|
||||
end_raw = query.end or datetime.now().strftime("%Y-%m-%d")
|
||||
_safe_date_literal(end_raw) # 校验 YYYY-MM-DD(fail-fast 同老契约)
|
||||
end_excl_lit = f"'{end_exclusive_str(end_raw)}'"
|
||||
interval_lit = _safe_interval_literal(query.interval)
|
||||
|
||||
CHUNK_SIZE = 400
|
||||
chunk_frames: List[pd.DataFrame] = []
|
||||
for i in range(0, len(pairs), CHUNK_SIZE):
|
||||
chunk = pairs[i:i + CHUNK_SIZE]
|
||||
# SARGable 日期区间(2026-08-25 P0): 裸列区间打进复合索引 datetime 列,
|
||||
# 每股只扫窗口行(30 天窗 ~22 行);substr 版扫全量日线史(3226 只=174s)
|
||||
sub_template = (
|
||||
"SELECT datetime, symbol, close_price FROM dbbardata "
|
||||
f"WHERE symbol=? AND exchange=? AND interval={interval_lit} "
|
||||
f"AND substr(datetime,1,10)>={start_lit} "
|
||||
f"AND substr(datetime,1,10)<={end_lit}"
|
||||
f"AND datetime>={start_lit} "
|
||||
f"AND datetime<{end_excl_lit}"
|
||||
)
|
||||
q = " UNION ALL ".join([sub_template] * len(chunk))
|
||||
params: List[Any] = []
|
||||
|
||||
@@ -782,14 +782,18 @@ class LocalUnifiedProvider(DataProvider): # type: ignore[misc]
|
||||
from datetime import timedelta
|
||||
start_lim = (datetime.strptime(date_str, "%Y-%m-%d") - timedelta(days=90)).strftime("%Y-%m-%d")
|
||||
start_lit = _safe_date_literal(start_lim)
|
||||
end_lit = _safe_date_literal(date_str)
|
||||
# SARGable 日期区间(2026-08-25 P0): 裸列区间+end+1 天排他——substr 版的
|
||||
# 90 天下界打不进索引, 每股仍扫全量日线史(语义等价, 见 fetchers.price)
|
||||
end_excl_lit = _safe_date_literal(
|
||||
(datetime.strptime(date_str, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
)
|
||||
interval_lit = _safe_interval_literal("d")
|
||||
# 纯 SELECT UNION ALL(子查询带 ORDER BY/LIMIT 触发 SQLite compound 限制);
|
||||
# 用 90 天下界限定范围(全历史 → 近 90 天), 走复合索引, pandas 端取最近 2 根。
|
||||
# 90 天下界 + 裸列区间走复合索引(每股只扫窗口行), pandas 端取最近 2 根。
|
||||
sub = (
|
||||
"SELECT symbol, exchange, close_price, high_price, low_price, volume, datetime "
|
||||
f"FROM dbbardata WHERE symbol=? AND exchange=? AND interval={interval_lit} "
|
||||
f"AND substr(datetime,1,10)>={start_lit} AND substr(datetime,1,10)<={end_lit}"
|
||||
f"AND datetime>={start_lit} AND datetime<{end_excl_lit}"
|
||||
)
|
||||
bars: Dict[tuple, list] = {}
|
||||
sym_exc = list({p[1] for p in pairs}) # 去重 (sym,exc)
|
||||
|
||||
@@ -242,3 +242,62 @@ class TestOptionalContract:
|
||||
fields=["close", "acc_net_value"],
|
||||
)
|
||||
assert df["acc_net_value"].isna().all()
|
||||
|
||||
|
||||
# ======================== SARGable 日期区间(2026-08-25 P0) ========================
|
||||
|
||||
class TestSargableDateRange:
|
||||
"""substr(datetime,1,10) 对索引列套函数 → 日期区间打不进复合索引 datetime 列,
|
||||
每股扫全量日线史取短窗(momentum RPS 池 3226 只 30 天窗 VPS 实测 174s;provider
|
||||
/datareader 四热路径同病)。裸列 ``datetime>=start AND datetime<end+1天`` 与
|
||||
substr 语义严格等价(d 裸日期/带时间戳两格式, 含 end 当日全部行、排除次日)。"""
|
||||
|
||||
def test_no_substr_datetime_regression(self):
|
||||
"""钉死四个热路径文件不再回潮非 SARGable 谓词。"""
|
||||
import pathlib
|
||||
root = pathlib.Path(__file__).resolve().parents[2]
|
||||
for rel in (
|
||||
"sanguo_portfolio/providers/fetchers/price.py",
|
||||
"sanguo_portfolio/providers/local_unified_provider.py",
|
||||
"sanguo_data/datareader.py",
|
||||
):
|
||||
src = (root / rel).read_text(encoding="utf-8")
|
||||
assert "substr(datetime" not in src, f"{rel} 回潮非 SARGable 谓词"
|
||||
|
||||
def test_panel_end_date_inclusive_bare_and_timestamp(self, unified_provider):
|
||||
"""end 当日全部行含(d 裸日期 + 15m 时间戳两格式), 次日排除。"""
|
||||
conn = sqlite3.connect(unified_provider.db_path)
|
||||
conn.executemany("INSERT INTO dbbardata VALUES(?,?,?,?,?,?,?,?,?,?,?)", [
|
||||
# d 裸日期: end 当日含 / 次日排
|
||||
("600600", "SSE", "2024-06-19", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 10.0),
|
||||
("600600", "SSE", "2024-06-21", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 12.0),
|
||||
# 15m 时间戳: end 当日含 / 次日排
|
||||
("600600", "SSE", "2024-06-20 14:35:00", "15m", 100, 1e3, 0, 1.0, 1.0, 1.0, 11.0),
|
||||
("600600", "SSE", "2024-06-21 09:35:00", "15m", 100, 1e3, 0, 1.0, 1.0, 1.0, 13.0),
|
||||
])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
# d 面: 窗口 06-18~06-20 → 只有裸日期 06-19 行
|
||||
panel_d = unified_provider.get_closes_panel(
|
||||
["600600.XSHG"], "2024-06-18", "2024-06-20", fq="raw")
|
||||
assert [str(d)[:10] for d in panel_d.index] == ["2024-06-19"]
|
||||
assert panel_d.iloc[0, 0] == 10.0
|
||||
# 15m 面: 同窗 → 只有 06-20 14:35 行(end 当日带时间戳不被右界误伤)
|
||||
panel_15 = unified_provider.get_closes_panel(
|
||||
["600600.XSHG"], "2024-06-18", "2024-06-20", interval="15m", fq="raw")
|
||||
assert [str(d)[:16] for d in panel_15.index] == ["2024-06-20 14:35"]
|
||||
assert panel_15.iloc[0, 0] == 11.0
|
||||
|
||||
def test_get_price_end_date_inclusive_bare_date(self, unified_provider):
|
||||
"""PriceFetcher 逐只路径同语义: 裸日期 end 当日含、次日排。"""
|
||||
conn = sqlite3.connect(unified_provider.db_path)
|
||||
conn.executemany("INSERT INTO dbbardata VALUES(?,?,?,?,?,?,?,?,?,?,?)", [
|
||||
("600600", "SSE", "2024-06-19", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 10.0),
|
||||
("600600", "SSE", "2024-06-21", "d", 100, 1e3, 0, 1.0, 1.0, 1.0, 12.0),
|
||||
])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
df = unified_provider.get_price(
|
||||
"600600.XSHG", start_date="2024-06-18", end_date="2024-06-20")
|
||||
assert len(df) == 1
|
||||
assert df.iloc[-1]["close"] == 10.0
|
||||
|
||||
Reference in New Issue
Block a user