claude_dev
4966d30a11
09-02任务①落地。§15采集的14类panel数据此前零读取出口,策略/因子session 想用只能手搓parquet路径;经业界对照(dsa get_limit_up_pool/tushare limit_list_d U/D/Z参数化/聚宽per-domain/OpenBB TET)定稿「泛型底座+域 语义门面」: ①get_event_panel(event_type,date/start+end,trading_days_only=True): 保真底座,白名单14类统一通道(=采集注册表口径,hot_rank未挂载不入列), 返回akshare原样中文列+trade_date(文件名合成);缺文件=合法缺失→空df; trading_days_only默认用get_trade_days滤非交易日——治快照族「周中法定 假日文件=上一交易日态复制品」的区间双计数(采集层只按周一~五落盘, 周六根本无文件,风险面=周中假日);=False按周一~五枚举(=采集口径) ②get_limit_pool(kind=zt|zbgc|dtgc,...):涨停池门面(温度计/炸板率/行业 集中度消费契约),kind三合一←tushare limit_list_d;英文标准列 code/consecutive_boards/seal_amount/break_count/industry…(键名对齐dsa); 映射表基于VPS真实parquet实测(dtgc实测含动态市盈率/封单资金/板上成交额/ 连续跌停/开板次数,与文档口径有差);akshare改中文列名时本层吸收漂移, 有行缺源列→DataSchemaError fail-fast;真空日(dtgc 0跌停空文件无列) →仍返标准列空表schema稳定 ③实现=TET Fetcher(fetchers/event_panel.py双Fetcher同文件,price.py先例), FETCHERS注册表+event_panel/limit_pool两键(未来MCP出口零成本);base.py 新增_DateRangeParams跨字段校验(date/start+end互斥二选一);LocalUnified Provider(回测)与SanguoMiniQmtProvider(实盘,委托self._unified)双侧同款 ——方法面钉死测试强制无缺口,双侧同schema支持副本对照;miniQMT/QMT/ PTrade均无此类接口(业界惯例=外部源补),实盘同读本地文件 测试:21新测试(白名单fail-fast/日期语义/保真读/假日滤/门面schema/ 列漂移fail-fast/真空日空表/注册表)+全量793绿;活文档§16+§15指引行 [vps]
269 lines
12 KiB
Python
269 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""事件/快照 panel 数据 Fetcher 测试(2026-09-02 A 档使用层出口)。
|
|
|
|
两层接口(设计=泛型底座 + 涨停池语义门面,对齐 OpenBB TET/tushare limit_list_d):
|
|
1. ``EventPanelFetcher``(get_event_panel): 保真通道——akshare 原样中文列
|
|
+ ``trade_date``(文件名合成);新数据类型零成本接入;缺文件=合法空。
|
|
2. ``LimitPoolFetcher``(get_limit_pool): 涨停池门面——kind 三合一
|
|
(zt/zbgc/dtgc ← tushare limit_list_d 的 U/D/Z 参数化),英文标准列
|
|
(键名对齐 dsa get_limit_up_pool),列缺失 fail-fast(吸收 akshare 列漂移)。
|
|
|
|
关键契约:
|
|
- ``trading_days_only=True``(默认)用 get_trade_days 滤非交易日——治快照族
|
|
「节假日文件=上一交易日态复制品」的区间双计数(§15 语义)。
|
|
- 列名映射基于 VPS 真实 parquet 实测(2026-09-02),非 akshare 文档臆测:
|
|
dtgc 实测含 动态市盈率/封单资金/板上成交额/连续跌停/开板次数。
|
|
- 真空日(如 dtgc 0 跌停)空文件无列:门面仍返标准列空表(schema 稳定)。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
|
|
import pandas as pd
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from sanguo_portfolio.providers.fetchers import (
|
|
FETCHERS,
|
|
DataSchemaError,
|
|
EventPanelFetcher,
|
|
LimitPoolFetcher,
|
|
)
|
|
from sanguo_portfolio.providers.local_unified_provider import LocalUnifiedProvider
|
|
|
|
# 真实列名(VPS 20260819_zt_pool_dtgc.parquet 实测)
|
|
ZT_COLS = {
|
|
"序号", "代码", "名称", "涨跌幅", "最新价", "成交额", "流通市值", "总市值",
|
|
"换手率", "封板资金", "首次封板时间", "最后封板时间", "炸板次数",
|
|
"涨停统计", "连板数", "所属行业",
|
|
}
|
|
ZBGC_COLS = {
|
|
"序号", "代码", "名称", "涨跌幅", "最新价", "涨停价", "成交额", "流通市值",
|
|
"总市值", "换手率", "涨速", "首次封板时间", "炸板次数", "涨停统计",
|
|
"振幅", "所属行业",
|
|
}
|
|
DTGC_COLS = {
|
|
"序号", "代码", "名称", "涨跌幅", "最新价", "成交额", "流通市值", "总市值",
|
|
"动态市盈率", "换手率", "封单资金", "最后封板时间", "板上成交额",
|
|
"连续跌停", "开板次数", "所属行业",
|
|
}
|
|
|
|
# 门面标准键(每 kind 的完整 schema;trade_date 由读取层追加)
|
|
ZT_KEYS = {
|
|
"code", "name", "change_pct", "price", "amount", "float_market_cap",
|
|
"total_market_cap", "turnover_rate", "seal_amount", "first_seal_time",
|
|
"last_seal_time", "break_count", "limit_stat", "consecutive_boards",
|
|
"industry", "trade_date",
|
|
}
|
|
ZBGC_KEYS = ZT_KEYS - {"seal_amount", "last_seal_time", "consecutive_boards"} | {
|
|
"limit_price", "surge_speed", "amplitude"}
|
|
DTGC_KEYS = ZT_KEYS - {"first_seal_time", "break_count",
|
|
"consecutive_boards", "limit_stat"} | {
|
|
"pe_ttm", "board_amount", "consecutive_limit_downs", "open_count"}
|
|
|
|
|
|
def _mk_zt_row(code: str, boards: int, industry: str = "白酒") -> dict:
|
|
return {
|
|
"序号": 1, "代码": code, "名称": "样本股", "涨跌幅": 10.0, "最新价": 11.0,
|
|
"成交额": 1e8, "流通市值": 5e9, "总市值": 8e9, "换手率": 5.0,
|
|
"封板资金": 2e8, "首次封板时间": "092500", "最后封板时间": "150000",
|
|
"炸板次数": 0, "涨停统计": "2天/2板", "连板数": boards, "所属行业": industry,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def provider(tmp_path):
|
|
"""交易日历(2024-06-18/20,19=周三非交易日) + static 各型样本文件。
|
|
|
|
2024-06-19=周中「法定假日」形态: 采集层按周一~五落盘会为它写
|
|
「上一交易日态复制品」(快照族语义)——用它钉死 trading_days_only 过滤。
|
|
"""
|
|
db = tmp_path / "quant_trading.db"
|
|
c = sqlite3.connect(str(db))
|
|
c.execute(
|
|
"CREATE TABLE dbbardata(symbol TEXT, exchange TEXT, datetime TEXT, "
|
|
"interval TEXT, volume REAL, turnover REAL, open_interest REAL, "
|
|
"open_price REAL, high_price REAL, low_price REAL, close_price REAL)"
|
|
)
|
|
for d in ("2024-06-18", "2024-06-20"):
|
|
c.execute(
|
|
"INSERT INTO dbbardata VALUES(?,?,?,?,?,?,?,?,?,?,?)",
|
|
("600519", "SSE", f"{d} 00:00:00", "d", 1000, 1e6, 0,
|
|
1.0, 1.0, 1.0, 1.0))
|
|
c.commit()
|
|
c.close()
|
|
|
|
static = tmp_path / "static"
|
|
|
|
def _write(t: str, day: str, rows: list) -> None:
|
|
d = static / t
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
pd.DataFrame(rows, columns=list(rows[0]) if rows else None).to_parquet(
|
|
d / f"{day}_{t}.parquet", index=False)
|
|
|
|
_write("zt_pool", "20240618", [_mk_zt_row("600519", 2), _mk_zt_row("000001", 3)])
|
|
_write("zt_pool", "20240619", [_mk_zt_row("600519", 1)]) # 假日复制品
|
|
_write("zt_pool_zbgc", "20240619", [{
|
|
"序号": 1, "代码": "300999", "名称": "炸板股", "涨跌幅": 9.5, "最新价": 8.8,
|
|
"涨停价": 9.9, "成交额": 5e7, "流通市值": 2e9, "总市值": 3e9, "换手率": 12.0,
|
|
"涨速": 3.2, "首次封板时间": "093000", "炸板次数": 2, "涨停统计": "1天/0板",
|
|
"振幅": 8.0, "所属行业": "半导体"}])
|
|
_write("zt_pool_dtgc", "20240618", []) # 真空日(0 跌停)=空文件无列
|
|
_write("gdhs", "20240331", [
|
|
{"代码": "600519", "股东户数": 80000, "区间增减": -0.05},
|
|
{"代码": "000001", "股东户数": 500000, "区间增减": 0.02}])
|
|
return LocalUnifiedProvider({"db_path": str(db), "data_dir": str(tmp_path)})
|
|
|
|
|
|
# ======================== QueryParams strict(fail-fast) ========================
|
|
|
|
class TestEventPanelQueryParams:
|
|
|
|
def test_unknown_type_rejected(self, provider):
|
|
with pytest.raises(ValidationError, match="no_such_type"):
|
|
provider.get_event_panel("no_such_type", date="2024-06-18")
|
|
|
|
def test_hot_rank_not_mounted(self, provider):
|
|
"""hot_rank 墙测量未拍板挂载——白名单拒绝(与 PANEL_TYPES 口径一致)。"""
|
|
with pytest.raises(ValidationError, match="hot_rank"):
|
|
provider.get_event_panel("hot_rank", date="2024-06-18")
|
|
|
|
def test_date_and_start_exclusive(self, provider):
|
|
with pytest.raises(ValidationError, match="互斥"):
|
|
provider.get_event_panel(
|
|
"zt_pool", date="2024-06-18", start="2024-06-18",
|
|
end="2024-06-20")
|
|
|
|
def test_neither_date_nor_start(self, provider):
|
|
with pytest.raises(ValidationError, match="其一"):
|
|
provider.get_event_panel("zt_pool")
|
|
|
|
def test_start_requires_end(self, provider):
|
|
with pytest.raises(ValidationError, match="start 和 end"):
|
|
provider.get_event_panel("zt_pool", start="2024-06-18")
|
|
|
|
def test_bad_date_format(self, provider):
|
|
with pytest.raises(ValidationError):
|
|
provider.get_event_panel("zt_pool", date="20240618")
|
|
|
|
|
|
class TestLimitPoolQueryParams:
|
|
|
|
def test_unknown_kind_rejected(self, provider):
|
|
with pytest.raises(ValidationError, match="kind"):
|
|
provider.get_limit_pool(kind="ztbad", date="2024-06-18")
|
|
|
|
def test_date_semantics_shared(self, provider):
|
|
with pytest.raises(ValidationError, match="其一"):
|
|
provider.get_limit_pool(kind="zt")
|
|
|
|
|
|
# ======================== 泛型底座(保真通道) ========================
|
|
|
|
class TestEventPanelFetch:
|
|
|
|
def test_single_date_raw_columns_plus_trade_date(self, provider):
|
|
df = provider.get_event_panel("zt_pool", date="2024-06-18")
|
|
assert len(df) == 2
|
|
assert set(df.columns) >= ZT_COLS
|
|
assert "trade_date" in df.columns
|
|
assert set(df["trade_date"]) == {"2024-06-18"}
|
|
|
|
def test_range_trading_days_only_drops_holiday_file(self, provider):
|
|
"""默认过滤: 周中假日的复制品文件必须被滤掉(防区间双计数)。"""
|
|
df = provider.get_event_panel(
|
|
"zt_pool", start="2024-06-18", end="2024-06-20")
|
|
# 18(2行);19=非交易日复制品被滤;20 无文件(合法洞)
|
|
assert len(df) == 2
|
|
assert set(df["trade_date"]) == {"2024-06-18"}
|
|
|
|
def test_range_raw_mode_includes_holiday(self, provider):
|
|
df = provider.get_event_panel(
|
|
"zt_pool", start="2024-06-18", end="2024-06-20",
|
|
trading_days_only=False)
|
|
assert len(df) == 3
|
|
assert "2024-06-19" in set(df["trade_date"])
|
|
|
|
def test_missing_dir_returns_empty(self, provider):
|
|
"""从未采集的类型(xueqiu_hot 目录不存在)=合法缺失,空 df 不报错。"""
|
|
df = provider.get_event_panel("xueqiu_hot", date="2024-06-18")
|
|
assert isinstance(df, pd.DataFrame) and df.empty
|
|
|
|
def test_empty_file_day_contributes_nothing(self, provider):
|
|
df = provider.get_event_panel(
|
|
"zt_pool_dtgc", start="2024-06-18", end="2024-06-20")
|
|
assert df.empty # 18=真空文件,19/20 无文件
|
|
|
|
def test_gdhs_period_read(self, provider):
|
|
"""gdhs per-period: unit={季度末}_gdhs,同一底座直接可读。"""
|
|
df = provider.get_event_panel("gdhs", date="2024-03-31")
|
|
assert len(df) == 2
|
|
assert set(df["trade_date"]) == {"2024-03-31"}
|
|
|
|
|
|
# ======================== 涨停池门面(标准模型) ========================
|
|
|
|
class TestLimitPoolFetch:
|
|
|
|
def test_zt_english_schema(self, provider):
|
|
df = provider.get_limit_pool(kind="zt", date="2024-06-18")
|
|
assert set(df.columns) == ZT_KEYS
|
|
row = df.iloc[0]
|
|
assert row["consecutive_boards"] == 2
|
|
assert row["seal_amount"] == 2e8
|
|
assert row["break_count"] == 0
|
|
assert row["industry"] == "白酒"
|
|
assert row["limit_stat"] == "2天/2板"
|
|
|
|
def test_zbgc_schema(self, provider):
|
|
df = provider.get_limit_pool(kind="zbgc", date="2024-06-19")
|
|
assert set(df.columns) == ZBGC_KEYS
|
|
assert df.iloc[0]["break_count"] == 2
|
|
assert df.iloc[0]["surge_speed"] == 3.2
|
|
|
|
def test_dtgc_empty_day_keeps_standard_schema(self, provider):
|
|
"""真空日(0 跌停)空文件无列——门面仍返标准列空表(schema 稳定)。"""
|
|
df = provider.get_limit_pool(kind="dtgc", date="2024-06-18")
|
|
assert df.empty
|
|
assert set(df.columns) == DTGC_KEYS
|
|
|
|
def test_dtgc_real_columns_mapped(self, provider, tmp_path):
|
|
"""dtgc 真实列(实测: 封单资金/连续跌停/开板次数...)→ 标准键。"""
|
|
f = tmp_path / "static" / "zt_pool_dtgc" / "20240620_zt_pool_dtgc.parquet"
|
|
pd.DataFrame([{
|
|
"序号": 1, "代码": "600999", "名称": "跌停股", "涨跌幅": -10.0,
|
|
"最新价": 5.0, "成交额": 3e7, "流通市值": 1e9, "总市值": 2e9,
|
|
"动态市盈率": 15.0, "换手率": 8.0, "封单资金": 6e7,
|
|
"最后封板时间": "145950", "板上成交额": 1e6, "连续跌停": 2,
|
|
"开板次数": 1, "所属行业": "地产"}]).to_parquet(f, index=False)
|
|
df = provider.get_limit_pool(kind="dtgc", date="2024-06-20")
|
|
assert len(df) == 1
|
|
assert set(df.columns) == DTGC_KEYS
|
|
row = df.iloc[0]
|
|
assert row["seal_amount"] == 6e7
|
|
assert row["consecutive_limit_downs"] == 2
|
|
assert row["open_count"] == 1
|
|
assert row["pe_ttm"] == 15.0
|
|
|
|
def test_missing_code_column_fail_fast(self, provider, tmp_path):
|
|
"""脏数据: 有行但缺 代码 → DataSchemaError(门面吸收列漂移的点)。"""
|
|
f = tmp_path / "static" / "zt_pool" / "20240620_zt_pool.parquet"
|
|
pd.DataFrame([{"名称": "无名股", "连板数": 1}]).to_parquet(f, index=False)
|
|
with pytest.raises(DataSchemaError, match="代码"):
|
|
provider.get_limit_pool(kind="zt", date="2024-06-20")
|
|
|
|
def test_range_trading_filter_applies(self, provider):
|
|
df = provider.get_limit_pool(kind="zt", start="2024-06-18",
|
|
end="2024-06-20")
|
|
assert len(df) == 2
|
|
assert "2024-06-19" not in set(df["trade_date"])
|
|
|
|
|
|
# ======================== 注册表 ========================
|
|
|
|
class TestRegistry:
|
|
|
|
def test_fetchers_registered(self):
|
|
assert FETCHERS["event_panel"] is EventPanelFetcher
|
|
assert FETCHERS["limit_pool"] is LimitPoolFetcher
|