claude_dev
f074a6b420
xtdata.download_financial_data 阻塞无超时,休市/服务不响应卡死整个回测 (实证:周六休市HS300全成分首次download卡死,已缓存股票0.5s)。 包threading+join(120s)超时跳过读缓存(数据不全但回测不卡死)。 Layer1 MVP链路完整闭环:runner→BulletTrade engine→AllWeather策略→miniQMT→JSON (7.49%收益/37交易日/选股5只ETF/净值37点/指标total_return+sharpe+max_drawdown全有)。 休市致股票财务空走ETF兜底,策略数值无意义但链路100%通;周一download恢复走真实选股。
675 lines
27 KiB
Python
675 lines
27 KiB
Python
"""SanguoMiniQmtProvider:继承 MiniQMTProvider,补齐 ``get_fundamentals``。
|
||
|
||
BulletTrade 的 MiniQMTProvider 实现了行情/成分/涨跌停/证券信息,**唯一缺口**是
|
||
base.py:159 的 ``get_fundamentals``(默认抛 NotImplementedError)。本子类填这个缺口。
|
||
|
||
数据源映射(miniQMT 实证 2026-07-18,见 docs/portfolio_backtest_result.md):
|
||
- ``xtdata.get_financial_data(stock_list)`` → dict[stock][table_name] → DataFrame
|
||
table_name: 'PershareIndex' / 'Capital' / 'Balance' / 'Income' / 'CashFlow'
|
||
- ``xtdata.get_market_data_ex`` → close 行情(**end_time 必须 YYYYMMDD,带 dash 报错**)
|
||
- ``Capital[total_capital/circulating_capital]`` 单位 = **股**(不是万股),不需 ×10000
|
||
|
||
字段名差异(miniQMT 实际 vs 聚宽/本 provider 历史 alias):
|
||
- PershareIndex: du_return_on_equity→roe, s_fa_eps_basic→eps,
|
||
sales_gross_profit→gross_profit_margin, du_profit_rate→net_profit_margin,
|
||
inc_revenue_rate→inc_revenue_year_on_year,
|
||
inc_net_profit_rate→inc_operation_profit_year_on_year,
|
||
inc_total_revenue_annual→inc_total_revenue_year_on_year
|
||
- Balance: tot_liab→total_liability,
|
||
tot_shrhldr_eqy_excl_min_int→total_sheet_owner_equities,
|
||
undistributed_profit→retained_profit,
|
||
shortterm_loan→short_loan, long_term_loans→long_loan
|
||
- ROA:PershareIndex 没有,用 ROE × (equity / total_assets) 自算
|
||
|
||
聚宽 query(...) ORM 太复杂,这里支持两种入参:
|
||
1. dict: ``{'stocks': [...], 'date': 'YYYY-MM-DD'}`` 直接返合并 DataFrame(推荐,策略用)
|
||
2. jq-style query_object: 支持 ``filter(==/>/</between/in_)``/``order_by``/``limit``
|
||
复杂 filter 抛 NotImplementedError(标注清楚)
|
||
|
||
Mac 没有 xtquant,所有 xtquant 调用通过 ``self._ensure_xtdata()``,可被 mock 注入。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from datetime import datetime
|
||
from typing import Any, Dict, List, Optional, Union
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# bullet-trade 可能正在装,容错 import
|
||
try:
|
||
from bullet_trade.data.providers.miniqmt import MiniQMTProvider # type: ignore
|
||
_HAS_BT_BASE = True
|
||
_BT_IMPORT_ERROR: Optional[Exception] = None
|
||
except ImportError as _e: # Mac dev 环境可能未装,允许模块加载
|
||
MiniQMTProvider = object # type: ignore[misc,assignment]
|
||
_HAS_BT_BASE = False
|
||
_BT_IMPORT_ERROR = _e
|
||
|
||
|
||
# 聚宽 valuation/indicator/balance 列名 → 我们合并 DataFrame 的列名
|
||
# (统一用聚宽列名,方便策略层直接 pandas 筛选)
|
||
JQ_COLUMN_ALIASES: Dict[str, str] = {
|
||
# valuation 表(自算)
|
||
"market_cap": "market_cap",
|
||
"circulating_market_cap": "circulating_market_cap",
|
||
"pe_ratio": "pe_ratio",
|
||
"pb_ratio": "pb_ratio",
|
||
"ps_ratio": "ps_ratio",
|
||
"pcf_ratio": "pcf_ratio",
|
||
"code": "code",
|
||
# indicator(PershareIndex 直接拿)
|
||
"roe": "roe",
|
||
"roa": "roa",
|
||
"eps": "eps",
|
||
"gross_profit_margin": "gross_profit_margin",
|
||
"net_profit_margin": "net_profit_margin",
|
||
"inc_revenue_year_on_year": "inc_revenue_year_on_year",
|
||
"inc_operation_profit_year_on_year": "inc_operation_profit_year_on_year",
|
||
"inc_total_revenue_year_on_year": "inc_total_revenue_year_on_year",
|
||
# balance(合并表)
|
||
"total_liability": "total_liability",
|
||
"total_sheet_owner_equities": "total_sheet_owner_equities",
|
||
"retained_profit": "retained_profit",
|
||
}
|
||
|
||
|
||
class SanguoMiniQmtProvider(MiniQMTProvider): # type: ignore[misc]
|
||
"""miniQMT + 自算估值/ROIC 的 fundamentals provider。
|
||
|
||
继承 MiniQMTProvider 的行情/成分/证券信息/涨跌停能力,**只补 get_fundamentals**。
|
||
策略层用 ``get_fundamentals_df(stocks, date)`` 直接拿合并 DataFrame 做 pandas 筛选,
|
||
避开解析聚宽 query ORM。
|
||
"""
|
||
|
||
name: str = "sanguo_miniqmt"
|
||
|
||
def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
|
||
if not _HAS_BT_BASE:
|
||
raise RuntimeError(
|
||
f"bullet-trade 未安装,SanguoMiniQmtProvider 无法继承 MiniQMTProvider: "
|
||
f"{_BT_IMPORT_ERROR}"
|
||
)
|
||
super().__init__(config or {})
|
||
|
||
# ------------------------ 主入口 ------------------------
|
||
def get_fundamentals(
|
||
self,
|
||
query_object: Any,
|
||
date: Optional[Union[str, datetime]] = None,
|
||
statDate: Optional[str] = None,
|
||
) -> pd.DataFrame:
|
||
"""聚宽风格 query + 直接 dict 两种入参。
|
||
|
||
Args:
|
||
query_object:
|
||
- dict: ``{'stocks': [...], 'date': 'YYYY-MM-DD'}`` 直接返合并 DataFrame
|
||
- 其它: 当作 jq query ORM 解析(支持 filter/order_by/limit 子集)
|
||
date: 查询日期(YYYY-MM-DD 或 datetime),用于取 close 和报告期
|
||
statDate: 聚宽 statDate 风格('2023Q3'/'2023'),暂不支持,传则忽略并告警
|
||
|
||
Returns:
|
||
聚宽 get_fundamentals 语义的 DataFrame:列见 ``JQ_COLUMN_ALIASES``。
|
||
"""
|
||
if statDate is not None:
|
||
logger.warning("statDate=%s 暂不支持,忽略,用最近报告期", statDate)
|
||
|
||
# 入参分支 1: dict 直接给 stocks
|
||
if isinstance(query_object, dict):
|
||
stocks = list(query_object.get("stocks") or [])
|
||
query_date = query_object.get("date") or date
|
||
df = self.get_fundamentals_df(stocks, query_date)
|
||
extra_filter = query_object.get("filter")
|
||
if extra_filter and callable(extra_filter):
|
||
df = df[extra_filter(df)]
|
||
order_by = query_object.get("order_by")
|
||
if order_by:
|
||
df = _apply_order_by(df, order_by)
|
||
limit = query_object.get("limit")
|
||
if isinstance(limit, int) and limit > 0:
|
||
df = df.head(limit)
|
||
return df
|
||
|
||
# 入参分支 2: 当作 jq-style query ORM
|
||
df = self._resolve_stocks_from_query(query_object, date)
|
||
return df
|
||
|
||
# ------------------------ 策略层便捷方法 ------------------------
|
||
def get_fundamentals_df(
|
||
self,
|
||
stocks: List[str],
|
||
date: Optional[Union[str, datetime]] = None,
|
||
) -> pd.DataFrame:
|
||
"""合并 PershareIndex + Balance + Income + CashFlow + Capital + close。
|
||
|
||
返回 DataFrame 每行一只股票,index 是 jq-style code(如 ``600519.XSHG``)。
|
||
列含:code/market_cap/circulating_market_cap/pe_ratio/pb_ratio/ps_ratio/pcf_ratio
|
||
+ roe/roa/eps/gross_profit_margin/net_profit_margin/inc_revenue_year_on_year/
|
||
inc_operation_profit_year_on_year/inc_total_revenue_year_on_year
|
||
+ total_liability/total_sheet_owner_equities/retained_profit
|
||
+ roic(自算)+归母净利润/营收/经营现金流(供策略再算其它因子)
|
||
"""
|
||
if not stocks:
|
||
return pd.DataFrame(columns=list(JQ_COLUMN_ALIASES.values()))
|
||
|
||
xt = self._ensure_xtdata()
|
||
date_str = _to_date_str(date)
|
||
|
||
# Step 1: 拉财务数据(miniQMT 返回 dict[stock_code] -> dict[table] -> DataFrame)
|
||
# 成分股用 jq code,xtdata 要 QMT 风格("600519.SH"),通过 _normalize_security_code
|
||
qmt_stocks = [self._normalize_security_code(s) for s in stocks]
|
||
try:
|
||
if self.auto_download:
|
||
# download_financial_data 阻塞无超时,休市/服务不响应会卡死整个回测;
|
||
# 包 threading 超时,超时则跳过读缓存(数据可能不全但回测不卡死)
|
||
self._download_financial_safe(
|
||
qmt_stocks, ["PershareIndex", "Balance", "Capital"]
|
||
)
|
||
fin_data = xt.get_financial_data(qmt_stocks)
|
||
except Exception as exc:
|
||
logger.warning("get_financial_data 失败,返空表: %s", exc)
|
||
return pd.DataFrame(columns=list(JQ_COLUMN_ALIASES.values()))
|
||
|
||
# Step 2: 拉当日收盘(用 date_str 或最近一日)
|
||
close_map = self._fetch_close(qmt_stocks, date_str)
|
||
|
||
# Step 3: 组装每行
|
||
rows: List[Dict[str, Any]] = []
|
||
for jq_code, qmt_code in zip(stocks, qmt_stocks):
|
||
stock_fin = fin_data.get(qmt_code) or fin_data.get(jq_code) or {}
|
||
close = close_map.get(qmt_code) or close_map.get(jq_code)
|
||
row = self._build_row(jq_code, qmt_code, stock_fin, close)
|
||
rows.append(row)
|
||
|
||
df = pd.DataFrame(rows)
|
||
if "code" in df.columns:
|
||
df = df.set_index("code", drop=False)
|
||
return df
|
||
|
||
def _download_financial_safe(
|
||
self, qmt_stocks: List[str], tables: List[str], timeout: float = 120.0
|
||
) -> None:
|
||
"""download_financial_data 包 threading 超时。
|
||
|
||
xtdata.download_financial_data 阻塞且无超时参数,休市/服务不响应时卡死
|
||
整个回测。用线程 + join(timeout):超时则放弃(读缓存),daemon 线程随进程退出清理。
|
||
"""
|
||
import threading
|
||
xt = self._ensure_xtdata()
|
||
err: List[str] = []
|
||
|
||
def _worker() -> None:
|
||
try:
|
||
xt.download_financial_data(qmt_stocks, tables)
|
||
except Exception as exc:
|
||
err.append(str(exc))
|
||
|
||
t = threading.Thread(target=_worker, daemon=True)
|
||
t.start()
|
||
t.join(timeout)
|
||
if t.is_alive():
|
||
logger.warning(
|
||
"download_financial_data 超时 %.0fs (%d 只),跳过读缓存",
|
||
timeout, len(qmt_stocks),
|
||
)
|
||
elif err:
|
||
logger.debug("download_financial_data 失败(继续读缓存): %s", err[0])
|
||
|
||
# ------------------------ 内部组装 ------------------------
|
||
def _build_row(
|
||
self,
|
||
jq_code: str,
|
||
qmt_code: str,
|
||
stock_fin: Dict[str, Any],
|
||
close: Optional[float],
|
||
) -> Dict[str, Any]:
|
||
"""合并 PershareIndex(最新一行) + Balance + Income + CashFlow + Capital + close。"""
|
||
from .. import factors # lazy import,避免循环
|
||
|
||
row: Dict[str, Any] = {"code": jq_code}
|
||
|
||
# Capital(股本,实证单位 = 股,不再 ×10000)
|
||
capital = _latest_row(stock_fin.get("Capital"))
|
||
total_capital = _to_float(_get(capital, "total_capital"))
|
||
circulating_capital = _to_float(_get(capital, "circulating_capital"))
|
||
|
||
# Balance(实测字段名 tot_liab / tot_shrhldr_eqy_excl_min_int / undistributed_profit /
|
||
# shortterm_loan / long_term_loans;历史 alias 用 total_liability / retained_profit /
|
||
# short_loan / long_loan,_get_multi 兼容两套)
|
||
balance = _latest_row(stock_fin.get("Balance"))
|
||
tot_shrhldr_eqy = _to_float(_get_multi(balance, ["tot_shrhldr_eqy_excl_min_int", "total_sheet_owner_equities"]))
|
||
total_liability = _to_float(_get_multi(balance, ["total_liability", "tot_liab"]))
|
||
total_sheet_owner_equities = _to_float(_get_multi(balance, ["total_sheet_owner_equities", "tot_shrhldr_eqy_excl_min_int", "total_equity"]))
|
||
retained_profit = _to_float(_get_multi(balance, ["retained_profit", "undistributed_profit"]))
|
||
cash_equivalents = _to_float(_get_multi(balance, ["cash_equivalents", "monetary_funds"]))
|
||
short_loan = _to_float(_get_multi(balance, ["short_loan", "shortterm_loan"])) or 0.0
|
||
long_loan = _to_float(_get_multi(balance, ["long_loan", "long_term_loans"])) or 0.0
|
||
bonds_payable = _to_float(_get(balance, "bonds_payable")) or 0.0
|
||
total_assets = _to_float(_get_multi(balance, ["tot_assets", "total_assets"]))
|
||
interest_bearing_debt = short_loan + long_loan + bonds_payable
|
||
|
||
# Income(miniQMT 实证 trading hours 常下载超时,空表时用 PershareIndex EPS 反推)
|
||
income = _latest_row(stock_fin.get("Income"))
|
||
net_profit = _to_float(_get_multi(income, ["net_profit_excl_min_int", "n_income", "net_profit_incl_min_int"]))
|
||
revenue = _to_float(_get_multi(income, ["operating_revenue", "revenue", "total_revenue", "operating revenue"]))
|
||
oper_profit = _to_float(_get_multi(income, ["oper_profit", "operating_profit"]))
|
||
inc_tax = _to_float(_get_multi(income, ["inc_tax", "income_tax"]))
|
||
profit_before_tax = _to_float(_get_multi(income, ["profit_before_tax"]))
|
||
|
||
# CashFlow(同 Income,trading hours 常空)
|
||
cashflow = _latest_row(stock_fin.get("CashFlow"))
|
||
net_oper_cash_flow = _to_float(_get_multi(cashflow, ["n_cashflow_act", "net_operate_cash_flow", "net_cash_flow_oper"]))
|
||
|
||
# PershareIndex(实测字段名 du_return_on_equity/s_fa_eps_basic/sales_gross_profit 等)
|
||
psh = _latest_row(stock_fin.get("PershareIndex"))
|
||
roe = _to_float(_get_multi(psh, ["roe", "du_return_on_equity", "equity_roe"]))
|
||
roa = _to_float(_get_multi(psh, ["roa", "return_on_assets", "total_roe"]))
|
||
eps = _to_float(_get_multi(psh, ["eps", "s_fa_eps_basic", "s_fa_eps_diluted"]))
|
||
gross_profit_margin = _to_float(_get_multi(psh, ["gross_profit_margin", "sales_gross_profit", "gross_profit"]))
|
||
net_profit_margin = _to_float(_get_multi(psh, ["net_profit_margin", "du_profit_rate", "net_profit"]))
|
||
inc_revenue_yoy = _to_float(_get_multi(psh, ["inc_revenue_year_on_year", "inc_revenue_rate", "inc_revenue"]))
|
||
inc_operation_profit_yoy = _to_float(_get_multi(psh, [
|
||
"inc_operation_profit_year_on_year", "inc_net_profit_rate", "inc_net_profit",
|
||
]))
|
||
# 营业总收入同比(annual 口径优先,回退 rate)
|
||
inc_total_revenue_yoy = _to_float(_get_multi(psh, [
|
||
"inc_total_revenue_year_on_year", "inc_total_revenue_annual", "inc_revenue_rate",
|
||
]))
|
||
actual_tax_rate = _to_float(_get(psh, "actual_tax_rate"))
|
||
|
||
# Income 空表兜底:用 EPS × total_capital 得单季净利润(calc_pe 内部 ×4 近似 TTM)
|
||
# 注意:不能在这里 ×4,calc_pe 已经 ×4,重复 ×4 会致 PE 偏小 4 倍
|
||
if (net_profit is None) and eps is not None and total_capital and total_capital > 0:
|
||
net_profit = eps * total_capital
|
||
logger.debug("%s Income 空,用 EPS×股本 近似单季 net_profit=%s", jq_code, net_profit)
|
||
|
||
# ROA 兜底:PershareIndex 没有 roa 字段,用 ROE × (归母权益/总资产) 自算
|
||
if (roa is None) and roe is not None and tot_shrhldr_eqy and total_assets and total_assets > 0:
|
||
# ROE 是百分数(如 10.57),保持百分数口径
|
||
roa = roe * (tot_shrhldr_eqy / total_assets)
|
||
|
||
# 估值(自算,close None 时全置 NaN)
|
||
if close is not None and total_capital and total_capital > 0:
|
||
# 聚宽口径:市值亿元
|
||
row["market_cap"] = factors.valuation.to_yi(close * total_capital)
|
||
row["circulating_market_cap"] = factors.valuation.to_yi(
|
||
close * (circulating_capital or total_capital)
|
||
)
|
||
if net_profit is not None:
|
||
row["pe_ratio"] = factors.valuation.calc_pe(close, net_profit, total_capital)
|
||
else:
|
||
row["pe_ratio"] = float("nan")
|
||
if tot_shrhldr_eqy:
|
||
row["pb_ratio"] = factors.valuation.calc_pb(close, tot_shrhldr_eqy, total_capital)
|
||
else:
|
||
row["pb_ratio"] = float("nan")
|
||
if revenue is not None:
|
||
row["ps_ratio"] = factors.valuation.calc_ps(close, revenue, total_capital)
|
||
else:
|
||
row["ps_ratio"] = float("nan")
|
||
if net_oper_cash_flow is not None:
|
||
row["pcf_ratio"] = factors.valuation.calc_pcf(close, net_oper_cash_flow, total_capital)
|
||
else:
|
||
row["pcf_ratio"] = float("nan")
|
||
else:
|
||
row["market_cap"] = float("nan")
|
||
row["circulating_market_cap"] = float("nan")
|
||
row["pe_ratio"] = float("nan")
|
||
row["pb_ratio"] = float("nan")
|
||
row["ps_ratio"] = float("nan")
|
||
row["pcf_ratio"] = float("nan")
|
||
|
||
# indicator
|
||
# 实证 miniQMT PershareIndex 返回的是百分数(茅台 ROE=10.57 表示 10.57%),
|
||
# 聚宽 valuation/indicator 是小数(0.1057)。策略层阈值 roe>0.15 按聚宽口径,
|
||
# 这里统一归一到小数,×0.01。NaN 透传。
|
||
row["roe"] = _pct_to_decimal(roe)
|
||
row["roa"] = _pct_to_decimal(roa)
|
||
row["eps"] = _or_nan(eps)
|
||
row["gross_profit_margin"] = _pct_to_decimal(gross_profit_margin)
|
||
row["net_profit_margin"] = _pct_to_decimal(net_profit_margin)
|
||
row["inc_revenue_year_on_year"] = _pct_to_decimal(inc_revenue_yoy)
|
||
row["inc_operation_profit_year_on_year"] = _pct_to_decimal(inc_operation_profit_yoy)
|
||
row["inc_total_revenue_year_on_year"] = _pct_to_decimal(inc_total_revenue_yoy)
|
||
|
||
# balance(策略层会算 total_liability/total_sheet_owner_equities 比率)
|
||
row["total_liability"] = _or_nan(total_liability)
|
||
row["total_sheet_owner_equities"] = _or_nan(total_sheet_owner_equities)
|
||
row["retained_profit"] = _or_nan(retained_profit)
|
||
|
||
# ROIC 自算
|
||
if oper_profit is not None and tot_shrhldr_eqy and interest_bearing_debt is not None and cash_equivalents is not None:
|
||
row["roic"] = factors.roic.calc_roic(
|
||
oper_profit, actual_tax_rate, tot_shrhldr_eqy,
|
||
interest_bearing_debt, cash_equivalents,
|
||
inc_tax=inc_tax, profit_before_tax=profit_before_tax,
|
||
)
|
||
else:
|
||
row["roic"] = float("nan")
|
||
|
||
# 留原始字段给策略做更多自算
|
||
row["_net_profit"] = _or_nan(net_profit)
|
||
row["_revenue"] = _or_nan(revenue)
|
||
row["_oper_profit"] = _or_nan(oper_profit)
|
||
row["_total_capital"] = total_capital
|
||
row["_close"] = close if close is not None else float("nan")
|
||
return row
|
||
|
||
# ------------------------ close 行情 ------------------------
|
||
def _fetch_close(self, qmt_stocks: List[str], date_str: Optional[str]) -> Dict[str, float]:
|
||
"""取 ``date`` 当日(或最近一日)收盘价。失败返空 dict。
|
||
|
||
实证 miniQMT ``get_market_data_ex`` 的 ``end_time`` 必须 YYYYMMDD(带 dash 报
|
||
"结束时间错误"),这里归一。
|
||
"""
|
||
xt = self._ensure_xtdata()
|
||
try:
|
||
# get_market_data_ex([], stock_list, period='1d', end_time=date, count=1)
|
||
end = _to_yyyymmdd(date_str) or ""
|
||
result = xt.get_market_data_ex(
|
||
[], qmt_stocks, period="1d", start_time="", end_time=end, count=1
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("get_market_data_ex 失败: %s", exc)
|
||
return {}
|
||
if not result:
|
||
return {}
|
||
out: Dict[str, float] = {}
|
||
for code, df in result.items():
|
||
if df is None or len(df) == 0:
|
||
continue
|
||
try:
|
||
close = float(df["close"].iloc[-1])
|
||
out[code] = close
|
||
except Exception:
|
||
continue
|
||
return out
|
||
|
||
# ------------------------ jq query 解析(最小子集) ------------------------
|
||
def _resolve_stocks_from_query(
|
||
self, query_object: Any, date: Optional[Union[str, datetime]]
|
||
) -> pd.DataFrame:
|
||
"""解析 jq-style query 的 stocks 列表 + filter/order_by/limit。
|
||
|
||
支持:
|
||
- ``query(valuation, indicator).filter(valuation.code.in_(...), indicator.roe > 0.15)``
|
||
- ``...order_by(valuation.market_cap.asc()/.desc())``
|
||
- ``...limit(N)``
|
||
复杂 filter(OR / 跨表 join / 自定义函数)抛 NotImplementedError。
|
||
"""
|
||
# 解析 stocks:从 filter 的 code.in_(...) 提取
|
||
stocks = _extract_stocks_from_query(query_object)
|
||
if not stocks:
|
||
logger.warning("无法从 query 提取 stocks(可能用了不支持的 filter),返空表")
|
||
return pd.DataFrame(columns=list(JQ_COLUMN_ALIASES.values()))
|
||
df = self.get_fundamentals_df(stocks, date)
|
||
# 应用 filter(简单比较/范围)
|
||
df = _apply_query_filters(df, query_object)
|
||
# 应用 order_by
|
||
order_by = _extract_order_by(query_object)
|
||
if order_by:
|
||
df = _apply_order_by(df, order_by)
|
||
# 应用 limit
|
||
limit = _extract_limit(query_object)
|
||
if isinstance(limit, int) and limit > 0:
|
||
df = df.head(limit)
|
||
return df
|
||
|
||
|
||
# ======================== jq query ORM 解析辅助 ========================
|
||
# bullet-trade 的 query 是轻量 ORM(filter/order_by/limit 返回 self)。
|
||
# 我们不依赖它的具体类型,鸭子取属性即可。
|
||
|
||
def _extract_stocks_from_query(query_object: Any) -> List[str]:
|
||
"""从 query 的 code.in_(...) 子句提取股票池。
|
||
|
||
bullet-trade 的 query 对象上 filter 条件可能存在 ``_filters`` / ``_wheres`` 等内部字段,
|
||
我们一律用鸭子反射 + 字符串模式提取 in_(...)。
|
||
"""
|
||
text = _stringify_query(query_object)
|
||
if not text:
|
||
return []
|
||
import re
|
||
# 匹配 in_('600519.XSHG', '000001.XSHE', ...)
|
||
matches = re.findall(r"in_\(\s*\[([^\]]*)\]\s*\)", text)
|
||
if not matches:
|
||
matches = re.findall(r"in_\(([^)]+)\)", text)
|
||
if not matches:
|
||
return []
|
||
found: List[str] = []
|
||
for chunk in matches:
|
||
for code in re.findall(r"['\"]([0-9A-Za-z]+\.[A-Z]+)['\"]", chunk):
|
||
found.append(code)
|
||
return _dedup(found)
|
||
|
||
|
||
def _stringify_query(query_object: Any) -> str:
|
||
"""把 query 对象转字符串,各种失败容错。"""
|
||
try:
|
||
return str(query_object)
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def _apply_query_filters(df: pd.DataFrame, query_object: Any) -> pd.DataFrame:
|
||
"""从 query 提取 filter 条件,逐条应用到 df。
|
||
|
||
支持:col == v / > v / < v / >= v / <= v / between(a,b)。
|
||
不支持 OR / 跨表 / 自定义函数 → NotImplementedError(标注 v2)。
|
||
"""
|
||
text = _stringify_query(query_object)
|
||
if not text:
|
||
return df
|
||
import re
|
||
|
||
# 简单比较: col > 0.15 / col < 30 / col == 1 / col >= 0 / col <= 0
|
||
# 字段名带点(valuation.market_cap / indicator.roe),取点后那段
|
||
patterns = [
|
||
(r"(\w+\.)?(\w+)\s*(>=|<=|==|!=|>|<)\s*([0-9eE.+-]+)", "compare"),
|
||
(r"(\w+\.)?(\w+)\.between\(\s*([0-9eE.+-]+)\s*,\s*([0-9eE.+-]+)\s*\)", "between"),
|
||
]
|
||
for pat, kind in patterns:
|
||
for m in re.finditer(pat, text):
|
||
col = m.group(2)
|
||
if col not in df.columns:
|
||
continue
|
||
if kind == "compare":
|
||
op, val = m.group(3), float(m.group(4))
|
||
df = _apply_compare(df, col, op, val)
|
||
else:
|
||
lo, hi = float(m.group(3)), float(m.group(4))
|
||
df = df[(df[col] >= lo) & (df[col] <= hi)]
|
||
return df
|
||
|
||
|
||
def _apply_compare(df: pd.DataFrame, col: str, op: str, val: float) -> pd.DataFrame:
|
||
if op == ">":
|
||
return df[df[col] > val]
|
||
if op == "<":
|
||
return df[df[col] < val]
|
||
if op == ">=":
|
||
return df[df[col] >= val]
|
||
if op == "<=":
|
||
return df[df[col] <= val]
|
||
if op == "==":
|
||
return df[df[col] == val]
|
||
if op == "!=":
|
||
return df[df[col] != val]
|
||
return df
|
||
|
||
|
||
def _extract_order_by(query_object: Any) -> List[tuple]:
|
||
"""提取 order_by 子句 → [(col, 'asc'|'desc'), ...]。
|
||
|
||
用字符串解析,匹配 valuation.market_cap.asc() / .desc()。
|
||
"""
|
||
text = _stringify_query(query_object)
|
||
if not text:
|
||
return []
|
||
import re
|
||
out = []
|
||
for m in re.finditer(r"(\w+\.)?(\w+)\.(asc|desc)\(\)", text):
|
||
out.append((m.group(2), m.group(3)))
|
||
return out
|
||
|
||
|
||
def _apply_order_by(df: pd.DataFrame, order_by: List[tuple]) -> pd.DataFrame:
|
||
if not order_by:
|
||
return df
|
||
for col, direction in reversed(order_by):
|
||
if col not in df.columns:
|
||
continue
|
||
ascending = direction != "desc"
|
||
df = df.sort_values(col, ascending=ascending, na_position="last")
|
||
return df
|
||
|
||
|
||
def _extract_limit(query_object: Any) -> Optional[int]:
|
||
text = _stringify_query(query_object)
|
||
if not text:
|
||
return None
|
||
import re
|
||
m = re.search(r"\.limit\(\s*(\d+)\s*\)", text)
|
||
return int(m.group(1)) if m else None
|
||
|
||
|
||
# ======================== 工具 ========================
|
||
def _latest_row(table: Any) -> Optional[pd.Series]:
|
||
"""取财务表 DataFrame 的最新一行(按报告期 date 倒序)。
|
||
|
||
xtquant 返回的 PershareIndex/Balance 等是 DataFrame,index 通常为报告期。
|
||
"""
|
||
if table is None:
|
||
return None
|
||
if isinstance(table, pd.DataFrame):
|
||
if len(table) == 0:
|
||
return None
|
||
# 尝试按 index(报告期)降序取最新
|
||
try:
|
||
return table.sort_index(ascending=False).iloc[0]
|
||
except Exception:
|
||
return table.iloc[-1]
|
||
if isinstance(table, dict):
|
||
return pd.Series(table)
|
||
return None
|
||
|
||
|
||
def _get(series_or_dict: Any, key: str) -> Any:
|
||
"""从 Series/dict 取 key,容错 key 不存在/大小写差异。"""
|
||
if series_or_dict is None:
|
||
return None
|
||
if isinstance(series_or_dict, pd.Series):
|
||
if key in series_or_dict:
|
||
return series_or_dict[key]
|
||
# case-insensitive fallback
|
||
lower_map = {k.lower(): k for k in series_or_dict.index}
|
||
if key.lower() in lower_map:
|
||
return series_or_dict[lower_map[key.lower()]]
|
||
return None
|
||
if isinstance(series_or_dict, dict):
|
||
if key in series_or_dict:
|
||
return series_or_dict[key]
|
||
for k, v in series_or_dict.items():
|
||
if k.lower() == key.lower():
|
||
return v
|
||
return None
|
||
return None
|
||
|
||
|
||
def _get_multi(series_or_dict: Any, keys: List[str]) -> Any:
|
||
"""按 keys 顺序尝试取,第一个非 None(且非 NaN)的值返回。
|
||
|
||
用于字段 alias 兼容(如 roe/du_return_on_equity/equity_roe 三套名字)。
|
||
"""
|
||
for k in keys:
|
||
v = _get(series_or_dict, k)
|
||
if v is None:
|
||
continue
|
||
# NaN 透传到下一个 key
|
||
try:
|
||
fv = float(v)
|
||
if np.isnan(fv):
|
||
continue
|
||
return v
|
||
except (TypeError, ValueError):
|
||
return v
|
||
return None
|
||
|
||
|
||
def _to_yyyymmdd(value: Optional[str]) -> Optional[str]:
|
||
"""YYYY-MM-DD → YYYYMMDD(miniQMT get_market_data_ex 要求)。已 YYYYMMDD 直接返。"""
|
||
if value is None:
|
||
return None
|
||
s = str(value).strip()
|
||
if not s:
|
||
return None
|
||
if "-" in s:
|
||
return s.replace("-", "")[:8]
|
||
return s[:8]
|
||
|
||
|
||
def _to_float(value: Any) -> Optional[float]:
|
||
"""Any → float,None/NaN/异常 → None。"""
|
||
if value is None:
|
||
return None
|
||
try:
|
||
out = float(value)
|
||
if np.isnan(out):
|
||
return None
|
||
return out
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _or_nan(value: Optional[float]) -> float:
|
||
if value is None:
|
||
return float("nan")
|
||
return float(value)
|
||
|
||
|
||
def _pct_to_decimal(value: Optional[float]) -> float:
|
||
"""百分数 → 小数(10.57 → 0.1057)对齐聚宽 indicator 口径。
|
||
|
||
NaN/None 透传。绝对值 < 1 时认为已经是小数,不转换(防御)。
|
||
"""
|
||
if value is None:
|
||
return float("nan")
|
||
try:
|
||
v = float(value)
|
||
except (TypeError, ValueError):
|
||
return float("nan")
|
||
if np.isnan(v):
|
||
return float("nan")
|
||
if abs(v) < 1:
|
||
return v
|
||
return v / 100.0
|
||
|
||
|
||
def _to_date_str(value: Optional[Union[str, datetime]]) -> Optional[str]:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, str):
|
||
return value[:10]
|
||
if isinstance(value, datetime):
|
||
return value.strftime("%Y-%m-%d")
|
||
try:
|
||
return str(value)[:10]
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _dedup(seq: List[str]) -> List[str]:
|
||
seen = set()
|
||
out = []
|
||
for x in seq:
|
||
if x not in seen:
|
||
seen.add(x)
|
||
out.append(x)
|
||
return out
|
||
|
||
|
||
__all__ = ["SanguoMiniQmtProvider", "JQ_COLUMN_ALIASES"]
|