fix(data): akshare top_holders Length mismatch bug 根治
bug 根因 (实证 akshare 1.18.x stock_gdfx_em.py:418):
报告期未披露 (如 sz002673 的 20260630 在 2026-07-26 还未发) 时,
东财 PageSDLTGD 接口返 sdltgd=[], akshare pd.DataFrame([])+reset_index()
得 1 列 df, 然后 columns=[12 列] 抛
ValueError('Length mismatch: Expected axis has 1 elements, new values have 12')
这是确定性无数据 (非瞬时故障), 但 call_ak_with_retry 当网络错重试 3 次
(14s 退避) + 噪声 ERROR 日志, 浪费时间且消耗断路器配额。
修复:
新增 _safe_top_10_em 包装 ak.stock_gdfx_free_top_10_em: 子串匹配
'Length mismatch' (pandas 错误信息稳定) → 返空 df 带 8 列 schema
(TOP_HOLDERS_COLUMNS), 不抛; 其他 ValueError/ConnectionError 透传重试。
fetch_top_holders_one_period 改调此包装。
验证:
- Mac: sz002673/20260630 从 ~14s (3 重试) 降到 0.04s, 返 0×8 空 df
- 有效期 sz002673/20251231, sh600519/20250930 正常返 10×8 数据
- 9/9 新单测通过 (tests/data_platform/test_top_holders_parse.py)
This commit is contained in:
@@ -463,13 +463,53 @@ def fetch_financial_abstract(symbol: str) -> pd.DataFrame:
|
||||
|
||||
# ======================== top_holders (per-stock × per-period) ========================
|
||||
|
||||
# akshare stock_gdfx_free_top_10_em 成功时返回的 8 列 schema
|
||||
# (源码 stock_gdfx_em.py 实测). 报告期未披露 (sdltgd=[]) 时 akshare 抛
|
||||
# ValueError("Length mismatch"), 这里捕获后用此 schema 返空 df,
|
||||
# 保证 parquet 列与有效期一致 (downstream reader 不会列数漂移).
|
||||
TOP_HOLDERS_COLUMNS = [
|
||||
"名次", "股东名称", "股东性质", "股份类型",
|
||||
"持股数", "占总流通股本持股比例", "增减", "变动比率",
|
||||
]
|
||||
|
||||
|
||||
def _safe_top_10_em(symbol: str, date: str) -> pd.DataFrame:
|
||||
"""akshare stock_gdfx_free_top_10_em 包装: 容忍空 sdltgd 响应。
|
||||
|
||||
bug 根因 (实证 akshare 1.18.x stock_gdfx_em.py):
|
||||
报告期未披露 (如当年 Q2 季报未发) 时东财 PageSDLTGD 接口返 sdltgd=[],
|
||||
akshare pd.DataFrame([]).reset_index() 得 1 列 df, 然后 columns=[12 列]
|
||||
抛 ValueError("Length mismatch: Expected axis has 1 elements, new values have 12").
|
||||
这是确定性无数据 (非瞬时故障), 但 call_ak_with_retry 会当网络错重试 3 次
|
||||
(14s 退避) + 噪声 ERROR 日志。
|
||||
|
||||
本包装预判该特定 ValueError (子串匹配 pandas 错误信息, 稳定):
|
||||
- Length mismatch → 返空 df (带 TOP_HOLDERS_COLUMNS schema), 不抛
|
||||
- 其他 ValueError / ConnectionError → 透传给 call_ak_with_retry 走重试
|
||||
"""
|
||||
try:
|
||||
return ak.stock_gdfx_free_top_10_em(symbol=symbol, date=date)
|
||||
except ValueError as e:
|
||||
if "Length mismatch" in str(e):
|
||||
logger.debug(
|
||||
"stock_gdfx_free_top_10_em(%s, %s) Length mismatch → sdltgd 空 (报告期未披露), 返空 df",
|
||||
symbol, date,
|
||||
)
|
||||
return pd.DataFrame(columns=TOP_HOLDERS_COLUMNS)
|
||||
raise
|
||||
|
||||
|
||||
def fetch_top_holders_one_period(
|
||||
symbol: str, period: str,
|
||||
) -> pd.DataFrame:
|
||||
"""stock_gdfx_free_top_10_em(symbol='sh600519', date='20250930') 单期。
|
||||
symbol 小写前缀, period YYYYMMDD."""
|
||||
symbol 小写前缀, period YYYYMMDD.
|
||||
|
||||
通过 _safe_top_10_em 包装: 报告期未披露 (sdltgd=[]) 时 akshare 抛
|
||||
Length mismatch ValueError, 这里捕获返空 df (避免 3 次无意义重试, 确定性
|
||||
无数据不应消耗断路器配额)。"""
|
||||
return _df_or_empty(call_ak_with_retry(
|
||||
ak.stock_gdfx_free_top_10_em,
|
||||
_safe_top_10_em,
|
||||
f"top_holders/{symbol}/{period}",
|
||||
symbol=symbol, date=period,
|
||||
))
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""top_holders 解析单元测试 (task: 修 akshare Length mismatch bug)。
|
||||
|
||||
bug 根因 (实证):
|
||||
akshare 1.18.x stock_gdfx_free_top_10_em 内部对东财 PageSDLTGD 接口的
|
||||
空响应 (sdltgd=[]) 处理不当:
|
||||
pd.DataFrame([]) → 0 列空 df
|
||||
.reset_index(inplace=True) → 1 列 ('index')
|
||||
temp_df.columns = [12 列] → ValueError: Length mismatch
|
||||
(Expected axis has 1 elements, new values have 12)
|
||||
|
||||
场景: 当报告期未披露 (如 sz002673 的 20260630 在 2026-07-26 还没发) 时,
|
||||
东财接口返 sdltgd=[], akshare 抛 ValueError。call_ak_with_retry 把 ValueError
|
||||
当瞬时故障重试 3 次 (14s 退避) 后返 None → 浪费时间 + 噪声日志。
|
||||
|
||||
修复策略:
|
||||
fetch_top_holders_one_period 改调 _safe_top_10_em 包装: 捕获 Length mismatch
|
||||
ValueError → 返空 df (带正确 schema 8 列), 不进重试; 其他异常透传。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "data_platform")
|
||||
_SCRIPT_DIR = os.path.abspath(_SCRIPT_DIR)
|
||||
if _SCRIPT_DIR not in sys.path:
|
||||
sys.path.insert(0, _SCRIPT_DIR)
|
||||
|
||||
import akshare_static_download as mod # noqa: E402
|
||||
|
||||
|
||||
# akshare 成功时返回的 8 列 (实证 stock_gdfx_free_top_10_em 源码 + Mac 实测)
|
||||
EXPECTED_COLS = [
|
||||
"名次", "股东名称", "股东性质", "股份类型",
|
||||
"持股数", "占总流通股本持股比例", "增减", "变动比率",
|
||||
]
|
||||
|
||||
|
||||
def _make_valid_df() -> pd.DataFrame:
|
||||
"""构造一个有效的 top_holders DataFrame (匹配 akshare 成功返回的 schema)。"""
|
||||
return pd.DataFrame({
|
||||
"名次": [1, 2],
|
||||
"股东名称": ["股东A", "股东B"],
|
||||
"股东性质": ["投资公司", "其它"],
|
||||
"股份类型": ["A股", "A股"],
|
||||
"持股数": [1000000, 500000],
|
||||
"占总流通股本持股比例": [10.5, 5.2],
|
||||
"增减": ["不变", "新增"],
|
||||
"变动比率": [0.0, 1.5],
|
||||
})
|
||||
|
||||
|
||||
class TestSafeTop10EmLengthMismatch:
|
||||
"""_safe_top_10_em 捕获 Length mismatch → 返空 df (不重试, 不抛)。"""
|
||||
|
||||
def test_length_mismatch_returns_empty_df_with_schema(self):
|
||||
"""akshare 抛 'Length mismatch' ValueError → 返空 df 带 8 列 schema。"""
|
||||
from akshare_static_download import _safe_top_10_em
|
||||
|
||||
err = ValueError(
|
||||
"Length mismatch: Expected axis has 1 elements, new values have 12 elements"
|
||||
)
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err):
|
||||
df = _safe_top_10_em(symbol="sz002673", date="20260630")
|
||||
|
||||
assert isinstance(df, pd.DataFrame)
|
||||
assert len(df) == 0
|
||||
assert df.columns.tolist() == EXPECTED_COLS
|
||||
|
||||
def test_length_mismatch_does_not_raise(self):
|
||||
"""Length mismatch 必须被吞掉 (call_ak_with_retry 才不会重试)。"""
|
||||
from akshare_static_download import _safe_top_10_em
|
||||
|
||||
err = ValueError("Length mismatch: Expected axis has 1 elements, new values have 12")
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err):
|
||||
# 不应抛
|
||||
df = _safe_top_10_em(symbol="sz002673", date="20260630")
|
||||
assert df.empty
|
||||
|
||||
def test_length_mismatch_message_check_is_substring_based(self):
|
||||
"""确认匹配逻辑是子串匹配 (pandas 错误信息稳定)。"""
|
||||
from akshare_static_download import _safe_top_10_em
|
||||
|
||||
# 不同 axis 数字 (akshare 升级可能改列数), 只要含 'Length mismatch' 就识别
|
||||
err = ValueError("Length mismatch: Expected axis has 5 elements, new values have 20")
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err):
|
||||
df = _safe_top_10_em(symbol="sh600519", date="20260630")
|
||||
assert df.empty
|
||||
|
||||
|
||||
class TestSafeTop10EmPassthrough:
|
||||
"""_safe_top_10_em 对成功/其他异常的透传行为。"""
|
||||
|
||||
def test_success_passes_through(self):
|
||||
"""akshare 正常返 df → 透传不变 (symbol/date 参数正确传)。"""
|
||||
from akshare_static_download import _safe_top_10_em
|
||||
|
||||
valid = _make_valid_df()
|
||||
captured = {}
|
||||
|
||||
def fake(symbol, date):
|
||||
captured["symbol"] = symbol
|
||||
captured["date"] = date
|
||||
return valid
|
||||
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=fake):
|
||||
df = _safe_top_10_em(symbol="sh600519", date="20250930")
|
||||
|
||||
# 参数正确传递
|
||||
assert captured["symbol"] == "sh600519"
|
||||
assert captured["date"] == "20250930"
|
||||
# 返回值原样透传 (同一对象)
|
||||
assert df is valid
|
||||
assert df.columns.tolist() == EXPECTED_COLS
|
||||
|
||||
def test_other_value_error_reraises(self):
|
||||
"""非 Length mismatch 的 ValueError 不应被吞 (让重试机制处理)。"""
|
||||
from akshare_static_download import _safe_top_10_em
|
||||
|
||||
other_err = ValueError("Some other pandas error")
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=other_err):
|
||||
with pytest.raises(ValueError, match="Some other pandas error"):
|
||||
_safe_top_10_em(symbol="sh600519", date="20250930")
|
||||
|
||||
def test_connection_error_reraises(self):
|
||||
"""网络异常 ConnectionError 不应被吞 (让重试机制处理)。"""
|
||||
from akshare_static_download import _safe_top_10_em
|
||||
|
||||
import requests
|
||||
net_err = requests.ConnectionError("network down")
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=net_err):
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
_safe_top_10_em(symbol="sh600519", date="20250930")
|
||||
|
||||
|
||||
class TestFetchTopHoldersOnePeriod:
|
||||
"""fetch_top_holders_one_period 集成 _safe_top_10_em 的端到端行为。"""
|
||||
|
||||
def test_undisclosed_period_returns_empty_df_no_retry(self, tmp_path, monkeypatch):
|
||||
"""报告期未披露 (Length mismatch) → 直接返空 df, call_ak_with_retry 不重试。
|
||||
|
||||
验证: side_effect 只被调一次 (无重试), 返空 df 带 schema。
|
||||
"""
|
||||
# 强制 call_ak_with_retry 的重试退避为 0, 避免测试慢
|
||||
monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0])
|
||||
|
||||
err = ValueError("Length mismatch: Expected axis has 1 elements, new values have 12")
|
||||
call_count = {"n": 0}
|
||||
|
||||
def side_effect(*args, **kwargs):
|
||||
call_count["n"] += 1
|
||||
raise err
|
||||
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=side_effect):
|
||||
df = mod.fetch_top_holders_one_period(symbol="sz002673", period="20260630")
|
||||
|
||||
# 关键断言: 只调了 1 次 (Length mismatch 不重试, 不像过去 3 次)
|
||||
assert call_count["n"] == 1, (
|
||||
f"Length mismatch 应该被 _safe_top_10_em 一次吞掉, 但调了 {call_count['n']} 次"
|
||||
)
|
||||
assert isinstance(df, pd.DataFrame)
|
||||
assert df.empty
|
||||
assert df.columns.tolist() == EXPECTED_COLS
|
||||
|
||||
def test_valid_period_returns_data(self, monkeypatch):
|
||||
"""有效报告期 → 正常返回数据 df。"""
|
||||
monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0])
|
||||
|
||||
valid = _make_valid_df()
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", return_value=valid):
|
||||
df = mod.fetch_top_holders_one_period(symbol="sh600519", period="20250930")
|
||||
|
||||
assert len(df) == 2
|
||||
assert df.columns.tolist() == EXPECTED_COLS
|
||||
|
||||
def test_network_error_still_retries(self, monkeypatch):
|
||||
"""网络异常仍然走重试 (验证 _safe_top_10_em 只过滤 Length mismatch, 不误伤)。"""
|
||||
monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0])
|
||||
|
||||
import requests
|
||||
call_count = {"n": 0}
|
||||
|
||||
def side_effect(*args, **kwargs):
|
||||
call_count["n"] += 1
|
||||
raise requests.ConnectionError("network down")
|
||||
|
||||
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=side_effect):
|
||||
df = mod.fetch_top_holders_one_period(symbol="sh600519", period="20250930")
|
||||
|
||||
# 网络异常应该重试 AK_MAX_RETRIES 次 (不是 1 次)
|
||||
assert call_count["n"] == mod.AK_MAX_RETRIES, (
|
||||
f"网络异常应重试 {mod.AK_MAX_RETRIES} 次, 实际 {call_count['n']} 次"
|
||||
)
|
||||
# 重试耗尽返空 df (无 schema, 因为确实是失败)
|
||||
assert isinstance(df, pd.DataFrame)
|
||||
Reference in New Issue
Block a user