fix(data): akshare top_holders Length mismatch bug 根治

bug 根因 (实证 akshare 1.18.x stock_gdfx_em.py:418):
  报告期未披露 (如 sz002673 的 20260630 在 2026-07-26 还未发) 时,
  东财 PageSDLTGD 接口返 sdltgd=[], akshare pd.DataFrame([])+reset_index()
  得 1 列 df, 然后 columns=[12 列] 抛
  ValueError('Length mismatch: Expected axis has 1 elements, new values have 12')

  这是确定性无数据 (非瞬时故障), 但 call_ak_with_retry 当网络错重试 3 次
  (14s 退避) + 噪声 ERROR 日志, 浪费时间且消耗断路器配额。

修复:
  新增 _safe_top_10_em 包装 ak.stock_gdfx_free_top_10_em: 子串匹配
  'Length mismatch' (pandas 错误信息稳定) → 返空 df 带 8 列 schema
  (TOP_HOLDERS_COLUMNS), 不抛; 其他 ValueError/ConnectionError 透传重试。
  fetch_top_holders_one_period 改调此包装。

验证:
  - Mac: sz002673/20260630 从 ~14s (3 重试) 降到 0.04s, 返 0×8 空 df
  - 有效期 sz002673/20251231, sh600519/20250930 正常返 10×8 数据
  - 9/9 新单测通过 (tests/data_platform/test_top_holders_parse.py)
This commit is contained in:
2026-07-26 11:54:19 +08:00
parent b06af336c3
commit 11f383a8e2
2 changed files with 240 additions and 2 deletions
@@ -463,13 +463,53 @@ def fetch_financial_abstract(symbol: str) -> pd.DataFrame:
# ======================== top_holders (per-stock × per-period) ========================
# akshare stock_gdfx_free_top_10_em 成功时返回的 8 列 schema
# (源码 stock_gdfx_em.py 实测). 报告期未披露 (sdltgd=[]) 时 akshare 抛
# ValueError("Length mismatch"), 这里捕获后用此 schema 返空 df,
# 保证 parquet 列与有效期一致 (downstream reader 不会列数漂移).
TOP_HOLDERS_COLUMNS = [
"名次", "股东名称", "股东性质", "股份类型",
"持股数", "占总流通股本持股比例", "增减", "变动比率",
]
def _safe_top_10_em(symbol: str, date: str) -> pd.DataFrame:
"""akshare stock_gdfx_free_top_10_em 包装: 容忍空 sdltgd 响应。
bug 根因 (实证 akshare 1.18.x stock_gdfx_em.py):
报告期未披露 (如当年 Q2 季报未发) 时东财 PageSDLTGD 接口返 sdltgd=[],
akshare pd.DataFrame([]).reset_index() 得 1 列 df, 然后 columns=[12 列]
抛 ValueError("Length mismatch: Expected axis has 1 elements, new values have 12").
这是确定性无数据 (非瞬时故障), 但 call_ak_with_retry 会当网络错重试 3 次
(14s 退避) + 噪声 ERROR 日志。
本包装预判该特定 ValueError (子串匹配 pandas 错误信息, 稳定):
- Length mismatch → 返空 df (带 TOP_HOLDERS_COLUMNS schema), 不抛
- 其他 ValueError / ConnectionError → 透传给 call_ak_with_retry 走重试
"""
try:
return ak.stock_gdfx_free_top_10_em(symbol=symbol, date=date)
except ValueError as e:
if "Length mismatch" in str(e):
logger.debug(
"stock_gdfx_free_top_10_em(%s, %s) Length mismatch → sdltgd 空 (报告期未披露), 返空 df",
symbol, date,
)
return pd.DataFrame(columns=TOP_HOLDERS_COLUMNS)
raise
def fetch_top_holders_one_period(
symbol: str, period: str,
) -> pd.DataFrame:
"""stock_gdfx_free_top_10_em(symbol='sh600519', date='20250930') 单期。
symbol 小写前缀, period YYYYMMDD."""
symbol 小写前缀, period YYYYMMDD.
通过 _safe_top_10_em 包装: 报告期未披露 (sdltgd=[]) 时 akshare 抛
Length mismatch ValueError, 这里捕获返空 df (避免 3 次无意义重试, 确定性
无数据不应消耗断路器配额)。"""
return _df_or_empty(call_ak_with_retry(
ak.stock_gdfx_free_top_10_em,
_safe_top_10_em,
f"top_holders/{symbol}/{period}",
symbol=symbol, date=period,
))
@@ -0,0 +1,198 @@
# -*- coding: utf-8 -*-
"""top_holders 解析单元测试 (task: 修 akshare Length mismatch bug)。
bug 根因 (实证):
akshare 1.18.x stock_gdfx_free_top_10_em 内部对东财 PageSDLTGD 接口的
空响应 (sdltgd=[]) 处理不当:
pd.DataFrame([]) → 0 列空 df
.reset_index(inplace=True) → 1 列 ('index')
temp_df.columns = [12 列] → ValueError: Length mismatch
(Expected axis has 1 elements, new values have 12)
场景: 当报告期未披露 (如 sz002673 的 20260630 在 2026-07-26 还没发) 时,
东财接口返 sdltgd=[], akshare 抛 ValueError。call_ak_with_retry 把 ValueError
当瞬时故障重试 3 次 (14s 退避) 后返 None → 浪费时间 + 噪声日志。
修复策略:
fetch_top_holders_one_period 改调 _safe_top_10_em 包装: 捕获 Length mismatch
ValueError → 返空 df (带正确 schema 8 列), 不进重试; 其他异常透传。
"""
import os
import sys
from unittest.mock import patch
import pandas as pd
import pytest
_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "data_platform")
_SCRIPT_DIR = os.path.abspath(_SCRIPT_DIR)
if _SCRIPT_DIR not in sys.path:
sys.path.insert(0, _SCRIPT_DIR)
import akshare_static_download as mod # noqa: E402
# akshare 成功时返回的 8 列 (实证 stock_gdfx_free_top_10_em 源码 + Mac 实测)
EXPECTED_COLS = [
"名次", "股东名称", "股东性质", "股份类型",
"持股数", "占总流通股本持股比例", "增减", "变动比率",
]
def _make_valid_df() -> pd.DataFrame:
"""构造一个有效的 top_holders DataFrame (匹配 akshare 成功返回的 schema)。"""
return pd.DataFrame({
"名次": [1, 2],
"股东名称": ["股东A", "股东B"],
"股东性质": ["投资公司", "其它"],
"股份类型": ["A股", "A股"],
"持股数": [1000000, 500000],
"占总流通股本持股比例": [10.5, 5.2],
"增减": ["不变", "新增"],
"变动比率": [0.0, 1.5],
})
class TestSafeTop10EmLengthMismatch:
"""_safe_top_10_em 捕获 Length mismatch → 返空 df (不重试, 不抛)。"""
def test_length_mismatch_returns_empty_df_with_schema(self):
"""akshare 抛 'Length mismatch' ValueError → 返空 df 带 8 列 schema。"""
from akshare_static_download import _safe_top_10_em
err = ValueError(
"Length mismatch: Expected axis has 1 elements, new values have 12 elements"
)
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err):
df = _safe_top_10_em(symbol="sz002673", date="20260630")
assert isinstance(df, pd.DataFrame)
assert len(df) == 0
assert df.columns.tolist() == EXPECTED_COLS
def test_length_mismatch_does_not_raise(self):
"""Length mismatch 必须被吞掉 (call_ak_with_retry 才不会重试)。"""
from akshare_static_download import _safe_top_10_em
err = ValueError("Length mismatch: Expected axis has 1 elements, new values have 12")
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err):
# 不应抛
df = _safe_top_10_em(symbol="sz002673", date="20260630")
assert df.empty
def test_length_mismatch_message_check_is_substring_based(self):
"""确认匹配逻辑是子串匹配 (pandas 错误信息稳定)。"""
from akshare_static_download import _safe_top_10_em
# 不同 axis 数字 (akshare 升级可能改列数), 只要含 'Length mismatch' 就识别
err = ValueError("Length mismatch: Expected axis has 5 elements, new values have 20")
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err):
df = _safe_top_10_em(symbol="sh600519", date="20260630")
assert df.empty
class TestSafeTop10EmPassthrough:
"""_safe_top_10_em 对成功/其他异常的透传行为。"""
def test_success_passes_through(self):
"""akshare 正常返 df → 透传不变 (symbol/date 参数正确传)。"""
from akshare_static_download import _safe_top_10_em
valid = _make_valid_df()
captured = {}
def fake(symbol, date):
captured["symbol"] = symbol
captured["date"] = date
return valid
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=fake):
df = _safe_top_10_em(symbol="sh600519", date="20250930")
# 参数正确传递
assert captured["symbol"] == "sh600519"
assert captured["date"] == "20250930"
# 返回值原样透传 (同一对象)
assert df is valid
assert df.columns.tolist() == EXPECTED_COLS
def test_other_value_error_reraises(self):
"""非 Length mismatch 的 ValueError 不应被吞 (让重试机制处理)。"""
from akshare_static_download import _safe_top_10_em
other_err = ValueError("Some other pandas error")
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=other_err):
with pytest.raises(ValueError, match="Some other pandas error"):
_safe_top_10_em(symbol="sh600519", date="20250930")
def test_connection_error_reraises(self):
"""网络异常 ConnectionError 不应被吞 (让重试机制处理)。"""
from akshare_static_download import _safe_top_10_em
import requests
net_err = requests.ConnectionError("network down")
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=net_err):
with pytest.raises(requests.ConnectionError):
_safe_top_10_em(symbol="sh600519", date="20250930")
class TestFetchTopHoldersOnePeriod:
"""fetch_top_holders_one_period 集成 _safe_top_10_em 的端到端行为。"""
def test_undisclosed_period_returns_empty_df_no_retry(self, tmp_path, monkeypatch):
"""报告期未披露 (Length mismatch) → 直接返空 df, call_ak_with_retry 不重试。
验证: side_effect 只被调一次 (无重试), 返空 df 带 schema。
"""
# 强制 call_ak_with_retry 的重试退避为 0, 避免测试慢
monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0])
err = ValueError("Length mismatch: Expected axis has 1 elements, new values have 12")
call_count = {"n": 0}
def side_effect(*args, **kwargs):
call_count["n"] += 1
raise err
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=side_effect):
df = mod.fetch_top_holders_one_period(symbol="sz002673", period="20260630")
# 关键断言: 只调了 1 次 (Length mismatch 不重试, 不像过去 3 次)
assert call_count["n"] == 1, (
f"Length mismatch 应该被 _safe_top_10_em 一次吞掉, 但调了 {call_count['n']}"
)
assert isinstance(df, pd.DataFrame)
assert df.empty
assert df.columns.tolist() == EXPECTED_COLS
def test_valid_period_returns_data(self, monkeypatch):
"""有效报告期 → 正常返回数据 df。"""
monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0])
valid = _make_valid_df()
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", return_value=valid):
df = mod.fetch_top_holders_one_period(symbol="sh600519", period="20250930")
assert len(df) == 2
assert df.columns.tolist() == EXPECTED_COLS
def test_network_error_still_retries(self, monkeypatch):
"""网络异常仍然走重试 (验证 _safe_top_10_em 只过滤 Length mismatch, 不误伤)。"""
monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0])
import requests
call_count = {"n": 0}
def side_effect(*args, **kwargs):
call_count["n"] += 1
raise requests.ConnectionError("network down")
with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=side_effect):
df = mod.fetch_top_holders_one_period(symbol="sh600519", period="20250930")
# 网络异常应该重试 AK_MAX_RETRIES 次 (不是 1 次)
assert call_count["n"] == mod.AK_MAX_RETRIES, (
f"网络异常应重试 {mod.AK_MAX_RETRIES} 次, 实际 {call_count['n']}"
)
# 重试耗尽返空 df (无 schema, 因为确实是失败)
assert isinstance(df, pd.DataFrame)