From 11f383a8e244f3e26e3bd2bfe6a0c72f9ac7f127 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Sun, 26 Jul 2026 11:54:19 +0800 Subject: [PATCH] =?UTF-8?q?fix(data):=20akshare=20top=5Fholders=20Length?= =?UTF-8?q?=20mismatch=20bug=20=E6=A0=B9=E6=B2=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bug 根因 (实证 akshare 1.18.x stock_gdfx_em.py:418): 报告期未披露 (如 sz002673 的 20260630 在 2026-07-26 还未发) 时, 东财 PageSDLTGD 接口返 sdltgd=[], akshare pd.DataFrame([])+reset_index() 得 1 列 df, 然后 columns=[12 列] 抛 ValueError('Length mismatch: Expected axis has 1 elements, new values have 12') 这是确定性无数据 (非瞬时故障), 但 call_ak_with_retry 当网络错重试 3 次 (14s 退避) + 噪声 ERROR 日志, 浪费时间且消耗断路器配额。 修复: 新增 _safe_top_10_em 包装 ak.stock_gdfx_free_top_10_em: 子串匹配 'Length mismatch' (pandas 错误信息稳定) → 返空 df 带 8 列 schema (TOP_HOLDERS_COLUMNS), 不抛; 其他 ValueError/ConnectionError 透传重试。 fetch_top_holders_one_period 改调此包装。 验证: - Mac: sz002673/20260630 从 ~14s (3 重试) 降到 0.04s, 返 0×8 空 df - 有效期 sz002673/20251231, sh600519/20250930 正常返 10×8 数据 - 9/9 新单测通过 (tests/data_platform/test_top_holders_parse.py) --- .../data_platform/akshare_static_download.py | 44 +++- tests/data_platform/test_top_holders_parse.py | 198 ++++++++++++++++++ 2 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 tests/data_platform/test_top_holders_parse.py diff --git a/scripts/data_platform/akshare_static_download.py b/scripts/data_platform/akshare_static_download.py index 0c512af..e3d901c 100644 --- a/scripts/data_platform/akshare_static_download.py +++ b/scripts/data_platform/akshare_static_download.py @@ -463,13 +463,53 @@ def fetch_financial_abstract(symbol: str) -> pd.DataFrame: # ======================== top_holders (per-stock × per-period) ======================== +# akshare stock_gdfx_free_top_10_em 成功时返回的 8 列 schema +# (源码 stock_gdfx_em.py 实测). 报告期未披露 (sdltgd=[]) 时 akshare 抛 +# ValueError("Length mismatch"), 这里捕获后用此 schema 返空 df, +# 保证 parquet 列与有效期一致 (downstream reader 不会列数漂移). +TOP_HOLDERS_COLUMNS = [ + "名次", "股东名称", "股东性质", "股份类型", + "持股数", "占总流通股本持股比例", "增减", "变动比率", +] + + +def _safe_top_10_em(symbol: str, date: str) -> pd.DataFrame: + """akshare stock_gdfx_free_top_10_em 包装: 容忍空 sdltgd 响应。 + + bug 根因 (实证 akshare 1.18.x stock_gdfx_em.py): + 报告期未披露 (如当年 Q2 季报未发) 时东财 PageSDLTGD 接口返 sdltgd=[], + akshare pd.DataFrame([]).reset_index() 得 1 列 df, 然后 columns=[12 列] + 抛 ValueError("Length mismatch: Expected axis has 1 elements, new values have 12"). + 这是确定性无数据 (非瞬时故障), 但 call_ak_with_retry 会当网络错重试 3 次 + (14s 退避) + 噪声 ERROR 日志。 + + 本包装预判该特定 ValueError (子串匹配 pandas 错误信息, 稳定): + - Length mismatch → 返空 df (带 TOP_HOLDERS_COLUMNS schema), 不抛 + - 其他 ValueError / ConnectionError → 透传给 call_ak_with_retry 走重试 + """ + try: + return ak.stock_gdfx_free_top_10_em(symbol=symbol, date=date) + except ValueError as e: + if "Length mismatch" in str(e): + logger.debug( + "stock_gdfx_free_top_10_em(%s, %s) Length mismatch → sdltgd 空 (报告期未披露), 返空 df", + symbol, date, + ) + return pd.DataFrame(columns=TOP_HOLDERS_COLUMNS) + raise + + def fetch_top_holders_one_period( symbol: str, period: str, ) -> pd.DataFrame: """stock_gdfx_free_top_10_em(symbol='sh600519', date='20250930') 单期。 - symbol 小写前缀, period YYYYMMDD.""" + symbol 小写前缀, period YYYYMMDD. + + 通过 _safe_top_10_em 包装: 报告期未披露 (sdltgd=[]) 时 akshare 抛 + Length mismatch ValueError, 这里捕获返空 df (避免 3 次无意义重试, 确定性 + 无数据不应消耗断路器配额)。""" return _df_or_empty(call_ak_with_retry( - ak.stock_gdfx_free_top_10_em, + _safe_top_10_em, f"top_holders/{symbol}/{period}", symbol=symbol, date=period, )) diff --git a/tests/data_platform/test_top_holders_parse.py b/tests/data_platform/test_top_holders_parse.py new file mode 100644 index 0000000..e97f124 --- /dev/null +++ b/tests/data_platform/test_top_holders_parse.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +"""top_holders 解析单元测试 (task: 修 akshare Length mismatch bug)。 + +bug 根因 (实证): + akshare 1.18.x stock_gdfx_free_top_10_em 内部对东财 PageSDLTGD 接口的 + 空响应 (sdltgd=[]) 处理不当: + pd.DataFrame([]) → 0 列空 df + .reset_index(inplace=True) → 1 列 ('index') + temp_df.columns = [12 列] → ValueError: Length mismatch + (Expected axis has 1 elements, new values have 12) + + 场景: 当报告期未披露 (如 sz002673 的 20260630 在 2026-07-26 还没发) 时, + 东财接口返 sdltgd=[], akshare 抛 ValueError。call_ak_with_retry 把 ValueError + 当瞬时故障重试 3 次 (14s 退避) 后返 None → 浪费时间 + 噪声日志。 + +修复策略: + fetch_top_holders_one_period 改调 _safe_top_10_em 包装: 捕获 Length mismatch + ValueError → 返空 df (带正确 schema 8 列), 不进重试; 其他异常透传。 +""" +import os +import sys +from unittest.mock import patch + +import pandas as pd +import pytest + +_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "data_platform") +_SCRIPT_DIR = os.path.abspath(_SCRIPT_DIR) +if _SCRIPT_DIR not in sys.path: + sys.path.insert(0, _SCRIPT_DIR) + +import akshare_static_download as mod # noqa: E402 + + +# akshare 成功时返回的 8 列 (实证 stock_gdfx_free_top_10_em 源码 + Mac 实测) +EXPECTED_COLS = [ + "名次", "股东名称", "股东性质", "股份类型", + "持股数", "占总流通股本持股比例", "增减", "变动比率", +] + + +def _make_valid_df() -> pd.DataFrame: + """构造一个有效的 top_holders DataFrame (匹配 akshare 成功返回的 schema)。""" + return pd.DataFrame({ + "名次": [1, 2], + "股东名称": ["股东A", "股东B"], + "股东性质": ["投资公司", "其它"], + "股份类型": ["A股", "A股"], + "持股数": [1000000, 500000], + "占总流通股本持股比例": [10.5, 5.2], + "增减": ["不变", "新增"], + "变动比率": [0.0, 1.5], + }) + + +class TestSafeTop10EmLengthMismatch: + """_safe_top_10_em 捕获 Length mismatch → 返空 df (不重试, 不抛)。""" + + def test_length_mismatch_returns_empty_df_with_schema(self): + """akshare 抛 'Length mismatch' ValueError → 返空 df 带 8 列 schema。""" + from akshare_static_download import _safe_top_10_em + + err = ValueError( + "Length mismatch: Expected axis has 1 elements, new values have 12 elements" + ) + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err): + df = _safe_top_10_em(symbol="sz002673", date="20260630") + + assert isinstance(df, pd.DataFrame) + assert len(df) == 0 + assert df.columns.tolist() == EXPECTED_COLS + + def test_length_mismatch_does_not_raise(self): + """Length mismatch 必须被吞掉 (call_ak_with_retry 才不会重试)。""" + from akshare_static_download import _safe_top_10_em + + err = ValueError("Length mismatch: Expected axis has 1 elements, new values have 12") + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err): + # 不应抛 + df = _safe_top_10_em(symbol="sz002673", date="20260630") + assert df.empty + + def test_length_mismatch_message_check_is_substring_based(self): + """确认匹配逻辑是子串匹配 (pandas 错误信息稳定)。""" + from akshare_static_download import _safe_top_10_em + + # 不同 axis 数字 (akshare 升级可能改列数), 只要含 'Length mismatch' 就识别 + err = ValueError("Length mismatch: Expected axis has 5 elements, new values have 20") + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=err): + df = _safe_top_10_em(symbol="sh600519", date="20260630") + assert df.empty + + +class TestSafeTop10EmPassthrough: + """_safe_top_10_em 对成功/其他异常的透传行为。""" + + def test_success_passes_through(self): + """akshare 正常返 df → 透传不变 (symbol/date 参数正确传)。""" + from akshare_static_download import _safe_top_10_em + + valid = _make_valid_df() + captured = {} + + def fake(symbol, date): + captured["symbol"] = symbol + captured["date"] = date + return valid + + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=fake): + df = _safe_top_10_em(symbol="sh600519", date="20250930") + + # 参数正确传递 + assert captured["symbol"] == "sh600519" + assert captured["date"] == "20250930" + # 返回值原样透传 (同一对象) + assert df is valid + assert df.columns.tolist() == EXPECTED_COLS + + def test_other_value_error_reraises(self): + """非 Length mismatch 的 ValueError 不应被吞 (让重试机制处理)。""" + from akshare_static_download import _safe_top_10_em + + other_err = ValueError("Some other pandas error") + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=other_err): + with pytest.raises(ValueError, match="Some other pandas error"): + _safe_top_10_em(symbol="sh600519", date="20250930") + + def test_connection_error_reraises(self): + """网络异常 ConnectionError 不应被吞 (让重试机制处理)。""" + from akshare_static_download import _safe_top_10_em + + import requests + net_err = requests.ConnectionError("network down") + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=net_err): + with pytest.raises(requests.ConnectionError): + _safe_top_10_em(symbol="sh600519", date="20250930") + + +class TestFetchTopHoldersOnePeriod: + """fetch_top_holders_one_period 集成 _safe_top_10_em 的端到端行为。""" + + def test_undisclosed_period_returns_empty_df_no_retry(self, tmp_path, monkeypatch): + """报告期未披露 (Length mismatch) → 直接返空 df, call_ak_with_retry 不重试。 + + 验证: side_effect 只被调一次 (无重试), 返空 df 带 schema。 + """ + # 强制 call_ak_with_retry 的重试退避为 0, 避免测试慢 + monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0]) + + err = ValueError("Length mismatch: Expected axis has 1 elements, new values have 12") + call_count = {"n": 0} + + def side_effect(*args, **kwargs): + call_count["n"] += 1 + raise err + + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=side_effect): + df = mod.fetch_top_holders_one_period(symbol="sz002673", period="20260630") + + # 关键断言: 只调了 1 次 (Length mismatch 不重试, 不像过去 3 次) + assert call_count["n"] == 1, ( + f"Length mismatch 应该被 _safe_top_10_em 一次吞掉, 但调了 {call_count['n']} 次" + ) + assert isinstance(df, pd.DataFrame) + assert df.empty + assert df.columns.tolist() == EXPECTED_COLS + + def test_valid_period_returns_data(self, monkeypatch): + """有效报告期 → 正常返回数据 df。""" + monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0]) + + valid = _make_valid_df() + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", return_value=valid): + df = mod.fetch_top_holders_one_period(symbol="sh600519", period="20250930") + + assert len(df) == 2 + assert df.columns.tolist() == EXPECTED_COLS + + def test_network_error_still_retries(self, monkeypatch): + """网络异常仍然走重试 (验证 _safe_top_10_em 只过滤 Length mismatch, 不误伤)。""" + monkeypatch.setattr(mod, "RETRY_BACKOFF", [0, 0, 0]) + + import requests + call_count = {"n": 0} + + def side_effect(*args, **kwargs): + call_count["n"] += 1 + raise requests.ConnectionError("network down") + + with patch.object(mod.ak, "stock_gdfx_free_top_10_em", side_effect=side_effect): + df = mod.fetch_top_holders_one_period(symbol="sh600519", period="20250930") + + # 网络异常应该重试 AK_MAX_RETRIES 次 (不是 1 次) + assert call_count["n"] == mod.AK_MAX_RETRIES, ( + f"网络异常应重试 {mod.AK_MAX_RETRIES} 次, 实际 {call_count['n']} 次" + ) + # 重试耗尽返空 df (无 schema, 因为确实是失败) + assert isinstance(df, pd.DataFrame)