From 694a0d2238aadfa29d311b217bbf89cb35b574b1 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Sun, 6 Sep 2026 08:29:58 +0800 Subject: [PATCH] =?UTF-8?q?fix(data):=20ak-weekly=20=E6=96=AD=E8=B7=AF?= =?UTF-8?q?=E5=99=A8=E9=A5=BF=E6=AD=BB=E4=BF=AE=E5=A4=8D=3D=5Fsafe=5Fnorth?= =?UTF-8?q?bound=5Fem+=E7=B1=BB=E5=9E=8B=E8=B0=83=E5=BA=8F=20[vps]?= =?UTF-8?q?=E2=80=94=E2=80=9409-05=2003:00=20=E9=A6=96=E8=B7=91=20rc=3D2?= =?UTF-8?q?=20=E6=A0=B9=E5=9B=A0:akshare=20stock=5Fhsgt=5Findividual=5Fem?= =?UTF-8?q?=20=E5=AF=B9=E6=97=A0=E5=8C=97=E5=90=91=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=A0=87=E7=9A=84(~1117/2883,301xxx=20=E7=AD=89)=E5=9C=A8=20da?= =?UTF-8?q?ta=5Fjson[result][pages]=20=E6=8A=9B=20TypeError=20NoneType=20n?= =?UTF-8?q?ot=20subscriptable=E2=86=92failed=E2=86=9230=20=E8=BF=9E?= =?UTF-8?q?=E8=B4=A5=E6=96=AD=E8=B7=AF=E5=99=A8=E2=86=92=E8=B7=B3=E8=BF=87?= =?UTF-8?q?=E5=90=8E=E7=BB=AD=E7=B1=BB=E5=9E=8B,share=5Fcapital/top=5Fhold?= =?UTF-8?q?ers/gdhs=20=E5=85=A8=E9=A5=BF=E6=AD=BB(gdhs=20=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E8=87=B3=E4=BB=8A=E4=B8=8D=E5=AD=98=E5=9C=A8);?= =?UTF-8?q?=E7=AB=AF=E7=82=B9=E5=81=A5=E5=BA=B7(600519=20=E8=BF=94=201683?= =?UTF-8?q?=20=E8=A1=8C,09-06=20=E6=99=A8=E5=A4=8D=E6=B5=8B);=E4=BF=AE=3D?= =?UTF-8?q?=E2=91=A0=5Fsafe=5Fnorthbound=5Fem=20=E5=AD=90=E4=B8=B2?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=20not=20subscriptable=20=E8=BF=94=E7=A9=BA?= =?UTF-8?q?=20df=20=E5=B8=A6=209=20=E5=88=97=20schema=E2=86=92=E8=B5=B0?= =?UTF-8?q?=E6=97=A0=E5=8C=97=E5=90=91=E6=8C=81=E4=BB=93=20empty=20?= =?UTF-8?q?=E5=86=99=E7=A9=BA+done=20=E6=97=A2=E5=AE=9A=E8=AF=AD=E4=B9=89?= =?UTF-8?q?=E4=B8=8D=E8=80=97=E9=87=8D=E8=AF=95/=E6=96=AD=E8=B7=AF?= =?UTF-8?q?=E5=99=A8(=E5=90=8C=20=5Fsafe=5Ftop=5F10=5Fem=20=E6=89=8B?= =?UTF-8?q?=E6=B3=95)=E2=91=A1weekly=20wrapper=20=E8=B0=83=E5=BA=8F=20gdhs?= =?UTF-8?q?,share=5Fcapital,top=5Fholders,northbound=20=E6=9C=80=E5=A4=A7?= =?UTF-8?q?=E6=9C=80=E9=99=A9=E8=80=85=E6=AE=BF=E5=90=8E=E9=98=B2=E5=86=8D?= =?UTF-8?q?=E8=BF=9E=E7=B4=AF=E2=91=A2=E9=A1=BA=E6=89=8B=20BOM;TDD=202=20?= =?UTF-8?q?=E6=96=B0=E6=B5=8B=E5=85=88=E7=BA=A2=E5=90=8E=E7=BB=BF+217=20?= =?UTF-8?q?=E5=85=A8=E7=BB=BF;=E5=8F=8C=E8=84=9A=E6=9C=AC=E5=B7=B2=20scp?= =?UTF-8?q?=20=E7=94=9F=E6=95=88=E4=BD=8D+Parse/py=5Fcompile=20=E9=A2=84?= =?UTF-8?q?=E6=A3=80=E8=BF=87;gdhs=20=E6=89=8B=E5=8A=A8=E8=A1=A5=E8=B7=91?= =?UTF-8?q?=E5=8F=A6=E8=A1=8C=20[vps]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/data_platform/ak_stock_wrapper.ps1 | 8 ++-- .../data_platform/akshare_static_download.py | 38 ++++++++++++++++++- .../test_akshare_static_download.py | 32 ++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/scripts/data_platform/ak_stock_wrapper.ps1 b/scripts/data_platform/ak_stock_wrapper.ps1 index 4885ba1..a750053 100644 --- a/scripts/data_platform/ak_stock_wrapper.ps1 +++ b/scripts/data_platform/ak_stock_wrapper.ps1 @@ -1,5 +1,7 @@ -# ak_stock_wrapper.ps1 — sanguo-ak-weekly schtask wrapper (weekly 周六03:00 akshare per-stock 慢) -# northbound+share_capital+top_holders+gdhs --force; per-stock 全量慢, 周末夜间; +# ak_stock_wrapper.ps1 — sanguo-ak-weekly schtask wrapper (weekly 周六03:00 akshare per-stock 慢) +# --force 全量刷新; 顺序=gdhs,share_capital,top_holders,northbound (09-06 调序: +# 09-05 northbound 1117/2883 无数据标的连败触发断路器, 饿死后续 3 类型—— +# 最大最险的 northbound 挪最后, 即使再断路也不连累前面); # gdhs=股东户数 per-period 全市场单次 (09-02 入列, 披露守卫>90天季度末, marker 去重无新期零网络) $env:http_proxy = '' $env:https_proxy = '' @@ -9,5 +11,5 @@ $ts = Get-Date -Format 'yyyyMMdd_HHmmss' $logDir = 'C:\sanguo_vnpy_v2\data\migration_logs' if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null } $log = Join-Path $logDir "ak_stock_$ts.txt" -C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\akshare_static_download.py --types northbound,share_capital,top_holders,gdhs --force *>> $log +C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\akshare_static_download.py --types gdhs,share_capital,top_holders,northbound --force *>> $log exit $LASTEXITCODE diff --git a/scripts/data_platform/akshare_static_download.py b/scripts/data_platform/akshare_static_download.py index 2fbbfe4..4891eeb 100644 --- a/scripts/data_platform/akshare_static_download.py +++ b/scripts/data_platform/akshare_static_download.py @@ -496,10 +496,44 @@ def fetch_valuation(symbol: str) -> Optional[pd.DataFrame]: return df +NORTHBOUND_COLUMNS = [ + "持股日期", "当日收盘价", "当日涨跌幅", "持股数量", "持股市值", + "持股数量占A股百分比", "今日增持股数", "今日增持资金", "今日持股市值变化", +] + + +def _safe_northbound_em(symbol: str) -> pd.DataFrame: + """akshare stock_hsgt_individual_em 包装: 容忍无北向数据标的的崩溃响应。 + + bug 根因 (实证 akshare 1.18.x stock_hsgt_em.py:1446, 2026-09-05 ak-weekly + 断路器事故): 无北向数据标的 (301xxx 等 ~1117/2883) datacenter 返 + result=None, __stock_hsgt_zh_a_em 的 data_json["result"]["pages"] 抛 + TypeError("'NoneType' object is not subscriptable") → failed → 30 连败 + 断路器 rc=2, 饿死后续 share_capital/top_holders/gdhs。端点健康 + (600519 正常返 1683 行)。 + + 子串匹配 "not subscriptable" (稳定签名, 同 _safe_top_10_em 手法): + → 返空 df (带 NORTHBOUND_COLUMNS schema), 走「无北向持仓」empty + 写空+done (脚本头部既定语义), 不消耗重试/断路器配额; + 其他 TypeError 透传给 call_ak_with_retry 走重试。 + """ + try: + return ak.stock_hsgt_individual_em(symbol=symbol) + except TypeError as e: + if "not subscriptable" in str(e): + logger.debug( + "stock_hsgt_individual_em(%s) → 空 (无北向数据), 返空 df", symbol) + return pd.DataFrame(columns=NORTHBOUND_COLUMNS) + raise + + def fetch_northbound(symbol: str) -> Optional[pd.DataFrame]: - """stock_hsgt_individual_em(symbol='600519') — 北向持股 (~1700行/股)。""" + """stock_hsgt_individual_em(symbol='600519') — 北向持股 (~1700行/股)。 + + 通过 _safe_northbound_em 包装: 无北向数据标的 akshare 抛 NoneType + not subscriptable, 捕获返空 df (empty 语义, 不耗断路器配额)。""" df, _status = call_ak_with_retry( - ak.stock_hsgt_individual_em, f"northbound/{symbol}", symbol=symbol, + _safe_northbound_em, f"northbound/{symbol}", symbol=symbol, ) return df diff --git a/tests/data_platform/test_akshare_static_download.py b/tests/data_platform/test_akshare_static_download.py index a248211..25b6544 100644 --- a/tests/data_platform/test_akshare_static_download.py +++ b/tests/data_platform/test_akshare_static_download.py @@ -9,6 +9,7 @@ import argparse import pandas as pd +import pytest import akshare_static_download as mod # noqa: E402 (conftest 已加 sys.path) @@ -205,3 +206,34 @@ class TestHotAndGdhsFamily: ids = [u[0] for u in units] assert "20250930_gdhs" in ids assert "20251231_gdhs" not in ids + + +# ======================== ③ northbound 无数据崩溃安全包装 ======================== + +class TestSafeNorthbound: + """_safe_northbound_em (2026-09-05 ak-weekly 断路器事故修复)。 + + akshare stock_hsgt_individual_em 对无北向数据标的 (301xxx 等) 在 + data_json["result"]["pages"] 处抛 TypeError(NoneType not subscriptable) + —— 周六实测 2883 股中 1117 连败, 末段 30 连败触发断路器 rc=2, + 饿死后续 share_capital/top_holders/gdhs (gdhs 目录至今不存在)。 + 端点本身健康 (600519 正常返 1683 行, 09-06 晨复测)。 + 确定性无数据 → 空 df 带 schema → 「无北向持仓」empty 写空+done + (脚本头部既定语义), 不消耗重试/断路器配额 (同 _safe_top_10_em 手法)。 + """ + + def test_no_data_crash_returns_empty_with_schema(self, monkeypatch): + def boom(symbol): + raise TypeError("'NoneType' object is not subscriptable") + monkeypatch.setattr(mod.ak, "stock_hsgt_individual_em", boom) + df = mod._safe_northbound_em("301638") + assert df.empty + assert list(df.columns) == mod.NORTHBOUND_COLUMNS + + def test_other_typeerror_propagates(self, monkeypatch): + """非无数据签名的 TypeError 透传 (走 call_ak_with_retry 重试)。""" + def boom(symbol): + raise TypeError("connection reset by peer") + monkeypatch.setattr(mod.ak, "stock_hsgt_individual_em", boom) + with pytest.raises(TypeError): + mod._safe_northbound_em("301638")