fix(data): 静态表下载两隐患根治——①北交所白烧重试跳过(东财三报表端点SH920xxx必败'NoneType'错,09-01季频洗库实测340股×3表×3retry+退避白烧≈5h;对齐top_holders BJ_PREFIXES同款跳过,valuation/financial_abstract实测BSE有数据不跳)②fetch失败语义修正:15个fetcher不再经_df_or_empty把重试耗尽(None)转空df→空parquet覆盖旧好数据+标done永不重试(09-01实证340空文件全BSE属侥幸,主板瞬时故障会真毁数据);改返None交download_one_unit既有failed分支=不写不标保旧数据复跑重试,真空df才写空+done;test_top_holders一处旧契约断言同步(失败返None);+8测试钉死 [vps]
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""静态表下载两大隐患修复测试 (2026-09-01 洗库事故, 修正①②)。
|
||||
|
||||
①: 三大报表类跳过北交所前缀 (920/43/83/87) — 东财端点必败白烧重试
|
||||
(09-01 实测 340 股 × 3 表 × 3 retry ≈ 5h), valuation 类实测 BSE 有数据不跳。
|
||||
②: fetch 失败(重试耗尽)返 None → download_one_unit 按 failed 处理,
|
||||
不写空 parquet 覆盖好数据、不标 done (旧版 _df_or_empty 转空 df 的暗坑)。
|
||||
"""
|
||||
import argparse
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import akshare_static_download as mod # noqa: E402 (conftest 已加 sys.path)
|
||||
|
||||
|
||||
def _args(**kw):
|
||||
base = dict(codes=None, limit=0)
|
||||
base.update(kw)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
# ======================== ① 北交所跳过 ========================
|
||||
|
||||
class TestStatementBseSkip:
|
||||
ALL_CODES = [
|
||||
("600519", "SH"), ("000001", "SZ"),
|
||||
("920985", "SH"), ("430047", "SZ"), # 北交所新/旧代码段
|
||||
]
|
||||
|
||||
def test_statements_skip_bse(self):
|
||||
"""balance/income/cashflow 三类跳北交所, 主板保留。"""
|
||||
for t, endpoint in (("balance", "balance_sheet"),
|
||||
("income", "income_sheet"),
|
||||
("cashflow", "cashflow_sheet")):
|
||||
units = mod.build_per_stock_units(
|
||||
t, endpoint, mod.fetch_balance_sheet, self.ALL_CODES, _args())
|
||||
codes = {uid.split(".")[0] for uid, _ in units}
|
||||
assert codes == {"600519", "000001"}, t
|
||||
|
||||
def test_valuation_keeps_bse(self):
|
||||
"""valuation/financial_abstract 实测 BSE 有数据, 不跳。"""
|
||||
for t in ("valuation", "financial_abstract"):
|
||||
units = mod.build_per_stock_units(
|
||||
t, t, mod.fetch_valuation, self.ALL_CODES, _args())
|
||||
assert len(units) == 4, t
|
||||
|
||||
def test_codes_filter_bse_only_yields_zero(self):
|
||||
"""--codes 显式指定北交所 + 报表类 → 0 unit (语义明确)。"""
|
||||
units = mod.build_per_stock_units(
|
||||
"balance", "balance_sheet", mod.fetch_balance_sheet,
|
||||
[("920985", "SH")], _args(codes="920985"))
|
||||
assert units == []
|
||||
|
||||
|
||||
# ======================== ② 失败不覆盖好数据 ========================
|
||||
|
||||
class TestFailedNoOverwrite:
|
||||
|
||||
def test_failed_fetch_writes_nothing(self, tmp_path, monkeypatch):
|
||||
"""fetch 返 None (重试耗尽) → failed, parquet/marker 都不落。"""
|
||||
monkeypatch.setattr(mod, "OUT_DIR", tmp_path)
|
||||
status, rows = mod.download_one_unit(
|
||||
"balance", "600519.SH_balance", lambda: None, force=True)
|
||||
assert status == "failed"
|
||||
assert not (tmp_path / "balance" / "600519.SH_balance.parquet").exists()
|
||||
assert not (tmp_path / "balance" / ".600519.SH_balance.akshare").exists()
|
||||
|
||||
def test_genuine_empty_writes_and_marks(self, tmp_path, monkeypatch):
|
||||
"""端点成功返真空 df → 写空 parquet + marker ("查过了确实无数据")。"""
|
||||
monkeypatch.setattr(mod, "OUT_DIR", tmp_path)
|
||||
status, _ = mod.download_one_unit(
|
||||
"balance", "600519.SH_balance",
|
||||
lambda: pd.DataFrame(), force=True)
|
||||
assert status == "empty"
|
||||
assert (tmp_path / "balance" / "600519.SH_balance.parquet").exists()
|
||||
assert (tmp_path / "balance" / ".600519.SH_balance.akshare").exists()
|
||||
|
||||
def test_ok_fetch_writes_data(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(mod, "OUT_DIR", tmp_path)
|
||||
status, rows = mod.download_one_unit(
|
||||
"balance", "600519.SH_balance",
|
||||
lambda: pd.DataFrame({"v": [1, 2]}), force=True)
|
||||
assert status == "ok" and rows == 2
|
||||
assert len(pd.read_parquet(
|
||||
tmp_path / "balance" / "600519.SH_balance.parquet")) == 2
|
||||
|
||||
def test_fetchers_return_none_when_retry_exhausted(self, monkeypatch):
|
||||
"""三类模式 (per-stock/per-date/per-period) 的 fetcher 失败统一返 None。"""
|
||||
monkeypatch.setattr(
|
||||
mod, "call_ak_with_retry",
|
||||
lambda *a, **k: (None, "failed"))
|
||||
assert mod.fetch_balance_sheet("SH600519") is None
|
||||
assert mod.fetch_dragon_tiger("20260901") is None
|
||||
assert mod.fetch_forecast("20260630") is None
|
||||
|
||||
def test_fetchers_return_df_on_ok(self, monkeypatch):
|
||||
df = pd.DataFrame({"a": [1]})
|
||||
monkeypatch.setattr(mod, "call_ak_with_retry",
|
||||
lambda *a, **k: (df, "ok"))
|
||||
assert mod.fetch_income_sheet("SH600519") is df
|
||||
assert mod.fetch_express("20260630") is df
|
||||
@@ -194,8 +194,9 @@ class TestFetchTopHoldersOnePeriod:
|
||||
assert call_count["n"] == mod.AK_MAX_RETRIES, (
|
||||
f"网络异常应重试 {mod.AK_MAX_RETRIES} 次, 实际 {call_count['n']} 次"
|
||||
)
|
||||
# 重试耗尽返空 df (无 schema, 因为确实是失败)
|
||||
assert isinstance(df, pd.DataFrame)
|
||||
# 重试耗尽返 None (2026-09-01 语义修正: 失败≠空数据, download_one_unit
|
||||
# 按 failed 处理, 不写空 parquet 覆盖好数据/不标 done)
|
||||
assert df is None
|
||||
|
||||
|
||||
class TestSafeTop10EmKeyErrorSdltgd:
|
||||
|
||||
Reference in New Issue
Block a user