diff --git a/scripts/data_platform/akshare_static_download.py b/scripts/data_platform/akshare_static_download.py index 4891eeb..4402adb 100644 --- a/scripts/data_platform/akshare_static_download.py +++ b/scripts/data_platform/akshare_static_download.py @@ -54,6 +54,8 @@ python akshare_static_download.py --types dragon_tiger --start 20260101 --end 20260715 # 强制重下 python akshare_static_download.py --types valuation --force + # gdhs 一次性补史 (滚动 3 年窗之外, marker 幂等): + python akshare_static_download.py --types gdhs --gdhs-start 20130331 输出目录结构 (OUT_DIR/data/static): {OUT_DIR}/ @@ -874,21 +876,31 @@ def download_one_unit( def build_gdhs_units( now: Optional[datetime.date] = None, + gdhs_start: Optional[str] = None, ) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: """构造 gdhs units: 近 3 年季度末 × 披露守卫, 每期 1 unit。 只枚举距今 >GDHS_DISCLOSE_DAYS 的季度末 (披露完毕), 防止未披露期拉回 空 df 走「真空写空+done」语义后被 marker 锁死永不重拉。marker 去重: 已抓期 skip, 无新期时零网络调用 (ak-weekly 每周六空转)。 + + gdhs_start (YYYYMMDD, --gdhs-start): 一次性补史入口, 枚举扩到该起点 + (2026-09-08 实测源头边界: 东财端点最早 2013Q1 有货, 2012Q4 及更早返空)。 + 已 marker 的期照旧 skip → 重跑幂等; 老期披露守卫天然满足。 """ now = now or datetime.date.today() cutoff = now - datetime.timedelta(days=GDHS_DISCLOSE_DAYS) + # floor=min(起点, 窗起点): 起点晚于滚动窗起点时退化为默认枚举, 绝不收窄周更覆盖 + window_floor = datetime.date(now.year - 2, 1, 1) + floor = (min(datetime.datetime.strptime(gdhs_start, "%Y%m%d").date(), window_floor) + if gdhs_start else window_floor) periods = [] - for y in range(now.year - 2, now.year + 1): + for y in range(floor.year, now.year + 1): for (m, d) in [(3, 31), (6, 30), (9, 30), (12, 31)]: p = datetime.date(y, m, d) - if p <= cutoff: - periods.append(f"{y}{m:02d}{d:02d}") + if p > cutoff or p < floor: + continue + periods.append(f"{y}{m:02d}{d:02d}") units = [(f"{p}_{GDHS}", partial(fetch_gdhs, p)) for p in periods] logger.info( "[%s] 季度末 %d 个 (披露守卫 >%d 天, %s..%s)", @@ -1175,6 +1187,9 @@ def parse_args() -> argparse.Namespace: help="限制处理股票数 (per-stock 类生效), 测试用", ) p.add_argument("--force", action="store_true", help="强制重下, 忽略 marker") + p.add_argument("--gdhs-start", default="", + help="gdhs 一次性补史起点 YYYYMMDD (源头最早 20130331); " + "默认仅滚动 3 年窗") p.add_argument( "--repair", action="store_true", help="只重取 missing/empty(size<1KB 或 df.empty)/corrupt 的 parquet, " @@ -1245,7 +1260,7 @@ def run_type_dispatch( return run_one_type(t, units, args) if t == GDHS: - units = build_gdhs_units() + units = build_gdhs_units(gdhs_start=args.gdhs_start or None) return run_one_type(t, units, args) if t in PER_PERIOD_TYPES: diff --git a/tests/data_platform/test_akshare_static_download.py b/tests/data_platform/test_akshare_static_download.py index 25b6544..33d269d 100644 --- a/tests/data_platform/test_akshare_static_download.py +++ b/tests/data_platform/test_akshare_static_download.py @@ -207,6 +207,27 @@ class TestHotAndGdhsFamily: assert "20250930_gdhs" in ids assert "20251231_gdhs" not in ids + def test_gdhs_units_backfill_flag_extends_history(self): + """--gdhs-start 一次性补史: 枚举扩到指定起点; 默认(无 flag)行为零变化。 + + 源头边界 (2026-09-08 实测): 东财端点最早 2013Q1 有货, 2012Q4 及更早 + 返回空 → akshare 内部 TypeError。补史从 20130331 起枚举无死期。 + """ + import datetime as _dt + now = _dt.date(2026, 9, 8) + # 默认: 滚动 3 年窗 9 期, 与线上现状一致 + ids = [u[0] for u in mod.build_gdhs_units(now=now)] + assert ids[0] == "20240331_gdhs" and len(ids) == 9 + # 补史: 2013Q1 起, 新增 44 期 (2013Q1..2023Q4), 与滚动窗无缝衔接 + ids2 = [u[0] for u in mod.build_gdhs_units(now=now, gdhs_start="20130331")] + assert ids2[0] == "20130331_gdhs" + assert len(ids2) == len(ids) + 44 + assert len(set(ids2)) == len(ids2) and ids2 == sorted(ids2) + assert ids2[-9:] == ids + # 起点晚于滚动窗起点 → 退化为默认枚举 + ids3 = [u[0] for u in mod.build_gdhs_units(now=now, gdhs_start="20250101")] + assert ids3 == ids + # ======================== ③ northbound 无数据崩溃安全包装 ========================