From 46a0569b9a4a66e31b276ce64bc667c81a8bd10c Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 2 Sep 2026 09:16:14 +0800 Subject: [PATCH] =?UTF-8?q?feat(data):=20A=E6=A1=A3=E5=85=A8=E9=87=8F?= =?UTF-8?q?=E8=90=BD=E5=9C=B0=E2=80=94=E2=80=94=E7=83=AD=E5=BA=A6=E4=B8=89?= =?UTF-8?q?=E4=BB=B6+=E8=82=A1=E4=B8=9C=E6=88=B7=E6=95=B0=E6=94=B6?= =?UTF-8?q?=E5=B0=BE(=E7=94=A8=E6=88=B7=E6=89=B9=E3=80=8C=E6=8C=89?= =?UTF-8?q?=E6=AD=A4=E6=96=B9=E5=BC=8F=E5=AE=9E=E6=96=BD,nas=E9=AA=8C?= =?UTF-8?q?=E8=AF=81=E5=90=8E=E6=8E=A8vps=E3=80=8D);=E2=91=A0xueqiu=5Fhot?= =?UTF-8?q?=E9=9B=AA=E7=90=835633=E8=A1=8C+sina=5Fsector=E6=96=B0=E6=B5=AA?= =?UTF-8?q?=E8=A1=8C=E4=B8=9A49=E8=A1=8C=E5=AF=B9=E8=B4=A6=E6=BA=90?= =?UTF-8?q?=E8=BF=9Bak-events=2019:30(=E5=90=8C=E6=AC=BE=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E8=AF=AD=E4=B9=89)=E2=91=A1hot=5Frank=E4=B8=9C=E8=B4=A2?= =?UTF-8?q?=E4=BA=BA=E6=B0=94=E6=A6=9Cfetcher/=E6=B3=A8=E5=86=8C=E5=85=88?= =?UTF-8?q?=E8=A1=8C,wrapper=E6=8C=82=E8=BD=BD=E7=95=99=E5=BE=85=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E7=AA=97=E6=B5=8B=E9=87=8F=E6=8B=8D=E6=9D=BF(?= =?UTF-8?q?=E5=A2=99=E6=9C=AA=E5=AE=9A=E8=AE=BA=E4=B8=8D=E5=B8=A6=E4=B8=8D?= =?UTF-8?q?=E7=A1=AE=E5=AE=9A=E5=8D=95=E5=85=83=E4=B8=8A=E7=BA=BF=E9=98=B2?= =?UTF-8?q?=E7=BC=BA=E6=97=A5=E5=91=8A=E8=AD=A6=E5=99=AA=E9=9F=B3)?= =?UTF-8?q?=E2=91=A2gdhs=E8=82=A1=E4=B8=9C=E6=88=B7=E6=95=B0=E8=BF=9Bak-we?= =?UTF-8?q?ekly=E5=91=A8=E5=85=AD=E7=AA=97=3Dper-period=E6=96=B0=E5=BD=A2?= =?UTF-8?q?=E7=8A=B6:build=5Fgdhs=5Funits=E5=8F=AA=E6=9E=9A=E4=B8=BE?= =?UTF-8?q?=E8=B7=9D=E4=BB=8A>90=E5=A4=A9=E5=AD=A3=E5=BA=A6=E6=9C=AB(?= =?UTF-8?q?=E6=8A=AB=E9=9C=B2=E5=AE=88=E5=8D=AB,=E9=98=B2=E6=9C=AA?= =?UTF-8?q?=E6=8A=AB=E9=9C=B2=E6=9C=9F=E6=8B=89=E7=A9=BA=E8=B5=B0=E7=9C=9F?= =?UTF-8?q?=E7=A9=BA+done=E8=AF=AD=E4=B9=89=E8=A2=ABmarker=E9=94=81?= =?UTF-8?q?=E6=AD=BB=E6=B0=B8=E4=B8=8D=E9=87=8D=E6=8B=89=E7=9A=8409-02?= =?UTF-8?q?=E8=AE=BE=E8=AE=A1=E5=9D=91),=E5=8D=95=E6=AC=A1=E5=85=A8?= =?UTF-8?q?=E5=B8=82=E5=9C=BA5342=E8=A1=8C,marker=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E6=97=A0=E6=96=B0=E6=9C=9F=E9=9B=B6=E7=BD=91=E7=BB=9C;vintage?= =?UTF-8?q?=E7=BC=BA=E6=97=A5=E5=90=8D=E5=8D=95+2=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E6=97=8F(hot=5Frank=E7=95=99=E4=BD=8D);+6=E6=B5=8B=E8=AF=95(?= =?UTF-8?q?=E6=B3=A8=E5=86=8C/None/df=E5=A5=91=E7=BA=A6/gdhs=E5=AE=88?= =?UTF-8?q?=E5=8D=AB=E4=B8=A4=E5=BD=A2=E6=80=81/=E7=BC=BA=E6=97=A5?= =?UTF-8?q?=E5=90=8D=E5=8D=95)215=E5=85=A8=E7=BB=BF=20[vps]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/data_platform/ak_events_wrapper.ps1 | 5 +- scripts/data_platform/ak_stock_wrapper.ps1 | 7 +- .../data_platform/akshare_static_download.py | 83 ++++++++++++++++++- scripts/data_platform/static_vintage_check.py | 2 + .../test_akshare_static_download.py | 47 +++++++++++ .../test_static_vintage_check.py | 7 ++ 6 files changed, 145 insertions(+), 6 deletions(-) diff --git a/scripts/data_platform/ak_events_wrapper.ps1 b/scripts/data_platform/ak_events_wrapper.ps1 index 16b7a5d..0fa7a25 100644 --- a/scripts/data_platform/ak_events_wrapper.ps1 +++ b/scripts/data_platform/ak_events_wrapper.ps1 @@ -1,6 +1,7 @@ # ak_events_wrapper.ps1 — sanguo-ak-events schtask wrapper (daily 19:30 akshare 事件类 per-date 当日) -# dragon_tiger/block_trade/margin_sse/restricted/zt_pool 三件套/新浪资金流×2/同花顺行业概念 --start today --end today; +# dragon_tiger/block_trade/margin_sse/restricted/zt_pool 三件套/新浪资金流×2/同花顺行业概念/雪球热度/新浪行业对账 --start today --end today; # per-date 每类1 unit 快; zt_pool 系 (2026-09-02 双机实测 push2ex 集群稳定) 供情绪/打板因子; +# hot_rank (东财人气榜) 墙待恢复窗测量, 拍板后加入 --types (fetcher/注册已就位); # 新浪资金流+同花顺榜 (2026-09-02 用户批 A 档: 东财 push2 墙死后免费无墙替代, 秒级零限流) $env:http_proxy = '' $env:https_proxy = '' @@ -11,5 +12,5 @@ $ts = Get-Date -Format 'yyyyMMdd_HHmmss' $logDir = 'C:\sanguo_vnpy_v2\data\migration_logs' if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null } $log = Join-Path $logDir "ak_events_$ts.txt" -C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\akshare_static_download.py --types dragon_tiger,block_trade,margin_sse,restricted,zt_pool,zt_pool_zbgc,zt_pool_dtgc,fund_flow_industry,fund_flow_concept,ths_industry,ths_concept --start $today --end $today *>> $log +C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\akshare_static_download.py --types dragon_tiger,block_trade,margin_sse,restricted,zt_pool,zt_pool_zbgc,zt_pool_dtgc,fund_flow_industry,fund_flow_concept,ths_industry,ths_concept,xueqiu_hot,sina_sector --start $today --end $today *>> $log exit $LASTEXITCODE diff --git a/scripts/data_platform/ak_stock_wrapper.ps1 b/scripts/data_platform/ak_stock_wrapper.ps1 index 50c1129..4885ba1 100644 --- a/scripts/data_platform/ak_stock_wrapper.ps1 +++ b/scripts/data_platform/ak_stock_wrapper.ps1 @@ -1,5 +1,6 @@ -# ak_stock_wrapper.ps1 — sanguo-ak-stock schtask wrapper (weekly 周六03:00 akshare per-stock 慢) -# northbound+share_capital+top_holders --force; per-stock 全量慢, 周末夜间 +# ak_stock_wrapper.ps1 — sanguo-ak-weekly schtask wrapper (weekly 周六03:00 akshare per-stock 慢) +# northbound+share_capital+top_holders+gdhs --force; per-stock 全量慢, 周末夜间; +# gdhs=股东户数 per-period 全市场单次 (09-02 入列, 披露守卫>90天季度末, marker 去重无新期零网络) $env:http_proxy = '' $env:https_proxy = '' $env:all_proxy = '' @@ -8,5 +9,5 @@ $ts = Get-Date -Format 'yyyyMMdd_HHmmss' $logDir = 'C:\sanguo_vnpy_v2\data\migration_logs' if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null } $log = Join-Path $logDir "ak_stock_$ts.txt" -C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\akshare_static_download.py --types northbound,share_capital,top_holders --force *>> $log +C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\akshare_static_download.py --types northbound,share_capital,top_holders,gdhs --force *>> $log exit $LASTEXITCODE diff --git a/scripts/data_platform/akshare_static_download.py b/scripts/data_platform/akshare_static_download.py index 7e6c1ee..2fbbfe4 100644 --- a/scripts/data_platform/akshare_static_download.py +++ b/scripts/data_platform/akshare_static_download.py @@ -143,7 +143,7 @@ PER_STOCK_TYPES = ( "financial_abstract", # stock_financial_abstract(symbol="600519") # top_holders 单独 (per-stock × per-period) ) -# 模式 B: per-date 类型 (11 类, margin_szse 跳过) +# 模式 B: per-date 类型 (14 类, margin_szse 跳过) PER_DATE_TYPES = ( "dragon_tiger", # stock_lhb_detail_em(start_date, end_date) "block_trade", # stock_dzjy_mrmx(symbol="A股", start_date, end_date) @@ -164,6 +164,12 @@ PER_DATE_TYPES = ( "fund_flow_concept", # 新浪 stock_fund_flow_concept('即时') 概念资金流 ~387 行 "ths_industry", # 同花顺 stock_board_industry_summary_ths() 行业一览 90 行×12 列 "ths_concept", # 同花顺 stock_board_concept_name_ths() 概念名单 ~375 行 + # 热度/对账三件 (09-02 用户批全量落地; 快照语义同上). hot_rank=东财 emappdata + # 集群 (墙待测, fetcher/注册先行, wrapper 挂载等恢复窗测量拍板); + # xueqiu_hot/sina_sector 双机实测无墙. + "hot_rank", # 东财 stock_hot_rank_em() 人气榜 ~100 行 + "xueqiu_hot", # 雪球 stock_hot_follow_xq('最热门') 关注热度 ~5633 行 + "sina_sector", # 新浪 stock_sector_spot() 行业 49 行 (同花顺 90 行的对账源) ) # 模式 C: per-period 类型 (2 类) PER_PERIOD_TYPES = ( @@ -177,6 +183,12 @@ ONE_SHOT_TYPES = ( ) # top_holders 特殊: per-stock × per-period TOP_HOLDERS = "top_holders" +# 股东户数 特殊: per-period (季度末) 单次全市场快照, 09-02 入列 ak-weekly。 +# ⚠️ 披露守卫: 只枚举距今 >GDHS_DISCLOSE_DAYS 的季度末 —— 未披露期拉回空 df +# 会走「真空写空+done」语义永不重拉 (09-02 设计坑), 守卫保证枚举出来的期 +# 数据必然已披露完, 空了才是真空。 +GDHS = "gdhs" +GDHS_DISCLOSE_DAYS = 90 # 季度末后 90 天 = 披露期结束, 剩余未披露的属长尾 # 北交所代码段 (920新段 + 83/87/43历史段): akshare 东财 stock_gdfx_free_top_10_em # 不支持北交所, build_top_holders_units 阶段直接跳过 (避免每只×20期×3retry 失败风暴). BJ_PREFIXES = ("920", "83", "87", "43") @@ -192,6 +204,7 @@ ALL_TYPES = ( + PER_DATE_TYPES + PER_PERIOD_TYPES + ONE_SHOT_TYPES + + (GDHS,) ) @@ -682,6 +695,41 @@ def fetch_ths_concept(date: str) -> Optional[pd.DataFrame]: return df +def fetch_hot_rank(date: str) -> Optional[pd.DataFrame]: + """stock_hot_rank_em() — 东财人气榜快照 (~100 行)。 + + emappdata 集群墙待恢复窗测量定论; 失败走 None 语义 (不写不标), 无害。 + """ + df, _status = call_ak_with_retry( + ak.stock_hot_rank_em, f"hot_rank/{date}", + ) + return df + + +def fetch_xueqiu_hot(date: str) -> Optional[pd.DataFrame]: + """stock_hot_follow_xq('最热门') — 雪球关注热度快照 (~5633 行)。""" + df, _status = call_ak_with_retry( + ak.stock_hot_follow_xq, f"xueqiu_hot/{date}", symbol="最热门", + ) + return df + + +def fetch_sina_sector(date: str) -> Optional[pd.DataFrame]: + """stock_sector_spot() — 新浪行业快照 (49 行, 同花顺行业榜对账源)。""" + df, _status = call_ak_with_retry( + ak.stock_sector_spot, f"sina_sector/{date}", + ) + return df + + +def fetch_gdhs(period: str) -> Optional[pd.DataFrame]: + """stock_zh_a_gdhs(symbol=period) — 股东户数全市场按期单次 (~5342 行)。""" + df, _status = call_ak_with_retry( + ak.stock_zh_a_gdhs, f"gdhs/{period}", symbol=period, + ) + return df + + # ======================== per-period fetch 函数 (2 类) ======================== def fetch_forecast(period: str) -> Optional[pd.DataFrame]: @@ -790,6 +838,32 @@ def download_one_unit( # ======================== 主循环 (通用, 适用所有四种模式) ======================== +def build_gdhs_units( + now: Optional[datetime.date] = None, +) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: + """构造 gdhs units: 近 3 年季度末 × 披露守卫, 每期 1 unit。 + + 只枚举距今 >GDHS_DISCLOSE_DAYS 的季度末 (披露完毕), 防止未披露期拉回 + 空 df 走「真空写空+done」语义后被 marker 锁死永不重拉。marker 去重: + 已抓期 skip, 无新期时零网络调用 (ak-weekly 每周六空转)。 + """ + now = now or datetime.date.today() + cutoff = now - datetime.timedelta(days=GDHS_DISCLOSE_DAYS) + periods = [] + for y in range(now.year - 2, now.year + 1): + for (m, d) in [(3, 31), (6, 30), (9, 30), (12, 31)]: + p = datetime.date(y, m, d) + if p <= cutoff: + periods.append(f"{y}{m:02d}{d:02d}") + units = [(f"{p}_{GDHS}", partial(fetch_gdhs, p)) for p in periods] + logger.info( + "[%s] 季度末 %d 个 (披露守卫 >%d 天, %s..%s)", + GDHS, len(units), GDHS_DISCLOSE_DAYS, + periods[0] if periods else "-", periods[-1] if periods else "-", + ) + return units + + def run_one_type( data_type: str, units: List[Tuple[str, Callable[[], pd.DataFrame]]], @@ -1129,10 +1203,17 @@ def run_type_dispatch( "fund_flow_concept": fetch_fund_flow_concept, "ths_industry": fetch_ths_industry, "ths_concept": fetch_ths_concept, + "hot_rank": fetch_hot_rank, + "xueqiu_hot": fetch_xueqiu_hot, + "sina_sector": fetch_sina_sector, } units = build_per_date_units(t, fetch_map[t], args) return run_one_type(t, units, args) + if t == GDHS: + units = build_gdhs_units() + return run_one_type(t, units, args) + if t in PER_PERIOD_TYPES: fetch_map = { "forecast": fetch_forecast, diff --git a/scripts/data_platform/static_vintage_check.py b/scripts/data_platform/static_vintage_check.py index a018243..ff6dc51 100644 --- a/scripts/data_platform/static_vintage_check.py +++ b/scripts/data_platform/static_vintage_check.py @@ -38,6 +38,8 @@ PANEL_TYPES = ( "zt_pool", "zt_pool_zbgc", "zt_pool_dtgc", # 可回补族 "fund_flow_industry", "fund_flow_concept", # 快照族 (永久) "ths_industry", "ths_concept", + "xueqiu_hot", "sina_sector", # 快照族 (09-02 全量落地) + # hot_rank 待恢复窗测量拍板挂载后再入列 (避免未上线就缺日告警) ) BACKFILLABLE_PANEL_TYPES = frozenset({"zt_pool", "zt_pool_zbgc", "zt_pool_dtgc"}) PANEL_BACKFILL_CALENDAR_DAYS = 42 # ≈30 交易日回补窗的日历日近似 diff --git a/tests/data_platform/test_akshare_static_download.py b/tests/data_platform/test_akshare_static_download.py index 648965c..a248211 100644 --- a/tests/data_platform/test_akshare_static_download.py +++ b/tests/data_platform/test_akshare_static_download.py @@ -158,3 +158,50 @@ class TestSinaThsFamily: for fetch in (mod.fetch_fund_flow_industry, mod.fetch_fund_flow_concept, mod.fetch_ths_industry, mod.fetch_ths_concept): assert fetch("20260902") is df + + +class TestHotAndGdhsFamily: + """热度三件 + 股东户数 (09-02 用户批全量落地)。""" + + def test_registry(self): + for t in ("hot_rank", "xueqiu_hot", "sina_sector"): + assert t in mod.PER_DATE_TYPES + assert mod.GDHS in mod.ALL_TYPES + + def test_fetchers_none_on_retry_exhausted(self, monkeypatch): + monkeypatch.setattr( + mod, "call_ak_with_retry", lambda *a, **k: (None, "failed")) + assert mod.fetch_hot_rank("20260902") is None + assert mod.fetch_xueqiu_hot("20260902") is None + assert mod.fetch_sina_sector("20260902") is None + assert mod.fetch_gdhs("20250630") is None + + def test_fetchers_df_on_ok(self, monkeypatch): + df = pd.DataFrame({"a": [1]}) + monkeypatch.setattr(mod, "call_ak_with_retry", + lambda *a, **k: (df, "ok")) + assert mod.fetch_hot_rank("20260902") is df + assert mod.fetch_xueqiu_hot("20260902") is df + assert mod.fetch_sina_sector("20260902") is df + assert mod.fetch_gdhs("20250630") is df + + def test_gdhs_units_disclosure_guard(self): + """只枚举距今 >90 天的季度末: 90 天内的季度末绝不入列 (防空档标 done 陷阱)。""" + import datetime as _dt + now = _dt.date(2026, 9, 2) + units = mod.build_gdhs_units(now=now) + ids = [u[0] for u in units] + # cutoff = 2026-06-04: 20260630 在 90 天内必须缺席; 20250630 必须在列 + assert "20250630_gdhs" in ids + assert all(not i.startswith(("20260630", "20260930")) for i in ids) + assert all(i[4:8] in ("0331", "0630", "0930", "1231") for i in ids) + # 近 3 年窗口 + assert ids[0].startswith(("2024", "2025")) + + def test_gdhs_units_empty_when_nothing_disclosed(self): + import datetime as _dt + # 年初 1 月: 最新的已披露季度末是去年 9/30 (1231 未满 90 天) + units = mod.build_gdhs_units(now=_dt.date(2026, 1, 15)) + ids = [u[0] for u in units] + assert "20250930_gdhs" in ids + assert "20251231_gdhs" not in ids diff --git a/tests/data_platform/test_static_vintage_check.py b/tests/data_platform/test_static_vintage_check.py index 9cec1db..2764cce 100644 --- a/tests/data_platform/test_static_vintage_check.py +++ b/tests/data_platform/test_static_vintage_check.py @@ -200,6 +200,13 @@ class TestPanelGapCheck: assert hole.isoformat() in ( status_panel["fund_flow_industry"]["holes_permanent"]) + def test_panel_registry_snapshot_family(self): + """09-02 全量落地批次注册正确: 快照族在列且不落可回补档; hot_rank 未挂载不入列。""" + for t in ("xueqiu_hot", "sina_sector", "fund_flow_concept", "ths_concept"): + assert t in svc.PANEL_TYPES + assert t not in svc.BACKFILLABLE_PANEL_TYPES + assert "hot_rank" not in svc.PANEL_TYPES # 拍板挂载后才入列 + # ======================== main / JSON 契约 ========================