feat(data): A档全量落地——热度三件+股东户数收尾(用户批「按此方式实施,nas验证后推vps」);①xueqiu_hot雪球5633行+sina_sector新浪行业49行对账源进ak-events 19:30(同款快照语义)②hot_rank东财人气榜fetcher/注册先行,wrapper挂载留待恢复窗测量拍板(墙未定论不带不确定单元上线防缺日告警噪音)③gdhs股东户数进ak-weekly周六窗=per-period新形状:build_gdhs_units只枚举距今>90天季度末(披露守卫,防未披露期拉空走真空+done语义被marker锁死永不重拉的09-02设计坑),单次全市场5342行,marker去重无新期零网络;vintage缺日名单+2快照族(hot_rank留位);+6测试(注册/None/df契约/gdhs守卫两形态/缺日名单)215全绿 [vps]
This commit is contained in:
@@ -143,7 +143,7 @@ PER_STOCK_TYPES = (
|
||||
"financial_abstract", # stock_financial_abstract(symbol="600519")
|
||||
# top_holders 单独 (per-stock × per-period)
|
||||
)
|
||||
# 模式 B: per-date 类型 (11 类, margin_szse 跳过)
|
||||
# 模式 B: per-date 类型 (14 类, margin_szse 跳过)
|
||||
PER_DATE_TYPES = (
|
||||
"dragon_tiger", # stock_lhb_detail_em(start_date, end_date)
|
||||
"block_trade", # stock_dzjy_mrmx(symbol="A股", start_date, end_date)
|
||||
@@ -164,6 +164,12 @@ PER_DATE_TYPES = (
|
||||
"fund_flow_concept", # 新浪 stock_fund_flow_concept('即时') 概念资金流 ~387 行
|
||||
"ths_industry", # 同花顺 stock_board_industry_summary_ths() 行业一览 90 行×12 列
|
||||
"ths_concept", # 同花顺 stock_board_concept_name_ths() 概念名单 ~375 行
|
||||
# 热度/对账三件 (09-02 用户批全量落地; 快照语义同上). hot_rank=东财 emappdata
|
||||
# 集群 (墙待测, fetcher/注册先行, wrapper 挂载等恢复窗测量拍板);
|
||||
# xueqiu_hot/sina_sector 双机实测无墙.
|
||||
"hot_rank", # 东财 stock_hot_rank_em() 人气榜 ~100 行
|
||||
"xueqiu_hot", # 雪球 stock_hot_follow_xq('最热门') 关注热度 ~5633 行
|
||||
"sina_sector", # 新浪 stock_sector_spot() 行业 49 行 (同花顺 90 行的对账源)
|
||||
)
|
||||
# 模式 C: per-period 类型 (2 类)
|
||||
PER_PERIOD_TYPES = (
|
||||
@@ -177,6 +183,12 @@ ONE_SHOT_TYPES = (
|
||||
)
|
||||
# top_holders 特殊: per-stock × per-period
|
||||
TOP_HOLDERS = "top_holders"
|
||||
# 股东户数 特殊: per-period (季度末) 单次全市场快照, 09-02 入列 ak-weekly。
|
||||
# ⚠️ 披露守卫: 只枚举距今 >GDHS_DISCLOSE_DAYS 的季度末 —— 未披露期拉回空 df
|
||||
# 会走「真空写空+done」语义永不重拉 (09-02 设计坑), 守卫保证枚举出来的期
|
||||
# 数据必然已披露完, 空了才是真空。
|
||||
GDHS = "gdhs"
|
||||
GDHS_DISCLOSE_DAYS = 90 # 季度末后 90 天 = 披露期结束, 剩余未披露的属长尾
|
||||
# 北交所代码段 (920新段 + 83/87/43历史段): akshare 东财 stock_gdfx_free_top_10_em
|
||||
# 不支持北交所, build_top_holders_units 阶段直接跳过 (避免每只×20期×3retry 失败风暴).
|
||||
BJ_PREFIXES = ("920", "83", "87", "43")
|
||||
@@ -192,6 +204,7 @@ ALL_TYPES = (
|
||||
+ PER_DATE_TYPES
|
||||
+ PER_PERIOD_TYPES
|
||||
+ ONE_SHOT_TYPES
|
||||
+ (GDHS,)
|
||||
)
|
||||
|
||||
|
||||
@@ -682,6 +695,41 @@ def fetch_ths_concept(date: str) -> Optional[pd.DataFrame]:
|
||||
return df
|
||||
|
||||
|
||||
def fetch_hot_rank(date: str) -> Optional[pd.DataFrame]:
|
||||
"""stock_hot_rank_em() — 东财人气榜快照 (~100 行)。
|
||||
|
||||
emappdata 集群墙待恢复窗测量定论; 失败走 None 语义 (不写不标), 无害。
|
||||
"""
|
||||
df, _status = call_ak_with_retry(
|
||||
ak.stock_hot_rank_em, f"hot_rank/{date}",
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def fetch_xueqiu_hot(date: str) -> Optional[pd.DataFrame]:
|
||||
"""stock_hot_follow_xq('最热门') — 雪球关注热度快照 (~5633 行)。"""
|
||||
df, _status = call_ak_with_retry(
|
||||
ak.stock_hot_follow_xq, f"xueqiu_hot/{date}", symbol="最热门",
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def fetch_sina_sector(date: str) -> Optional[pd.DataFrame]:
|
||||
"""stock_sector_spot() — 新浪行业快照 (49 行, 同花顺行业榜对账源)。"""
|
||||
df, _status = call_ak_with_retry(
|
||||
ak.stock_sector_spot, f"sina_sector/{date}",
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def fetch_gdhs(period: str) -> Optional[pd.DataFrame]:
|
||||
"""stock_zh_a_gdhs(symbol=period) — 股东户数全市场按期单次 (~5342 行)。"""
|
||||
df, _status = call_ak_with_retry(
|
||||
ak.stock_zh_a_gdhs, f"gdhs/{period}", symbol=period,
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
# ======================== per-period fetch 函数 (2 类) ========================
|
||||
|
||||
def fetch_forecast(period: str) -> Optional[pd.DataFrame]:
|
||||
@@ -790,6 +838,32 @@ def download_one_unit(
|
||||
|
||||
# ======================== 主循环 (通用, 适用所有四种模式) ========================
|
||||
|
||||
def build_gdhs_units(
|
||||
now: Optional[datetime.date] = None,
|
||||
) -> List[Tuple[str, Callable[[], pd.DataFrame]]]:
|
||||
"""构造 gdhs units: 近 3 年季度末 × 披露守卫, 每期 1 unit。
|
||||
|
||||
只枚举距今 >GDHS_DISCLOSE_DAYS 的季度末 (披露完毕), 防止未披露期拉回
|
||||
空 df 走「真空写空+done」语义后被 marker 锁死永不重拉。marker 去重:
|
||||
已抓期 skip, 无新期时零网络调用 (ak-weekly 每周六空转)。
|
||||
"""
|
||||
now = now or datetime.date.today()
|
||||
cutoff = now - datetime.timedelta(days=GDHS_DISCLOSE_DAYS)
|
||||
periods = []
|
||||
for y in range(now.year - 2, now.year + 1):
|
||||
for (m, d) in [(3, 31), (6, 30), (9, 30), (12, 31)]:
|
||||
p = datetime.date(y, m, d)
|
||||
if p <= cutoff:
|
||||
periods.append(f"{y}{m:02d}{d:02d}")
|
||||
units = [(f"{p}_{GDHS}", partial(fetch_gdhs, p)) for p in periods]
|
||||
logger.info(
|
||||
"[%s] 季度末 %d 个 (披露守卫 >%d 天, %s..%s)",
|
||||
GDHS, len(units), GDHS_DISCLOSE_DAYS,
|
||||
periods[0] if periods else "-", periods[-1] if periods else "-",
|
||||
)
|
||||
return units
|
||||
|
||||
|
||||
def run_one_type(
|
||||
data_type: str,
|
||||
units: List[Tuple[str, Callable[[], pd.DataFrame]]],
|
||||
@@ -1129,10 +1203,17 @@ def run_type_dispatch(
|
||||
"fund_flow_concept": fetch_fund_flow_concept,
|
||||
"ths_industry": fetch_ths_industry,
|
||||
"ths_concept": fetch_ths_concept,
|
||||
"hot_rank": fetch_hot_rank,
|
||||
"xueqiu_hot": fetch_xueqiu_hot,
|
||||
"sina_sector": fetch_sina_sector,
|
||||
}
|
||||
units = build_per_date_units(t, fetch_map[t], args)
|
||||
return run_one_type(t, units, args)
|
||||
|
||||
if t == GDHS:
|
||||
units = build_gdhs_units()
|
||||
return run_one_type(t, units, args)
|
||||
|
||||
if t in PER_PERIOD_TYPES:
|
||||
fetch_map = {
|
||||
"forecast": fetch_forecast,
|
||||
|
||||
Reference in New Issue
Block a user