feat(data): backfill ①年序翻转新→旧(近年优先供factor) [nas]
spec§18.3 字面「2000→今」与其自设验收「回补~1周后即可供factor事件回测」 矛盾(升序前8天只有2000-2002老数据,近年最后才到);翻转为年DESC同PDF④ 哲学: day1 即产当年数据,~1周完成近3-4年=事件因子可用,全量完结不受影响 (§18.5「600519 年报2001起逐年可查」照过)。空年照旧合法done零成本 含 cosmetic: reset_stage 归位 docstring 之后(前次批量插入错位)
This commit is contained in:
@@ -849,12 +849,13 @@ def run_daily(ctx, pool):
|
||||
# ---------- backfill lane(①ann_meta ②news_meta ③fulltext ④PDF;T5) ----------
|
||||
|
||||
def _run_ann_backfill(ctx, pool, cl_cn, ledgers, recents):
|
||||
"""①公告元数据回补 2000↔今全区间, 年序=新→旧(spec§18.7「~1周后可供 factor
|
||||
事件回测」只有近年优先才成立;空年=合法 done, 探测优先不设深度上限)。"""
|
||||
ctx.reset_stage()
|
||||
"""①公告元数据回补 2000→今, unit=(stock,year), 逐年按池序(探测优先:空年=合法)。"""
|
||||
today = dt.date.today()
|
||||
first_seen = today.isoformat()
|
||||
since_flush = 0
|
||||
for year in range(BACKFILL_FROM_YEAR, today.year + 1):
|
||||
for year in range(today.year, BACKFILL_FROM_YEAR - 1, -1):
|
||||
year_end = today.isoformat() if year == today.year else f"{year}-12-31"
|
||||
for code, org in pool:
|
||||
unit = f"{code}_{year}"
|
||||
@@ -890,8 +891,8 @@ def _run_ann_backfill(ctx, pool, cl_cn, ledgers, recents):
|
||||
|
||||
|
||||
def _run_news_backfill(ctx, pool, cl_em, ledgers, recents):
|
||||
ctx.reset_stage()
|
||||
"""②新闻标题级回补(np-listapi), unit=(stock,page), 翻到空页或 page≥200 止。"""
|
||||
ctx.reset_stage()
|
||||
for code, _org in pool:
|
||||
if is_done(ctx.lane, "news_end", code):
|
||||
continue
|
||||
@@ -937,9 +938,9 @@ def _run_news_backfill(ctx, pool, cl_em, ledgers, recents):
|
||||
|
||||
|
||||
def _run_fulltext_backfill(ctx, cl_art, ledgers, recents):
|
||||
ctx.reset_stage()
|
||||
"""③新闻全文回补: 扫 news_meta 库存 → 未入账 art 补正文; 账本即进度
|
||||
(无 marker 文件); 404 墓碑行(content_text=None)入账不再重试。"""
|
||||
ctx.reset_stage()
|
||||
rows_all = []
|
||||
for part in _iter_partitions("news_meta"):
|
||||
df = pd.read_parquet(part, columns=["art_code", "url", "show_time"])
|
||||
|
||||
Reference in New Issue
Block a user