diff --git a/scripts/data_platform/corpus_download.py b/scripts/data_platform/corpus_download.py index 2b74691..8908f9c 100644 --- a/scripts/data_platform/corpus_download.py +++ b/scripts/data_platform/corpus_download.py @@ -568,6 +568,8 @@ class Ctx: self.units = 0 self.stage_units = 0 self.depth = {} + self.clients = {} + self.t0 = time.monotonic() def reset_stage(self): """--limit 预算按段独立(冒烟要覆盖每段;曾因预算共享+pdf 段无门控 @@ -840,6 +842,7 @@ def run_daily(ctx, pool): cl_cn = DomainClient("cninfo") cl_em = DomainClient("eastmoney") cl_art = DomainClient("article") + ctx.clients = {"cninfo": cl_cn, "eastmoney": cl_em, "article": cl_art} candidates = _run_ann_increment(ctx, pool, cl_cn, ledgers, recents) _run_news_increment(ctx, pool, cl_em, ledgers, recents, new_articles := []) _run_fulltext(ctx, cl_art, ledgers, recents, new_articles) @@ -1083,6 +1086,7 @@ def run_backfill(ctx, pool): cl_cn = DomainClient("cninfo") cl_em = DomainClient("eastmoney") cl_art = DomainClient("article") + ctx.clients = {"cninfo": cl_cn, "eastmoney": cl_em, "article": cl_art} _run_ann_backfill(ctx, pool, cl_cn, ledgers, recents) _run_news_backfill(ctx, pool, cl_em, ledgers, recents) _run_fulltext_backfill(ctx, cl_art, ledgers, recents) @@ -1125,6 +1129,13 @@ def run_lane(lane, until=None, limit=None): log.info("墙钟到(--until %s): 完成当前 unit 后 checkpoint 退出", until) finally: _save_depth(ctx) + for name, cl in ctx.clients.items(): + if cl.n_requests: + log.info("stats %s: n_requests=%d avg_interval=%.2fs%s", + name, cl.n_requests, + (time.monotonic() - t0) / cl.n_requests, + " [429]" if cl.rate_limited else + " [cooled]" if cl.cooled else "") log.info("lane=%s done in %.0fs: units=%d failed=%d rc=%d", lane, time.monotonic() - t0, ctx.units, ctx.failed, ctx.rc()) return ctx.rc() diff --git a/scripts/data_platform/verify_corpus.py b/scripts/data_platform/verify_corpus.py new file mode 100644 index 0000000..af6a985 --- /dev/null +++ b/scripts/data_platform/verify_corpus.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""verify_corpus.py — 语料库只读验收器(spec §18.5/T6 晨检, 零写入零网络)。 + +在一次性容器里跑(同 corpus_download 的挂载): + docker run --rm --user 1024:100 --group-add 101 --entrypoint python \ + -v /volume1/stock:/volume1/stock sanguo_vnpy_v2:lock-aligned \ + /app/scripts/data_platform/verify_corpus.py [--root /volume1/stock/corpus] + +输出: 每域分区/行数/键重复/账本一致性/marker/深度/容量/最近日志尾部 stats 行。 +判定: 任何 DUPS>0 或 LEDGER