From d426619a965ba37eba2fb05474b0621d85b61809 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Mon, 7 Sep 2026 12:27:46 +0800 Subject: [PATCH] =?UTF-8?q?fix(data):=20stats=20=E6=94=B9=E5=9F=9F?= =?UTF-8?q?=E5=86=85=E9=A6=96=E6=9C=AB=E8=AF=B7=E6=B1=82=E9=97=B4=E9=9A=94?= =?UTF-8?q?(lane=20=E6=80=BB=E6=97=B6=E9=95=BF=E8=B7=A8=E6=AE=B5=E5=A4=B1?= =?UTF-8?q?=E7=9C=9F)=20[nas]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 09-07 首跑 daily 收官 stats 乌龙: cninfo n=3 avg=3621s(被全文段时长摊薄)/ eastmoney 2.26s(真实1.25s); 改 DomainClient 首末请求时间戳 avg=(t_last-t_first)/(n-1), 后续 run 起直读真节奏 --- scripts/data_platform/corpus_download.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/data_platform/corpus_download.py b/scripts/data_platform/corpus_download.py index 05b6e26..47437a4 100644 --- a/scripts/data_platform/corpus_download.py +++ b/scripts/data_platform/corpus_download.py @@ -144,6 +144,8 @@ class DomainClient: self.cooled = False self.n_requests = 0 self._last = 0.0 + self._t_first = None + self._t_last = None def _pace(self): interval = RATE.get(self.name, 1.0) + random.uniform(*JITTER) @@ -151,8 +153,18 @@ class DomainClient: if wait > 0: time.sleep(wait) self._last = time.monotonic() + if self._t_first is None: + self._t_first = self._last + self._t_last = self._last self.n_requests += 1 + def avg_interval(self): + """域内首末请求间隔/(n-1): 真实节奏(lane 总时长跨段失真, 09-07 实锤 + cninfo n=3 avg=3621s 的乌龙)。""" + if self.n_requests < 2 or self._t_first is None: + return None + return (self._t_last - self._t_first) / (self.n_requests - 1) + def _transport_fail(self, msg): self.consecutive_errors += 1 if self.consecutive_errors >= COOLDOWN_ERRORS: @@ -1176,9 +1188,10 @@ def run_lane(lane, until=None, limit=None): _save_depth(ctx) for name, cl in ctx.clients.items(): if cl.n_requests: - log.info("stats %s: n_requests=%d avg_interval=%.2fs%s", + avg = cl.avg_interval() + log.info("stats %s: n_requests=%d avg_interval=%s%s", name, cl.n_requests, - (time.monotonic() - t0) / cl.n_requests, + f"{avg:.2f}s" if avg is not None else "n/a", " [429]" if cl.rate_limited else " [cooled]" if cl.cooled else "") log.info("lane=%s done in %.0fs: units=%d failed=%d rc=%d",