diff --git a/scripts/data_platform/corpus_download.py b/scripts/data_platform/corpus_download.py index 05b6e26..47437a4 100644 --- a/scripts/data_platform/corpus_download.py +++ b/scripts/data_platform/corpus_download.py @@ -144,6 +144,8 @@ class DomainClient: self.cooled = False self.n_requests = 0 self._last = 0.0 + self._t_first = None + self._t_last = None def _pace(self): interval = RATE.get(self.name, 1.0) + random.uniform(*JITTER) @@ -151,8 +153,18 @@ class DomainClient: if wait > 0: time.sleep(wait) self._last = time.monotonic() + if self._t_first is None: + self._t_first = self._last + self._t_last = self._last self.n_requests += 1 + def avg_interval(self): + """域内首末请求间隔/(n-1): 真实节奏(lane 总时长跨段失真, 09-07 实锤 + cninfo n=3 avg=3621s 的乌龙)。""" + if self.n_requests < 2 or self._t_first is None: + return None + return (self._t_last - self._t_first) / (self.n_requests - 1) + def _transport_fail(self, msg): self.consecutive_errors += 1 if self.consecutive_errors >= COOLDOWN_ERRORS: @@ -1176,9 +1188,10 @@ def run_lane(lane, until=None, limit=None): _save_depth(ctx) for name, cl in ctx.clients.items(): if cl.n_requests: - log.info("stats %s: n_requests=%d avg_interval=%.2fs%s", + avg = cl.avg_interval() + log.info("stats %s: n_requests=%d avg_interval=%s%s", name, cl.n_requests, - (time.monotonic() - t0) / cl.n_requests, + f"{avg:.2f}s" if avg is not None else "n/a", " [429]" if cl.rate_limited else " [cooled]" if cl.cooled else "") log.info("lane=%s done in %.0fs: units=%d failed=%d rc=%d",