diff --git a/docs/superpowers/specs/2026-10-03-monitoring-design.md b/docs/superpowers/specs/2026-10-03-monitoring-design.md index 706bf3e1..e03ba184 100644 --- a/docs/superpowers/specs/2026-10-03-monitoring-design.md +++ b/docs/superpowers/specs/2026-10-03-monitoring-design.md @@ -239,7 +239,7 @@ kind 语义（检查分发依据，check_entry data_monitor.py:56-232）： ### §5.2 四层证据链与分类学（fusion §20.15.2 迁入，10-03 起权威在此） 「完全与否」四层（弱→强，**以④为准**）： - ①任务 exit 码（类型级失败不抬码，最弱） -- ②类型级统计行 `{"ok":N,"failed":M,"rows":R}` 解析+**失败签名分诊**（monitor_checks.py:185）：`RemoteDisconnected`=断连族红 / `NoneType`=源当日未出黄 / `TLS 418`=源改版红 / `KeyError/TypeError`=源字段变更红（09-27 P2 起字段形状类 1 次即抛 AkShapeError 豁免断路器）/ 断路器触发=熔断连累红（独立键 `data-akshare-breaker`，exit 2 会话） +- ②类型级统计行 `{"ok":N,"failed":M,"rows":R}` 解析+**失败签名分诊**（monitor_checks.py:185）——**10-05 三修**：①只扫 `ERROR`/[FATAL] 行（重试 WARNING 的 `Expecting value` 良性不再误中 source_changed）②针收紧 `418 Client Error`（裸 418 子串误中 rows=41823 进度行=anti_crawl 假阳性实证）③**签名按日志行 `[type]` 归因**（`signatures_by_type`）——他类型的反爬签名不再把本类型 failed 抬红（跨类型连坐修）：`RemoteDisconnected`=断连族红 / `NoneType`=源当日未出黄 / `418 Client Error`=反爬红 / `KeyError/TypeError`=源字段变更红（09-27 P2 起字段形状类 1 次即抛 AkShapeError 豁免断路器；**10-05 起 share_capital 次新股『公告日期』族归 empty 桶不再 failed**，见 §9.B-21）/ 断路器触发=熔断连累红（独立键 `data-akshare-breaker`，exit 2 会话，全局取） - ③marker 按日比对（标了≠数据对，0×0 案） - ④数据本体（存在+行数带宽+schema+值域） @@ -433,6 +433,8 @@ kind 语义（检查分发依据，check_entry data_monitor.py:56-232）： 18. **滚动带被双峰形态拉宽当误报修（margin 2000/6000）**：缺腿日混入窗口把 P1 拉宽是机理内行为。判别=带抓缺腿靠窗口多数为满腿日。修法=无（勿「修」带宽度，修缺腿本身）。 19. **fixture 远跳未来日期塌日志层/mtime（10-03 测试工法）**：E2E 把 _TODAY_OVERRIDE 跳到 fixture 数周后→26h 窗空+dir_mtime 全黄→rc=3 假红。修=远窗场景用 `_calendar_days` 单元测直钉，E2E 只用近日。 20. **两班语义误读**：代码零分叉（shift 仅入文件名），「晨班自愈回看」是时点效果不是独立路径——排障勿找「morning 分支」（§5.3）。 +21. **签名误针三连红假阳性（10-05 巡检案）**：share_capital failed=186（已知次新股族）被抬 RED——真凶=裸 `418` 子串误中 `rows=41823/ok=19418` 进度行（anti_crawl 假阳）+重试 WARNING 行的 `Expecting value`（source_changed 假阳）+签名全日志全局取（financial_abstract 的 WARNING 把 share_capital 连坐）。判别=签名命中处取上下文实证（`418_CTX` 探针法），勿凭签名名单直接定级。修=三修（ERROR 行预滤+针收紧+按 [type] 归因，monitor_checks.py `signatures_by_type`）；**配套根修=次新股『公告日期』KeyError 族归 empty 桶**（akshare_static_download.py `download_unit`，09-12 起从 empty 迁 failed 的分类回迁，次周重试保留）。 +22. **band 跨车道混基线（research_report 10-05）**：rows=58 踩带下沿 58.16 黄——看似「连续 5 周递减」实为**旧全量车道（~0828=859）与 6fed71f 新周窗车道（0925=83/1002=58）混在 30 文件窗里**+双假日周；band 地板跨车道无意义。判别=车道切换 commit 日对齐文件序（git log fetch 函数史）。修=无（窗滚动 30 周自然汰换旧道；首个完整新道观察点=1009 周后），黄灯如实保留不豁免。 ### §9.C 基建（infra）✅ 1. **cmd `2>nul | pipe` 组合重定向失效**：`dir X 2>nul | findstr Y` 的 stderr 照样冒出「系统找不到指定的路径」，与真缺文件不可辨；`&` 链里更乱。→ 判别法/正解：远端 Windows 存在性/属性探查一律 `powershell Test-Path`/`Get-Item`（10-03 报警甄别实案）。 2. **schtasks zh-CN locale 三坑**：`/d SAT` 周英文无效（静态表季频案）；日期格式随 locale（/sd 用 `2026/10/04` 式）；`/create` 传中文参数须 UTF-16。→ 注册脚本一律 ASCII+locale 中性格式。 diff --git a/scripts/data_platform/data_monitor.py b/scripts/data_platform/data_monitor.py index 56a2e974..192ee9ca 100644 --- a/scripts/data_platform/data_monitor.py +++ b/scripts/data_platform/data_monitor.py @@ -239,7 +239,7 @@ def collect_log_layer(data_root, now, window_h=_LOG_WINDOW_H): keep = [p for p in logs if (now - dt.datetime.fromtimestamp(os.path.getmtime(p)) ).total_seconds() / 3600 <= window_h] - stats, sigs = {}, [] + stats, sigs_by_type = {}, {} for p in keep: try: with open(p, encoding="utf-8", errors="replace") as f: @@ -250,18 +250,28 @@ def collect_log_layer(data_root, now, window_h=_LOG_WINDOW_H): prev = stats.get(t) stats[t] = s if prev is None or s.get("rows", 0) >= prev.get("rows", 0) \ else prev - sigs.extend(chk.scan_signatures(text)) - return stats, sigs + # 10-05: 签名按 [type] 归因(跨类型连坐修), 跨文件按 sig 去重合并 + for t, ss in chk.signatures_by_type(text).items(): + bucket = sigs_by_type.setdefault(t, []) + have = {x["sig"] for x in bucket} + for s in ss: + if s["sig"] not in have: + bucket.append(s) + have.add(s["sig"]) + return stats, sigs_by_type -def log_layer_results(stats, sigs, today): - """统计行 failed>0 → 按 sig 分级告警; 断路器独立键; 其余 ok → passed.""" +def log_layer_results(stats, sigs_by_type, today): + """统计行 failed>0 → 按「该类型自己的」ERROR 行签名分级(10-05 归因修: + 他类型反爬签名不再抬本类型 failed); 断路器独立键(全局); 其余 ok → passed.""" results, passed = [], [] - worst = max((s["severity"] for s in sigs), default=None) - breaker = any(s["sig"] == "breaker" for s in sigs) + breaker = any(s["sig"] == "breaker" + for ss in sigs_by_type.values() for s in ss) for t, s in sorted(stats.items()): if s.get("failed", 0) > 0: - ev = {"stats": s, "signatures": sigs} + tsigs = sigs_by_type.get(t, []) + worst = max((x["severity"] for x in tsigs), default=None) + ev = {"stats": s, "signatures": tsigs} sev = "red" if (worst == "red" or breaker) else "yellow" results.append(_event(f"data-{t}-failed", sev, f"{t} 类型级 failed={s['failed']}", @@ -271,19 +281,22 @@ def log_layer_results(stats, sigs, today): if breaker: results.append(_event("data-akshare-breaker", "red", "断路器触发(熔断连累: 后续类型整批跳过)", - "exit code 2 会话", {"signatures": sigs}, today=today)) + "exit code 2 会话", {"signatures": sigs_by_type}, + today=today)) else: passed.append("data-akshare-breaker") return results, passed -def log_only_results(entries, stats, sigs, today): - """valuation 等 log_only 项: 统计行存在且 failed=0(周班项平日无行属正常).""" +def log_only_results(entries, stats, sigs_by_type, today): + """valuation 等 log_only 项: 统计行存在且 failed=0(周班项平日无行属正常). + 10-05: red 判定用该类型自己的签名(归因修).""" results, passed = [], [] - red_sig = any(s["severity"] == "red" for s in sigs) for e in entries: key = f"data-{e['name']}-stat" s = stats.get(e["name"]) + red_sig = any(x["severity"] == "red" + for x in sigs_by_type.get(e["name"], [])) if s is None: if e["log_cadence"] == "weekly": continue @@ -464,11 +477,11 @@ def main(argv=None): source="infra" if r["key"].startswith("infra-") else "data", today=today)) - stats, sigs = collect_log_layer(args.data_root, now) - log_events, log_passed = log_layer_results(stats, sigs, today) + stats, sigs_by_type = collect_log_layer(args.data_root, now) + log_events, log_passed = log_layer_results(stats, sigs_by_type, today) lo_events, lo_passed = log_only_results( [e for e in reg.REGISTRY if e["kind"] == "log_only" - and e.get("enabled", True)], stats, sigs, today) + and e.get("enabled", True)], stats, sigs_by_type, today) events += log_events + lo_events passed += log_passed + lo_passed diff --git a/scripts/data_platform/monitor_checks.py b/scripts/data_platform/monitor_checks.py index d29a0631..26299c9c 100644 --- a/scripts/data_platform/monitor_checks.py +++ b/scripts/data_platform/monitor_checks.py @@ -165,7 +165,7 @@ SIGNATURES = [ ("断路器触发", "breaker", "red"), ("RemoteDisconnected", "disconnect", "red"), ("Expecting value", "source_changed", "red"), - ("418", "anti_crawl", "red"), + ("418 Client Error", "anti_crawl", "red"), ("'NoneType' object is not subscriptable", "source_empty", "yellow"), ("KeyError", "field_changed", "red"), ] @@ -182,15 +182,44 @@ def parse_stat_lines(text): return out +def _error_lines(text): + """只留 ERROR 行(重试 WARNING 良性) + [FATAL] 断路器行。""" + return [ln for ln in text.splitlines() + if " ERROR " in ln or "[FATAL]" in ln] + + def scan_signatures(text): + """10-05 三修: ①只扫 ERROR/[FATAL] 行——'Expecting value' 出现在重试 + WARNING 行(financial_abstract/000963 实证), 良性重试不再误中 + source_changed; ②'418' 针收紧为 '418 Client Error'——裸 418 子串误中 + rows=41823/ok=19418 进度行(anti_crawl 假阳性实证)。""" + lines = _error_lines(text) seen, out = set(), [] for needle, sig, severity in SIGNATURES: - if needle in text and sig not in seen: + if any(needle in ln for ln in lines) and sig not in seen: seen.add(sig) out.append({"sig": sig, "severity": severity}) return out +_TYPE_TAG = re.compile(r"\[(\w+)\]") + + +def signatures_by_type(text): + """签名按日志行 [type] 归因(10-05): 'ERROR [share_capital] ...' 行的 + 签名只归 share_capital——防跨类型连坐(xueqiu 的 418 把 share_capital + 的 failed 抬红实证)。断路器行归触发类型, data_monitor 侧全局取。""" + out = {} + for ln in _error_lines(text): + m = _TYPE_TAG.search(ln) + t = m.group(1) if m else "_global" + sigs = out.setdefault(t, []) + for needle, sig, severity in SIGNATURES: + if needle in ln and sig not in {s["sig"] for s in sigs}: + sigs.append({"sig": sig, "severity": severity}) + return out + + # ---------- 调度层: schtasks /query /fo LIST /v 解析(双语字段名) ---------- _FIELD_PATTERNS = { diff --git a/tests/data_platform/test_data_monitor.py b/tests/data_platform/test_data_monitor.py index 52068b56..4b9e77c3 100644 --- a/tests/data_platform/test_data_monitor.py +++ b/tests/data_platform/test_data_monitor.py @@ -182,6 +182,7 @@ def test_health_splits_checks_and_alerts_overall(tree, tmp_path): (tree / "static" / "logs" / "akshare_static_20260925_235000.log").write_text( "[zt_pool] 完成, 耗时 1.0 秒, 统计: " '{"ok": 0, "skipped": 0, "empty": 0, "failed": 1, "rows": 0}\n' + "2026-09-25 23:50:00,000 ERROR [zt_pool] fetch: " "http.client.RemoteDisconnected: Remote end closed\n", encoding="utf-8") assert _run(tree, db) == 3 health = json.loads((tree / "monitor" / "health_latest.json").read_text()) @@ -195,6 +196,7 @@ def test_failed_stat_with_signature_alerts_red(tree, tmp_path): (tree / "static" / "logs" / "akshare_static_20260925_220000.log").write_text( "[zt_pool] 完成, 耗时 1.0 秒, 统计: " '{"ok": 0, "skipped": 0, "empty": 0, "failed": 1, "rows": 0}\n' + "2026-09-25 22:00:00,000 ERROR [zt_pool] fetch: " "http.client.RemoteDisconnected: Remote end closed\n", encoding="utf-8") assert _run(tree, db) == 3 conn = store.connect(str(db)) diff --git a/tests/data_platform/test_monitor_checks.py b/tests/data_platform/test_monitor_checks.py index 575b77a9..534490ea 100644 --- a/tests/data_platform/test_monitor_checks.py +++ b/tests/data_platform/test_monitor_checks.py @@ -90,8 +90,8 @@ class TestLogParse: assert got["xueqiu_hot"]["rows"] == 5633 and got["xueqiu_hot"]["failed"] == 0 def test_scan_signatures_triage(self): - text = ("Traceback ... http.client.RemoteDisconnected: Remote end closed\n" - "another: 'NoneType' object is not subscriptable\n" + text = ("2026-10-04 11:00:00,000 ERROR [t] fetch: http.client.RemoteDisconnected: Remote end closed\n" + "2026-10-04 11:00:01,000 ERROR [t] parse: 'NoneType' object is not subscriptable\n" "[t] [FATAL] 断路器触发: 连续 30 个 unit 失败") sigs = {(d["sig"], d["severity"]) for d in mc.scan_signatures(text)} assert ("disconnect", "red") in sigs @@ -101,6 +101,33 @@ class TestLogParse: def test_scan_signatures_clean_text(self): assert mc.scan_signatures("all good") == [] + def test_warning_retry_not_a_signature(self): + """10-05: 重试 WARNING 'Expecting value' 良性(重试耗尽才 ERROR), + 不再误中 source_changed。""" + text = ("2026-10-04 11:00:00,000 WARNING [financial_abstract] " + "financial_abstract/000963 异常重试 1/3: Expecting value: line 1 column 1") + assert mc.scan_signatures(text) == [] + + def test_progress_numbers_not_anti_crawl(self): + """10-05: 裸 '418' 子串误中 rows=41823/ok=19418 进度行(假阳性实证), + 针收紧为 '418 Client Error'。""" + text = ("2026-10-04 11:00:00,000 INFO [top_holders] " + "进度 21200/104480 ok=19418 rows=41823") + assert mc.scan_signatures(text) == [] + hit = ("2026-10-04 11:00:00,000 ERROR [xueqiu_hot] xueqiu: " + "HTTPError: 418 Client Error for url: https://xueqiu.com") + assert {d["sig"] for d in mc.scan_signatures(hit)} == {"anti_crawl"} + + def test_signatures_by_type_attribution(self): + """10-05: 签名按日志行 [type] 归因——xueqiu 的 418 不再把 + share_capital 的 failed 抬红(跨类型连坐修)。""" + text = ("2026-10-04 11:00:00,000 ERROR [xueqiu_hot] HTTPError: 418 Client Error for url: x\n" + "2026-10-04 11:00:02,000 ERROR [share_capital] sc/920298 " + "字段形状类失败: KeyError: '公告日期'") + by_type = mc.signatures_by_type(text) + assert {d["sig"] for d in by_type["xueqiu_hot"]} == {"anti_crawl"} + assert {d["sig"] for d in by_type["share_capital"]} == {"field_changed"} + class TestSchtaskParse: def test_bilingual_and_na(self):