From 95f1a458c87fefbe5f289b4618d4081eb65a8938 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 7 Oct 2026 11:42:44 +0800 Subject: [PATCH] =?UTF-8?q?fix(nas):=20P3-23=20=E5=85=A8=20lane=20rc=20?= =?UTF-8?q?=E5=88=86=E6=A1=A3=E6=89=93=E6=A0=87+NAS=20=E7=9B=91=E6=8E=A7?= =?UTF-8?q?=20P3=20=E9=95=BF=E5=B0=BE=E4=BA=94=E4=BB=B6(10-07=20=E5=BC=80?= =?UTF-8?q?=E5=B7=A5=E6=89=B9)=E2=80=94=E2=80=94run=5Fcorpus=5Fstandalone.?= =?UTF-8?q?sh=20=E6=AF=8F=20lane=20exit=20=E8=A1=8C=E5=90=8E=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=20[verdict]=20lane=3DX=20rc=3DN=20outcome=3D=E8=AF=AD?= =?UTF-8?q?=E4=B9=89=E8=AF=8D(0=3Dok/125=3Ddocker=5Fdaemon=5Fdown/137=3Dsi?= =?UTF-8?q?gkilled/daily=201=3Dnoise=5Ffailed=203=3Dwallclock=5Fstop/funne?= =?UTF-8?q?l=201=3Dfail=5Floud=203=3Dcheckpoint/gate=201=3Dcrit=202=3Dwarn?= =?UTF-8?q?;=2010-07=20=E5=81=9C=E7=94=B5=E5=A4=9C=20daily137+=E4=B8=8B?= =?UTF-8?q?=E6=B8=B8125=20=E4=BA=BA=E8=82=89=E5=88=86=E8=AF=8A=E6=95=99?= =?UTF-8?q?=E8=AE=AD)/nas=5Fmonitor=5Fcheck:=20=5Flog=5Ftail=5Flines=2064K?= =?UTF-8?q?iB=20=E5=B0=BE=E8=AF=BB=E6=BB=91=E7=AA=97(cron.log=20=E5=85=8D?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E8=AF=BB)+vps-mirror=2017:45=20=E8=BE=B9?= =?UTF-8?q?=E8=A7=92(backfill=20=E6=97=A5=20DONE=20=E8=90=BD=E7=8F=AD?= =?UTF-8?q?=E5=B0=BE~17:34+,=E6=97=A7>=3D17=20=E5=8D=B3=E5=88=87=E5=BD=93?= =?UTF-8?q?=E6=97=A5=E5=81=87=E9=BB=84)+main=20=E9=A1=B6=E5=B1=82=E5=85=9C?= =?UTF-8?q?=E5=BA=95=E5=AF=B9=E9=BD=90=20VPS=20D-2(red=20infra-nas-monitor?= =?UTF-8?q?-crash=20=E4=BA=8B=E4=BB=B6=E4=BB=B6=E5=B8=A6=20generated=5Fat?= =?UTF-8?q?=3D=E9=9D=99=E9=BB=98=E6=A3=80=E6=B5=8B=E9=94=9A,=E5=81=A5?= =?UTF-8?q?=E5=BA=B7=E7=8F=AD=20green=5Fkeys=20=E6=81=92=E5=B8=A6=E5=B4=A9?= =?UTF-8?q?=E6=BA=83=E9=94=AE=E9=98=B2=20P3-1=20=E5=BC=8F=E6=8C=82?= =?UTF-8?q?=E8=B4=A6)/runner:=20accept-new=20TOFU+=E6=9C=AC=E4=BD=8D=20kno?= =?UTF-8?q?wn=5Fhosts(=E6=9B=BF=20=3Dno=20=E8=A3=B8=E5=A5=94)+KEY/VPS=20en?= =?UTF-8?q?v=20=E5=8F=AF=E8=A6=86=E7=9B=96+out=2014=20=E5=A4=A9=E4=BF=9D?= =?UTF-8?q?=E6=B4=81;=20runbook=20P3-23=20=E6=B3=A8=E8=AE=B0(=E5=9F=BA?= =?UTF-8?q?=E5=BB=BA=E6=A1=A3=E9=97=B8)=20[nas]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Code --- docs/deployment/vps-production-runbook.md | 2 +- scripts/nas_sync/nas_monitor_check.py | 73 ++++++++++++++---- scripts/nas_sync/run_corpus_standalone.sh | 39 ++++++++++ scripts/nas_sync/run_monitor_nas.sh | 15 +++- tests/data_platform/test_nas_monitor_check.py | 77 +++++++++++++++++++ 5 files changed, 189 insertions(+), 17 deletions(-) diff --git a/docs/deployment/vps-production-runbook.md b/docs/deployment/vps-production-runbook.md index 9bd68fd6..51a0f69c 100644 --- a/docs/deployment/vps-production-runbook.md +++ b/docs/deployment/vps-production-runbook.md @@ -227,7 +227,7 @@ ssh 49.232.102.198 "powershell -Command \"Get-Content C:\sanguo_bigqmt\qmt_relog - **ak-weekly 三新域（融合 spec §20.13，2026-09-22 挂载）**：`ak_stock_wrapper.ps1` 追加两条独立命令（不带 --force，marker 幂等增量）：`--types pledge_ratio,pledge_detail --start 20210108` + `--types research_report --start 20210701`。pledge_ratio 仅枚举周五（非周五 EM result=None 必失败），未发布周五 failed 不写 marker 次周自愈；pledge_detail=RPTA_APP_ACCUMDETAILS 254 页走 fetch_em_dc_pages 节流通道（1.2s/页≈6 分钟）月桶留存；research_report=reportapi 直连周窗 anchor=周五。 - **反向镜像链（融合 spec §20.11，2026-09-17 新增）**：`push_vps_mirror_daily.sh`+`vps_mirror_lib.py`（纯逻辑，宿主 python3.8 stdlib）——corpus **五子域**（09-22 起 +flash_meta 快讯域）+dmsk 五域 NAS→VPS **推送**（09-18 起挂 `sanguo-corpus-daily` DSM 任务尾部=`run_corpus_standalone.sh` EXIT trap 完赛即推，典型 ~10:1x/backfill 日 ~17:3x；rc 隔离=corpus rc∈0/1/3 才推[09-19 首验勘误：rc() 把「墙停+噪声 failed」报 1，旧 0/3 闸门 11 天只推 1 天，仅 rc=2 限流不推]、推送失败只记日志不进聚合 rc；**admin 账号**=避开 root 属主坑；原独立 16:00 DSM 方案作废），分区粒度 journal 指纹 diff 只增不删；部署位 `/volume1/stock/vps-mirror/`（宿主文件，改 repo 副本后 scp -O 同步，同 corpus/dmsk runner 纪律），journal/watermark_*.json/push.log 同目录；VPS 落 `data\corpus`+`data\fundamentals\dmsk`+族根 `_mirror_watermark.json`（消费方 `get_watermark()` 晨检对表）；任一失败不 commit journal/不落水位线=次日 diff 自愈；`--dry-run` 打印将推送清单=验收收敛证明。设计与消费契约=融合 spec §20.11。09-20 增**独立 static manifest lane**（spec §20.11.7）：`data_backup/static_overlay/manifests/*.list`（补洗变更清单，每行 `/`）文件级幂等推送、整单成功归档 pushed/、失败只 WARN 不影响 corpus/dmsk 聚合——三表 per-stock 文件由此 lane 推 VPS（不进 journal 指纹体系）。 - **flash 快讯 lane+xcheck 对账 lane（融合 spec §20.12.3/§20.12.4，09-22 上线）**：①`run_corpus_standalone.sh` 第三 lane `--lane flash`（金十快照+东财 kuaixun 翻页 T-1 边界，秒级完赛，rc 隔离不进聚合、独立跑不 gate 于 daily/backfill）→ corpus/flash_meta/ 落盘，镜像车五子域自动携带；②`run_corpus_standalone.sh` 第四 lane `xcheck.py --lane corpus_side`（ann_em 拉取+公告披露日对照+腾讯 qt 全市场估值快照对照，rc 隔离）；③`run_dmsk.sh` 第四 lane `xcheck.py --lane fund_side`（读 wash manifests 当日命中股→新浪三表近 12 期+两源 NOTICE_DATE 对照）。xcheck 域落 `corpus/xcheck/` **不推 VPS**（NAS 侧质检设施）；findings 报告+trace 追溯=融合 spec §20.12.4；容器名 sanguo-xcheck（杀=`docker stop sanguo-xcheck`）。首跑锚点：corpus cron.log `flash lane exit=0(隔离)` 行 + `xcheck corpus_side exit=0` 行 + dmsk cron.log `xcheck fund_side exit=0` 行 + 次日 push.log 出现 flash_meta 分区计数。 -- **funnel 漏斗 lane+integrity gate lane（融合 spec §20.16/流水线 spec §4.8 P4-3，10-02 上线）**：`run_corpus_standalone.sh` 第五/六段——⑤`corpus_funnel.py --until 23:00 --llm-cap 200 --token-budget 3000000 --no-llm`（第一层词表八类事件零成本全量→events_llm 直落；第二层 LLM 只碰词表 miss：候选入 `state/funnel_pending.jsonl` 队列、串行慢爬、token 预算断路 rc=3 次夜续——**10-05 审计 P2-16 前该断路仅文档宣称未接线，现已实接 3M=每夜费用上界**（≈整夜墙钟吞吐量级，正常夜 <0.5M 不触；backlog 夜 checkpoint 次夜续）；水位 `state/funnel_watermark.json` 只前向，历史回填 `--start` 按预算分批）⑥`integrity_gate.py --profile default`（三层门 crit exit1/warn exit2）。两段 rc 隔离不进聚合。**LLM env 三件与 P4-1 向导同款（§运维表 288 行）**：DSM 任务环境或 run_corpus.sh 头 export 后由 wrapper 条件传递进容器；缺 key 且当晚有候选 → funnel rc=1（fail loud，非故障面=未配 env）。冒烟：corpus cron.log `funnel exit=0` 行+`integrity gate exit=0` 行+`corpus/events_llm/dt=*/` 分区出现；`--no-llm` 可只验第一层。容器名 sanguo-funnel/sanguo-igate（杀=docker stop）。**10-05 用户拍板（B 案）：LLM 层停用待令——`--no-llm` 已常态接线**（bigmodel 余额耗尽 429 熔断后裁决）：词表层照跑零成本（events_llm 域 model="lexicon" 行继续落）、llm_calls=0、`funnel_pending.jsonl` 冻结只增不减（当夜新命中照常入队，LLM 不消费）；开闸=去该 flag 续消化，`--token-budget 3M` 接线保留复用即生效。 +- **funnel 漏斗 lane+integrity gate lane（融合 spec §20.16/流水线 spec §4.8 P4-3，10-02 上线）**：`run_corpus_standalone.sh` 第五/六段——⑤`corpus_funnel.py --until 23:00 --llm-cap 200 --token-budget 3000000 --no-llm`（第一层词表八类事件零成本全量→events_llm 直落；第二层 LLM 只碰词表 miss：候选入 `state/funnel_pending.jsonl` 队列、串行慢爬、token 预算断路 rc=3 次夜续——**10-05 审计 P2-16 前该断路仅文档宣称未接线，现已实接 3M=每夜费用上界**（≈整夜墙钟吞吐量级，正常夜 <0.5M 不触；backlog 夜 checkpoint 次夜续）；水位 `state/funnel_watermark.json` 只前向，历史回填 `--start` 按预算分批）⑥`integrity_gate.py --profile default`（三层门 crit exit1/warn exit2）。两段 rc 隔离不进聚合。**LLM env 三件与 P4-1 向导同款（§运维表 288 行）**：DSM 任务环境或 run_corpus.sh 头 export 后由 wrapper 条件传递进容器；缺 key 且当晚有候选 → funnel rc=1（fail loud，非故障面=未配 env）。冒烟：corpus cron.log `funnel exit=0` 行+`integrity gate exit=0` 行+`corpus/events_llm/dt=*/` 分区出现；`--no-llm` 可只验第一层。容器名 sanguo-funnel/sanguo-igate（杀=docker stop）。**10-05 用户拍板（B 案）：LLM 层停用待令——`--no-llm` 已常态接线**（bigmodel 余额耗尽 429 熔断后裁决）：词表层照跑零成本（events_llm 域 model="lexicon" 行继续落）、llm_calls=0、`funnel_pending.jsonl` 冻结只增不减（当夜新命中照常入队，LLM 不消费）；开闸=去该 flag 续消化，`--token-budget 3M` 接线保留复用即生效。**P3-23（10-07）：全 lane rc 分档打标**——每 lane exit 行后追加 `[verdict] lane=X rc=N outcome=语义词`（通用 0=ok/125=docker_daemon_down/137=sigkilled/130=interrupted/143=terminated；daily|backfill 1=noise_failed/2=rate_limited/3=wallclock_stop；funnel 1=fail_loud/3=checkpoint[锁让路/墙钟/预算次夜续]；gate 1=crit/2=warn），值班/守望 grep 语义位不再裸读 exit 码——10-07 停电夜（daily 137+下游 125）需逐条人肉分诊的教训。 - **因子月度批评·先批后评四段链（流水线 spec §4.2 决议 H+§12.4 决议 J，09-23 上线；09-27 加 stage4 归因批 10-04 首班起）**：双机同评——NAS+VPS 每月第一个周日 09:30 各跑同一批评链（①②③段），JSON 对拍件（`{host}_{asof}.json`）落各自报告目录，人工/VPS 侧 scp 汇集到 NAS 后 `python -m sanguo_factor.review_compare nas_*.json vps_*.json` 对拍（一致=落档进告警流程；分歧 exit 2=只开排查 issue 不出降权告警）。ⓐ**NAS 侧**：`run_factor_monthly.sh`（源码=repo `scripts/nas_sync/run_factor_monthly_standalone.sh`，改后 scp -O 同步生效位 `/volume1/stock/sanguo_vnpy_v2/`）——一次性容器四段串行 `monthly_batch`(注册表全量 12M 窗重评批,重活)→`monthly_review --host nas`(判定报告,JSON append-only)→`data_gap_check`→`attribution_batch`(归因批: 15 截面增量[12 源+size_log+在位者+挑战者,daily_section 幂等 merge,as_of 撞周末模块内归一最近 bar 日]→归因双跑[#69 影子 A/B 归因侧供给,挑战者落 challenger/ 子目录]→cp 双投递生效位[在位者件→admin 家 `data/attribution/` 根 glob 取最新;挑战者件带因子名;生效位目录容器挂 rw `/effective-data`,代码卷恒 :ro]；工作区 `/volume1/stock/factor_cross_section/attribution/` 留全历史)；as_of=上月末脚本头自算；rc 语义=①非零停链、②③ exit 2 是有效信号按 0 聚合、④非零=真失败(截面断供/归因挂/投递缺)进 final；容器金丝雀坑=**必须 `-e HOME=/tmp -e MPLCONFIGDIR=/tmp/mpl`**（--user 1024 下 HOME=/ → vnpy import 建不了 `/.vntrader`）。报告目录 `/volume1/stock/sanguo_vnpy_v2/reports/factor_monthly/`；日志=factor_monthly_cron.log。DSM 任务 `sanguo-factor-monthly`=**每周日 09:30 admin（UI 手建）+脚本头日号闸**——DSM 计划任务月度档只有「每月几号」没有「第一个周日」原生选项，调度只能建每周日，「是不是第一个周日」由脚本头 `dom>7 静默退出` 兜住。ⓑ**VPS 侧**：wrapper=`scripts/factor_research/factor_monthly_wrapper.ps1`（三段同链批评部分，`--host vps`，eval_db=`C:\sanguo_vnpy_v2\data\factor_eval.db`，报告/日志=`C:\sanguo_vnpy_v2\reports\factor_monthly\`；ASCII-only 纪律；**stage4 归因批不镜像**——归因双跑落 NAS 工作区+生效位，VPS 侧归因件走 infra 偏差日报链另行供给）；schtask 注册=`scripts/factor_research/register_factor_monthly_task.ps1`（**XML ScheduleByMonthDayOfWeek 原生「每月第一个周日」**，无需日号闸——zh-CN schtasks 不吃星期名组合走 ak-monthly 同款 XML 路线；SYSTEM/不限时，monthly_batch 重活首班实测耗时后再定是否拆任务错时）——vps-deploy 把两文件推上 VPS 后 ssh 跑一次注册脚本即成。首班=部署后第一个周日人工盯全程。 - **static 三表事件驱动补洗（融合 spec §20.11.7，2026-09-20 上线）**：`run_dmsk.sh` 第三 lane `static_overlay_wash.py`（dmsk daily/backfill 后 ~07:2x 起跑，--until 15:00 墙停，rc 隔离）——dmsk NOTICE_DATE 雷达命中股逐股 F10(emweb) 近 12 期 upsert 重写 `static/{income,balance,cashflow}/` per-stock 文件；pending 重试队列（F10 落库滞后/网络失败次日自愈，30 天老化）；周日 sync_parquet 全量回冲=雷达窗内自愈、更深归月度全量洗兜底；变更清单→晚间镜像车 manifest lane 推 VPS。 - 数据归属口径四原则与 14 条管线全景=融合 spec §20.10（数据面唯一权威查询入口）。 diff --git a/scripts/nas_sync/nas_monitor_check.py b/scripts/nas_sync/nas_monitor_check.py index 2065336f..c6bd32a9 100644 --- a/scripts/nas_sync/nas_monitor_check.py +++ b/scripts/nas_sync/nas_monitor_check.py @@ -90,15 +90,27 @@ def build_event(check_key, severity, title, detail, evidence, now): "evidence": evidence} -def _log_has_today_fail(log_path, now): +_TAIL_BYTES = 65536 # 尾读块: ≫ 200 行滑窗预算(监控行百字节级) + + +def _log_tail_lines(log_path, n=200): + """P3 长尾(cron.log 全量读): 原整个 read() 后 [-200:] 只用尾部——日志只增 + 不减, 全量读 O(文件)。64KiB 尾块读 + 滑窗语义不变。""" try: - with open(log_path, encoding="utf-8", errors="replace") as f: - text = f.read() + with open(log_path, "rb") as f: + f.seek(0, os.SEEK_END) + size = f.tell() + f.seek(max(0, size - _TAIL_BYTES)) + data = f.read() except OSError: - return False + return [] + return data.decode("utf-8", errors="replace").splitlines()[-n:] + + +def _log_has_today_fail(log_path, now): day = now.strftime("%Y-%m-%d") return any(day in line and ("FAIL" in line or "FATAL" in line) - for line in text.splitlines()[-200:]) + for line in _log_tail_lines(log_path)) # dmsk 季频锚: 各季末→法定披露截止(年报次年 04-30/半年报 08-31/三季 10-31/ @@ -177,10 +189,13 @@ def build_checks(now): add("nas-hot_rank-log", "hot_rank 日志无 FAIL", "yellow" if log_fail else "green", {"fail_today": log_fail}) - # vps-mirror 完成度: >=17:00 查当日 DONE fail=0; 班前(16:00 未到)查昨日 - # (09-26 实证: 早班查当日必空=假黄) - want_day = now.strftime("%Y-%m-%d") if now.hour >= 17 else \ - (now - dt.timedelta(days=1)).strftime("%Y-%m-%d") + # vps-mirror 完成度: 17:45 起查当日 DONE fail=0; 之前(含 16:00 班)查昨日 + # (09-26 实证: 早班查当日必空=假黄). P3 长尾(17:00-17:33 手工重跑边角): + # backfill 日 corpus 班尾 ~17:34 退出、镜像推送更晚, 旧「>=17 即切当日」 + # 把班未完判成缺 DONE 假黄 → 17:45(班尾+推送余量)起才切当日。 + want_day = now.strftime("%Y-%m-%d") \ + if (now.hour, now.minute) >= (17, 45) \ + else (now - dt.timedelta(days=1)).strftime("%Y-%m-%d") # 0929 修: push.log 里 DONE 行只带 [HH:MM:SS] 不带日期(只有 start 行 # 带日期), 旧按 want_day 过滤行永远只剩 start → 连黄(0926~28 实锤). # 改按「want_day 的 start 段内出现过 DONE fail=0」判: 遍历记当前 start @@ -235,8 +250,7 @@ def build_checks(now): return checks, events -def main(): - now = dt.datetime.now() +def _run(now): checks, events = build_checks(now) os.makedirs(OUT_DIR, exist_ok=True) day = now.strftime("%Y%m%d") @@ -256,11 +270,44 @@ def main(): "events": events, # 绿键回传: VPS 侧据此 resolve NAS 告警(09-26 补, 防永不转绿) "green_keys": [c["key"] for c in checks - if c["status"] == "green"]}, + if c["status"] == "green"] + # 健康班=崩溃兜底键绿(P3 长尾 NAS main 无兜底; P3-1 教训: + # 键只在崩溃件出现=VPS 侧永不 resolve 挂账) + + ["infra-nas-monitor-crash"]}, f, ensure_ascii=False, indent=1) print(f"[nas-monitor] overall={health['overall']} checks={len(checks)} " f"events={len(events)}") +def main(): + now = dt.datetime.now() + try: + _run(now) + except Exception: # noqa: BLE001 + # P3 长尾(NAS main 无兜底; 对齐 VPS 侧 D-2 _crash_exit): 裸崩溃时落 + # 一件 red 崩溃事件件(generated_at 同写=VPS 静默检测锚不瞎), 免等 + # 30h 静默窗才被 infra-nas-monitor-silent 兜住; rc=1 进 cron.log。 + import traceback + stamp = now.strftime("%Y%m%d_%H%M") + ev = build_event("infra-nas-monitor-crash", "red", + "NAS monitor 整班崩溃(顶层兜底)", + traceback.format_exc()[-600:], {"fallback": True}, + now) + ev["source"] = "infra" + try: + os.makedirs(OUT_DIR, exist_ok=True) + with open(os.path.join(OUT_DIR, f"nas_events_{stamp}.json"), "w", + encoding="utf-8") as f: + json.dump({"generated_at": now.isoformat(timespec="seconds"), + "events": [ev], "green_keys": []}, + f, ensure_ascii=False, indent=1) + print("[nas-monitor] CRASH fallback event written") + except OSError: + pass + return 1 + return 0 + + if __name__ == "__main__": - main() + import sys + sys.exit(main()) diff --git a/scripts/nas_sync/run_corpus_standalone.sh b/scripts/nas_sync/run_corpus_standalone.sh index f1eeac0b..b7aaad8c 100644 --- a/scripts/nas_sync/run_corpus_standalone.sh +++ b/scripts/nas_sync/run_corpus_standalone.sh @@ -43,6 +43,39 @@ UID_ADMIN=1024 # id admin (2026-09-06 实测) GID_ADMIN=100 # users GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) +# ===== P3-23(10-07): 调用侧 rc 分档打标 ===== +# 每 lane rc 落定后追加一行 [verdict] 语义词, 值班/守望 grep 语义位不再裸读 +# exit 码(10-07 停电夜 137/125 要逐条人肉分诊的教训)。映射: +# 通用: 0=ok 125=docker_daemon_down 137=sigkilled 130=interrupted 143=terminated +# daily/backfill(corpus_download): 1=noise_failed 2=rate_limited 3=wallclock_stop +# funnel(corpus_funnel): 1=fail_loud(config 缺/未捕获崩溃) 2=rate_limited +# 3=checkpoint(锁让路/墙钟/预算, 次夜续) +# gate(integrity_gate): 1=crit 2=warn +_outcome() { # $1=lane $2=rc → 语义词 + case "$2" in + 0) echo ok ;; + 125) echo docker_daemon_down ;; + 137) echo sigkilled ;; + 130) echo interrupted ;; + 143) echo terminated ;; + *) + case "$1" in + daily|backfill) + case "$2" in + 1) echo noise_failed ;; 2) echo rate_limited ;; + 3) echo wallclock_stop ;; *) echo fail ;; esac ;; + funnel) + case "$2" in + 1) echo fail_loud ;; 2) echo rate_limited ;; + 3) echo checkpoint ;; *) echo fail ;; esac ;; + gate) + case "$2" in + 1) echo crit ;; 2) echo warn ;; *) echo fail ;; esac ;; + *) echo fail ;; + esac ;; + esac +} + { echo "=== $(date '+%F %T') standalone-container corpus run start ===" "$DOCKER" run --rm --name sanguo-corpus --user "${UID_ADMIN}:${GID_ADMIN}" \ @@ -53,6 +86,7 @@ GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) /app/scripts/data_platform/corpus_download.py --lane daily --until 10:30 rc_daily=$? echo "=== $(date '+%F %T') corpus daily lane exit=$rc_daily ===" + echo "[verdict] $(date '+%F %T') lane=daily rc=$rc_daily outcome=$(_outcome daily "$rc_daily")" rc_back=0 # 09-19 首验勘误: rc=1 闸门收窄到 0|3 在实盘=永不续走——corpus_download.py rc() # 优先级 failed>0(哪怕 3~113/13000 噪声级)>墙钟, 实证 11 个 daily 腿 10 天 rc=1, @@ -67,6 +101,7 @@ GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) /app/scripts/data_platform/corpus_download.py --lane backfill --until 17:30 rc_back=$? echo "=== $(date '+%F %T') corpus backfill lane exit=$rc_back ===" + echo "[verdict] $(date '+%F %T') lane=backfill rc=$rc_back outcome=$(_outcome backfill "$rc_back")" else echo "=== daily 非零($rc_daily), 跳过 backfill lane(次日自愈) ===" fi @@ -81,6 +116,7 @@ GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) /app/scripts/data_platform/corpus_download.py --lane flash rc_flash=$? echo "=== $(date '+%F %T') corpus flash lane exit=$rc_flash(隔离,不进聚合) ===" + echo "[verdict] $(date '+%F %T') lane=flash rc=$rc_flash outcome=$(_outcome flash "$rc_flash")" # ===== xcheck corpus_side lane(spec §20.12.4, 2026-09-22: 跨 vendor 对账) ===== # ann_em 拉取+公告披露日对照(ann_meta vs np-anotice) + 腾讯 qt 全市场估值 # 快照落盘+peTTM/pbMRQ 对照(vs valuation_baostock); 独立容器; rc 隔离 @@ -93,6 +129,7 @@ GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) /app/scripts/data_platform/xcheck.py --lane corpus_side rc_xc=$? echo "=== $(date '+%F %T') xcheck corpus_side exit=$rc_xc(隔离,不进聚合) ===" + echo "[verdict] $(date '+%F %T') lane=xcheck rc=$rc_xc outcome=$(_outcome xcheck "$rc_xc")" # ===== funnel lane(spec §4.8 P4-3, 2026-10-02: 两层漏斗+事件快照) ===== # 第一层词表零成本全量; 第二层 LLM 只碰词表 miss(串行慢爬+token 预算断路 # rc=3 checkpoint 次夜续)。10-05 P2-16 落地 --token-budget 3M=每夜费用 @@ -116,6 +153,7 @@ GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) /app/scripts/data_platform/corpus_funnel.py --until 23:00 --llm-cap 200 --token-budget 3000000 --no-llm rc_funnel=$? echo "=== $(date '+%F %T') funnel exit=$rc_funnel(隔离,不进聚合) ===" + echo "[verdict] $(date '+%F %T') lane=funnel rc=$rc_funnel outcome=$(_outcome funnel "$rc_funnel")" # ===== integrity gate(P4-3 前置纪律②, spec §20.16; rc 隔离) ===== # crit>0 exit 1 / warn exit 2 / 绿 exit 0; 报告 stdout 进 cron.log。 "$DOCKER" run --rm --name sanguo-igate --user "${UID_ADMIN}:${GID_ADMIN}" \ @@ -127,6 +165,7 @@ GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带) --profile default rc_gate=$? echo "=== $(date '+%F %T') integrity gate exit=$rc_gate(隔离,不进聚合) ===" + echo "[verdict] $(date '+%F %T') lane=gate rc=$rc_gate outcome=$(_outcome gate "$rc_gate")" } >> "$LOG" 2>&1 # ===== hot_rank NAS 采集+推 VPS(spec §20.14, 09-25: emappdata 云 IP 断连迁移) ===== # 独立 runner(采集容器+scp 推送+补推未推日期), 自身恒 exit 0 隔离, 失败只记 diff --git a/scripts/nas_sync/run_monitor_nas.sh b/scripts/nas_sync/run_monitor_nas.sh index fbb63713..96ce8004 100755 --- a/scripts/nas_sync/run_monitor_nas.sh +++ b/scripts/nas_sync/run_monitor_nas.sh @@ -10,12 +10,16 @@ # 部署位: /volume1/stock/monitor_nas/(宿主文件, CI 不覆盖; 改 repo 副本后 # scp -O 手动同步——同 corpus/dmsk/vps-mirror runner 纪律). set -u -KEY=/var/services/homes/admin/.ssh/id_ed25519_nas -VPS=Administrator@49.232.102.198 +# P3 长尾(硬编码 IP/私钥): env 可覆盖, 缺省=现网值(行为零变化)。 +KEY=${SANGUO_MONITOR_SSH_KEY:-/var/services/homes/admin/.ssh/id_ed25519_nas} +VPS=${SANGUO_MONITOR_VPS:-Administrator@49.232.102.198} VPS_INBOX=C:/sanguo_vnpy_v2/data/monitor/inbox ROOT=/volume1/stock/monitor_nas PY=/usr/bin/python3 -SCP_OPTS=(-i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=20) +# P3 长尾(StrictHostKeyChecking=no 全裸奔): accept-new=TOFU——首连即录 +# $ROOT/known_hosts, 之后主机键变更即拒(MITM 可见); 零部署摩擦(首跑自动建档)。 +SCP_OPTS=(-i "$KEY" -o StrictHostKeyChecking=accept-new + -o UserKnownHostsFile="$ROOT/known_hosts" -o ConnectTimeout=20) mkdir -p "$ROOT/out" { @@ -42,6 +46,11 @@ mkdir -p "$ROOT/out" echo "=== $(date '+%T') push FAIL: $base(下次班重试) ===" fi done + # out/ 保洁(P3 长尾: 产物无清理只增不减): health/events/pushed marker + # 只留 14 天(VPS migration_logs 侧已在 b954b02 保洁, 本处=NAS 半边)。 + find "$ROOT/out" -maxdepth 1 -type f \( -name 'nas_health_*.json' \ + -o -name 'nas_events_*.json' -o -name '.pushed_*' \) -mtime +14 -delete \ + 2>/dev/null || true } >> "$ROOT/cron.log" 2>&1 exit 0 diff --git a/tests/data_platform/test_nas_monitor_check.py b/tests/data_platform/test_nas_monitor_check.py index c2fd84df..293eb624 100644 --- a/tests/data_platform/test_nas_monitor_check.py +++ b/tests/data_platform/test_nas_monitor_check.py @@ -1,5 +1,6 @@ """NAS 监控 v2 纯函数: last_weekday/mtime 年龄/build_events(告警 schema 对齐 §20.15.4).""" import datetime as dt +import json import os import sys @@ -150,3 +151,79 @@ def test_mirror_start_line_without_date_resets_cur_day(tmp_path, monkeypatch): checks, _ = nmc.build_checks(dt.datetime(2026, 9, 28, 17, 35)) mirror = [c for c in checks if c["key"] == "nas-vps-mirror"][0] assert mirror["status"] == "yellow" + + +# ---------- 10-07 P3 长尾批(NAS 群): 尾读滑窗/17:45 边角/崩溃兜底 ---------- + +def test_log_tail_lines_window_and_sliding(tmp_path): + """P3 长尾(cron.log 全量读): 64KiB 尾读语义=原 [-200:] 滑窗不变.""" + log = tmp_path / "cron.log" + log.write_text("\n".join(f"line-{i}" for i in range(300)) + "\n", + encoding="utf-8") + got = nmc._log_tail_lines(str(log)) + assert len(got) == 200 and got[0] == "line-100" and got[-1] == "line-299" + small = tmp_path / "s.log" + small.write_text("a\nb\n", encoding="utf-8") + assert nmc._log_tail_lines(str(small)) == ["a", "b"] + assert nmc._log_tail_lines(str(tmp_path / "none.log")) == [] + + +def test_today_fail_beyond_tail_window_ignored(tmp_path): + filler = [f"=== 2026-09-01 06:00:{i:02d} ok ===" for i in range(250)] + log = tmp_path / "cron.log" + # FAIL 在头部, 250 行垫后 → 滑窗外 + log.write_text("\n".join(["=== 2026-09-25 11:00:00 push FAIL ==="] + filler), + encoding="utf-8") + assert nmc._log_has_today_fail(str(log), dt.datetime(2026, 9, 25, 16)) is False + log.write_text("\n".join(filler[:100] + + ["=== 2026-09-25 11:00:00 push FAIL ==="]), + encoding="utf-8") + assert nmc._log_has_today_fail(str(log), dt.datetime(2026, 9, 25, 16)) is True + + +def test_mirror_1745_boundary_manual_rerun_window(tmp_path, monkeypatch): + """P3 长尾(17:00-17:33 手工重跑边角): backfill 日镜像 DONE 落在 corpus 班尾 + ~17:34+推送更晚, 旧「>=17 即切当日」把班未完判缺 DONE 假黄 → 17:45 前查 + 昨日(DONE 在)=绿, 17:45 起才切当日(start 无 DONE)=黄.""" + monkeypatch.setattr(nmc, "MIRROR_DIR", str(tmp_path)) + (tmp_path / "push.log").write_text( + "=== 2026-10-06 17:32:31 vps-mirror start (dry=0) ===\n" + "[17:34:00] vps-mirror DONE pushed=4 fail=0\n" + "=== 2026-10-07 17:05:00 vps-mirror start (dry=0) ===\n", + encoding="utf-8") + checks, _ = nmc.build_checks(dt.datetime(2026, 10, 7, 17, 10)) + mirror = [c for c in checks if c["key"] == "nas-vps-mirror"][0] + assert mirror["status"] == "green" + checks, _ = nmc.build_checks(dt.datetime(2026, 10, 7, 17, 46)) + mirror = [c for c in checks if c["key"] == "nas-vps-mirror"][0] + assert mirror["status"] == "yellow" + + +def test_crash_fallback_writes_red_event_and_exits_1(tmp_path, monkeypatch): + """P3 长尾(NAS main 无兜底; 对齐 VPS D-2): 裸崩溃落 red 崩溃件 + (generated_at 同写=静默检测锚), rc=1 进 cron.log.""" + monkeypatch.setattr(nmc, "OUT_DIR", str(tmp_path)) + + def boom(now): + raise RuntimeError("整班崩") + monkeypatch.setattr(nmc, "build_checks", boom) + assert nmc.main() == 1 + files = list(tmp_path.glob("nas_events_*.json")) + assert len(files) == 1 + payload = json.loads(files[0].read_text(encoding="utf-8")) + assert payload["generated_at"] and payload["green_keys"] == [] + ev = payload["events"][0] + assert ev["check_key"] == "infra-nas-monitor-crash" + assert ev["severity"] == "red" and ev["source"] == "infra" + assert "RuntimeError" in ev["detail"] + + +def test_main_healthy_run_resolves_crash_key(tmp_path, monkeypatch): + """健康班 green_keys 恒带崩溃兜底键(P3-1 教训: 键只在崩溃件出现=VPS 侧 + 永不 resolve 挂账).""" + monkeypatch.setattr(nmc, "OUT_DIR", str(tmp_path)) + monkeypatch.setattr(nmc, "build_checks", lambda now: ([], [])) + assert nmc.main() == 0 + payload = json.loads( + next(tmp_path.glob("nas_events_*.json")).read_text(encoding="utf-8")) + assert "infra-nas-monitor-crash" in payload["green_keys"]