267 lines
11 KiB
Python
267 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
# nas_monitor_check.py — NAS 数据面监控 v2(spec §20.15 NAS 侧)
|
|
#
|
|
# 宿主 /usr/bin/python3.8 纯 stdlib(同 vps-mirror 家族, 不进容器: 只读体检).
|
|
# 由 run_monitor_nas.sh 挂 sanguo-vps-mirror 16:00 班尾调用:
|
|
# corpus 06:00(当班)/dmsk 07:10(当班)/hot_rank 昨 17:35/03:00 sync 链均已收尾.
|
|
# 产物 → /volume1/stock/monitor_nas/out/, events 由 runner scp 推 VPS inbox,
|
|
# VPS data_monitor 班次摄取落 alerts.db(单写者, NAS 不写远端 sqlite).
|
|
# 零事件也写 events 文件(带 generated_at, VPS 侧做 NAS 静默检测).
|
|
import datetime as dt
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
|
|
CORPUS_ROOT = "/volume1/stock/corpus"
|
|
DMSK_ROOT = "/volume1/stock/fundamentals/dmsk"
|
|
HOTRANK_DIR = "/volume1/stock/hot_rank_nas"
|
|
MIRROR_DIR = "/volume1/stock/vps-mirror"
|
|
XCHECK_REPORTS = os.path.join(CORPUS_ROOT, "xcheck", "reports")
|
|
OUT_DIR = "/volume1/stock/monitor_nas/out"
|
|
DISK_WARN_GB = 100 # 初值, 首周观察校准(spec §20.15.1 磁盘水位 NAS 侧)
|
|
|
|
# 沪深北休市表镜像(N-2 10-05)——=monitor_registry.EXCHANGE_HOLIDAYS, CI 同步测
|
|
# (test_holiday_table_in_sync_with_vps_registry)钉两表相等, 单边改表即红(同
|
|
# 7f9b413 strategy 侧镜像纪律); 年更义务经 VPS 侧年更闸+同步测传导到本表。
|
|
# 旧 last_weekday 只剥周末: 0925 中秋/国庆连休窗把假日当「昨交易日」——当前被
|
|
# hot_rank producer 的日历日节奏掩住(files 假期照落), 语义仍错。表缺年=不扣
|
|
# (fail-safe, 黄照旧绝不静默转绿)。
|
|
EXCHANGE_HOLIDAYS = {
|
|
2026: {
|
|
"20260101", "20260102", # 元旦
|
|
"20260216", "20260217", "20260218", "20260219", "20260220", "20260223", # 春节
|
|
"20260406", # 清明
|
|
"20260501", "20260504", "20260505", # 劳动节
|
|
"20260619", # 端午
|
|
"20260925", # 中秋
|
|
"20261001", "20261002", "20261005", "20261006", "20261007", # 国庆(1008 复市)
|
|
},
|
|
}
|
|
|
|
|
|
def last_weekday(before):
|
|
"""strictly before 且为交易日(周一~五且非休市表日)的最近一天(YYYYMMDD)
|
|
——16:00 查时当日 17:35 班未采; N-2: 接休市表镜像, 跳过连休假日."""
|
|
d = before - dt.timedelta(days=1)
|
|
while True:
|
|
hol = EXCHANGE_HOLIDAYS.get(d.year)
|
|
if d.weekday() < 5 and (hol is None or d.strftime("%Y%m%d") not in hol):
|
|
return d.strftime("%Y%m%d")
|
|
d -= dt.timedelta(days=1)
|
|
|
|
|
|
def mtime_age_hours(path, now):
|
|
try:
|
|
return (now - dt.datetime.fromtimestamp(
|
|
os.path.getmtime(path))).total_seconds() / 3600
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def dir_latest_mtime(path):
|
|
latest = 0.0
|
|
for root, _dirs, files in os.walk(path):
|
|
for fn in files:
|
|
try:
|
|
latest = max(latest, os.path.getmtime(os.path.join(root, fn)))
|
|
except OSError:
|
|
pass
|
|
return latest
|
|
|
|
|
|
def dir_age_hours(path, now):
|
|
if not os.path.isdir(path):
|
|
return None
|
|
return (now - dt.datetime.fromtimestamp(
|
|
dir_latest_mtime(path))).total_seconds() / 3600
|
|
|
|
|
|
def freshness(age_h, limit_h):
|
|
if age_h is None:
|
|
return "yellow"
|
|
return "green" if age_h <= limit_h else "yellow"
|
|
|
|
|
|
def build_event(check_key, severity, title, detail, evidence, now):
|
|
return {"alert_id": f"{check_key}-{now.strftime('%Y%m%d')}",
|
|
"check_key": check_key, "source": "data", "host": "nas",
|
|
"severity": severity, "title": title, "detail": detail,
|
|
"evidence": evidence}
|
|
|
|
|
|
def _log_has_today_fail(log_path, now):
|
|
try:
|
|
with open(log_path, encoding="utf-8", errors="replace") as f:
|
|
text = f.read()
|
|
except OSError:
|
|
return False
|
|
day = now.strftime("%Y-%m-%d")
|
|
return any(day in line and ("FAIL" in line or "FATAL" in line)
|
|
for line in text.splitlines()[-200:])
|
|
|
|
|
|
# dmsk 季频锚: 各季末→法定披露截止(年报次年 04-30/半年报 08-31/三季 10-31/
|
|
# 一季 04-30); 截止未过的季度只要求上一季全量在(在披露的季度水位可仍停上季末)
|
|
# (1010 二审 P2-1: 半年报法定=上半年结束后 2 个月内=08-31, 原写 09-30 向
|
|
# 宽松错——Q2 缺失最晚延迟 30 天才检出)
|
|
_QUARTER_DEADLINES = {(3, 31): (4, 30), (6, 30): (8, 31), (9, 30): (10, 31)}
|
|
|
|
|
|
def _dmsk_latest_partition(root):
|
|
"""五域(balance/income/cashflow/express/forecast)最新 dt= 分区(ISO)或 None.
|
|
|
|
只认 YYYY-MM-DD 零填充格式(1010 二审 P3-7): 裸字典序对上游改
|
|
dt=20260630 之类紧凑格式会静默错序, 非标格式一律忽略."""
|
|
iso = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
best = None
|
|
for p in glob.glob(os.path.join(root, "*", "dt=*")):
|
|
d = p.rsplit("dt=", 1)[-1]
|
|
if not iso.match(d):
|
|
continue
|
|
if best is None or d > best:
|
|
best = d
|
|
return best
|
|
|
|
|
|
def _dmsk_min_expected(now):
|
|
"""最近一个「法定披露截止已过」的季末(ISO)或 None."""
|
|
for back in range(0, 400):
|
|
d = now.date() - dt.timedelta(days=back)
|
|
key = (d.month, d.day)
|
|
if key in _QUARTER_DEADLINES:
|
|
dl_m, dl_d = _QUARTER_DEADLINES[key]
|
|
if dt.date(d.year, dl_m, dl_d) <= now.date():
|
|
return d.isoformat()
|
|
elif key == (12, 31) and dt.date(d.year + 1, 4, 30) <= now.date():
|
|
return d.isoformat()
|
|
return None
|
|
|
|
|
|
def build_checks(now):
|
|
"""返回 (checks 列表[health 用], events 列表[非绿项转告警])."""
|
|
checks = []
|
|
|
|
def add(key, label, status, evidence):
|
|
checks.append({"key": key, "label": label, "status": status,
|
|
"evidence": evidence})
|
|
|
|
# corpus 双 lane(news/ann 分区新鲜度; 06:00 班, 次日 16:00 查=34h 上限)
|
|
for lane in ("news_meta", "ann_meta"):
|
|
age = dir_age_hours(os.path.join(CORPUS_ROOT, lane), now)
|
|
add(f"nas-corpus_{lane}", f"corpus {lane} 新鲜",
|
|
freshness(age, 34), {"age_h": round(age, 1) if age is not None else None})
|
|
|
|
# dmsk 三表 dt 分区(07:10 班)——季频披露驱动域: 无新披露不写盘, mtime 无
|
|
# 结构意义(09-27 实证 Q2 报告期水位 06-30 停滞, 80h mtime 必假黄). 0928
|
|
# 改季频锚: 最新分区 dt ≥ 最近已过披露截止的季末; 五域取 max(三表法定
|
|
# 必披露, express/forecast 条件披露只助推 latest 不抬门槛)
|
|
latest_dt = _dmsk_latest_partition(DMSK_ROOT)
|
|
min_exp = _dmsk_min_expected(now)
|
|
add("nas-dmsk", "dmsk 三表季频水位",
|
|
"green" if (latest_dt and min_exp and latest_dt >= min_exp) else "yellow",
|
|
{"latest_dt": latest_dt, "min_expected": min_exp})
|
|
|
|
# hot_rank: 昨交易日文件 + .pushed marker + 当日 cron.log FAIL 行
|
|
want_day = last_weekday(now.date())
|
|
hr_file = os.path.join(HOTRANK_DIR, f"{want_day}_hot_rank.parquet")
|
|
pushed = os.path.join(HOTRANK_DIR, f".pushed_{want_day}")
|
|
hr_ok = os.path.exists(hr_file)
|
|
push_ok = os.path.exists(pushed)
|
|
add("nas-hot_rank-file", "hot_rank 昨交易日文件",
|
|
"green" if hr_ok else "red", {"expected": want_day, "exists": hr_ok})
|
|
add("nas-hot_rank-pushed", "hot_rank 已推 VPS",
|
|
"green" if push_ok else "yellow",
|
|
{"marker": f".pushed_{want_day}", "exists": push_ok})
|
|
log_fail = _log_has_today_fail(os.path.join(HOTRANK_DIR, "cron.log"), now)
|
|
add("nas-hot_rank-log", "hot_rank 日志无 FAIL",
|
|
"yellow" if log_fail else "green", {"fail_today": log_fail})
|
|
|
|
# vps-mirror 完成度: >=17:00 查当日 DONE fail=0; 班前(16:00 未到)查昨日
|
|
# (09-26 实证: 早班查当日必空=假黄)
|
|
want_day = now.strftime("%Y-%m-%d") if now.hour >= 17 else \
|
|
(now - dt.timedelta(days=1)).strftime("%Y-%m-%d")
|
|
# 0929 修: push.log 里 DONE 行只带 [HH:MM:SS] 不带日期(只有 start 行
|
|
# 带日期), 旧按 want_day 过滤行永远只剩 start → 连黄(0926~28 实锤).
|
|
# 改按「want_day 的 start 段内出现过 DONE fail=0」判: 遍历记当前 start
|
|
# 日期, DONE 归属其最近一次 start.
|
|
mirror_ok = False
|
|
try:
|
|
with open(os.path.join(MIRROR_DIR, "push.log"), encoding="utf-8",
|
|
errors="replace") as fh:
|
|
cur_day = None
|
|
done_days = set()
|
|
for line in fh:
|
|
if "vps-mirror start" in line:
|
|
# start 行无日期(格式变化)→cur_day 置 None 而非沿用旧值,
|
|
# 防 DONE 错归日(1010 二审 P3-11)
|
|
cur_day = None
|
|
for part in line.split():
|
|
if len(part) == 10 and part[:2] == "20" and part[4] == "-":
|
|
cur_day = part
|
|
break
|
|
elif "vps-mirror DONE" in line and cur_day and "fail=0" in line:
|
|
done_days.add(cur_day)
|
|
mirror_ok = want_day in done_days
|
|
except OSError:
|
|
pass
|
|
add("nas-vps-mirror", "vps-mirror 当日完成", "green" if mirror_ok else "yellow",
|
|
{"done_fail0": mirror_ok})
|
|
|
|
# xcheck findings 新鲜度(dmsk 班尾跑, 分区 dt=)
|
|
xc_parts = sorted(glob.glob(os.path.join(XCHECK_REPORTS, "dt=*")))
|
|
xc_age = None
|
|
if xc_parts:
|
|
xc_age = dir_age_hours(xc_parts[-1], now)
|
|
add("nas-xcheck", "xcheck findings 新鲜", freshness(xc_age, 34),
|
|
{"age_h": round(xc_age, 1) if xc_age is not None else None,
|
|
"latest": xc_parts[-1].split("=")[-1] if xc_parts else None})
|
|
|
|
# 磁盘水位 /volume1
|
|
try:
|
|
st = os.statvfs("/volume1")
|
|
free_gb = st.f_bavail * st.f_frsize / 1024 ** 3
|
|
add("nas-disk", "/volume1 剩余",
|
|
"green" if free_gb >= DISK_WARN_GB else "yellow",
|
|
{"free_gb": round(free_gb, 1), "warn_gb": DISK_WARN_GB})
|
|
except OSError:
|
|
add("nas-disk", "/volume1 剩余", "yellow", {"error": "statvfs"})
|
|
|
|
events = [build_event(c["key"],
|
|
"red" if c["status"] == "red" else "yellow",
|
|
c["label"], json.dumps(c["evidence"], ensure_ascii=False),
|
|
c["evidence"], now)
|
|
for c in checks if c["status"] != "green"]
|
|
return checks, events
|
|
|
|
|
|
def main():
|
|
now = dt.datetime.now()
|
|
checks, events = build_checks(now)
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
day = now.strftime("%Y%m%d")
|
|
health = {"date": now.date().isoformat(), "host": "nas",
|
|
"generated_at": now.isoformat(timespec="seconds"),
|
|
"overall": "red" if any(c["status"] == "red" for c in checks)
|
|
else ("yellow" if any(c["status"] == "yellow" for c in checks)
|
|
else "green"),
|
|
"checks": checks}
|
|
with open(os.path.join(OUT_DIR, f"nas_health_{day}.json"), "w",
|
|
encoding="utf-8") as f:
|
|
json.dump(health, f, ensure_ascii=False, indent=1)
|
|
stamp = now.strftime("%Y%m%d_%H%M")
|
|
with open(os.path.join(OUT_DIR, f"nas_events_{stamp}.json"), "w",
|
|
encoding="utf-8") as f:
|
|
json.dump({"generated_at": now.isoformat(timespec="seconds"),
|
|
"events": events,
|
|
# 绿键回传: VPS 侧据此 resolve NAS 告警(09-26 补, 防永不转绿)
|
|
"green_keys": [c["key"] for c in checks
|
|
if c["status"] == "green"]},
|
|
f, ensure_ascii=False, indent=1)
|
|
print(f"[nas-monitor] overall={health['overall']} checks={len(checks)} "
|
|
f"events={len(events)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|