fix(monitor): 监控班次二 P2 批——D-5 db_sample 日历不可达去 fail-open(样本可读弱检/库不可达黄)+D-6 month_bucket 损坏件 rows=None 黄(对齐 per_date)+D-8 check_disk 阈值参数化消费注册表 warn_gb/crit_gb(旧硬编码 8/20=改表不改行为校准陷阱,crit_gb 全仓无消费)+D-9 vintage/NAS ts fromisoformat 捕 TypeError+D-10 db_sample close 进 finally+AL-1 upsert 更新 severity(同日黄→红升级即时落表,旧 UPDATE 不动=次日新 id 才纠正)+purge 收口 MON-202610-04(open 黄改 last_seen 28d=活着不删,旧 first_seen 14d 硬删无声抹持续黄+断 count 升级链;resolved 统一 30d,open 红永不删)+N-1 run_monitor_nas.sh rc 先存后拼($(date) 同词先展开重置 $?=exit 恒 0,rc 隔离下 NAS 崩溃唯一现场线索)+N-2 NAS 日历豁免短期方案(last_weekday 接休市表镜像+CI 同步测钉与 monitor_registry 相等=7f9b413 镜像纪律;10-06 旧码会把 10-05 假日当昨交易日;corpus 34h 日历日节奏实证无需豁免)+P3 归因([FATAL] 级别标记不当类型归 _global,scan_signatures 注记 test-only 勿接线)+T-2 NAS red 分支(hot_rank 文件缺)测试补缺+migration_logs 保洁(data_monitor_wrapper 双班剪 14d 前旧件,13 wrapper 无限增长收口);monitoring spec §5.1/§5.2/§5.3-④/§5.4/§5.5-④/§7.6/§12 随码(C-A1 universe 异常实串/C-A2 签名表勘正补 Expecting value/C-A6 disk 阈值表述+锚点行数刷新) [vps] [nas]
This commit is contained in:
@@ -152,7 +152,8 @@ def _check_entry_impl(e, ctx):
|
||||
return done("stale", "yellow", f"<={e['max_age_days']}d", f"{age}d")
|
||||
lo, hi = e["row_band"]
|
||||
rows = chk.parquet_rows(got[1])
|
||||
if rows is not None and not (lo <= rows <= hi):
|
||||
if rows is None or not (lo <= rows <= hi):
|
||||
# D-6(10-05): 损坏件 rows=None 不再静默绿——对齐 per_date 判定
|
||||
return done("rows", "yellow", f"{lo}~{hi}", rows)
|
||||
return done("stale", "green", ym, ym)
|
||||
|
||||
@@ -183,23 +184,33 @@ def _check_entry_impl(e, ctx):
|
||||
if kind == "db_sample":
|
||||
exp = chk.expected_daily(ctx["days"], ctx["now"], e.get("ready", "22:30"),
|
||||
e.get("lag", 0))
|
||||
worst = None
|
||||
worst, readable = None, False
|
||||
for sym, exc in e["samples"]:
|
||||
got_d = None
|
||||
try:
|
||||
conn = _ro_conn(os.path.join(ctx["data_root"], "quant_trading.db"))
|
||||
row = conn.execute(
|
||||
"SELECT MAX(datetime) FROM dbbardata WHERE symbol=? AND "
|
||||
"exchange=? AND interval='d'", (sym, exc)).fetchone()
|
||||
conn.close()
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT MAX(datetime) FROM dbbardata WHERE symbol=? AND "
|
||||
"exchange=? AND interval='d'", (sym, exc)).fetchone()
|
||||
finally:
|
||||
conn.close() # D-10(10-05): close 进 finally, 异常路径不泄漏
|
||||
got_d = (row[0] or "").replace("-", "")
|
||||
except Exception: # noqa: BLE001 只读探测, 库坏=黄红由调用方判
|
||||
got_d = None
|
||||
if got_d:
|
||||
readable = True
|
||||
if exp and (got_d is None or got_d < exp) and worst is None:
|
||||
worst = {"symbol": f"{sym}.{exc}", "expected": exp, "actual": got_d}
|
||||
if exp and worst:
|
||||
sev = "yellow" if exp not in ctx["confirmed_days"] \
|
||||
else e.get("severity", "red")
|
||||
return done("stale", sev, exp, worst["actual"], worst)
|
||||
if exp is None:
|
||||
# D-5(10-05): 日历不可达不再无条件绿(fail-open)——弱检查=样本可读
|
||||
# (对齐 per_date 存在性弱检), 库也不可达=黄
|
||||
return done("stale", "green" if readable else "yellow", "n/a",
|
||||
"sample-readable" if readable else "db-unreadable")
|
||||
return done("stale", "green", exp, "ok")
|
||||
|
||||
if kind == "vintage_json":
|
||||
@@ -213,8 +224,8 @@ def _check_entry_impl(e, ctx):
|
||||
try:
|
||||
stale = not checked or (ctx["now"] - dt.datetime.fromisoformat(
|
||||
checked)).total_seconds() > 26 * 3600
|
||||
except ValueError:
|
||||
stale = True
|
||||
except (ValueError, TypeError):
|
||||
stale = True # D-9(10-05): 非字符串 fromisoformat 抛 TypeError 不炸
|
||||
if stale:
|
||||
return done("verdict", "yellow", "<=26h", checked)
|
||||
# 结论无顶层字段(static_vintage_check 契约): 挖洞藏在 tables.*.holes_*
|
||||
@@ -237,7 +248,10 @@ def _check_entry_impl(e, ctx):
|
||||
"exists", exists)
|
||||
|
||||
if kind == "disk":
|
||||
r = chk.check_disk(ctx["data_root"])
|
||||
# D-8(10-05): 阈值消费注册表(旧硬编码 8/20 与注册表脱钩, crit_gb 全仓
|
||||
# 无消费=改表不改行为的校准陷阱)
|
||||
r = chk.check_disk(ctx["data_root"], warn_gb=e.get("warn_gb", 20),
|
||||
crit_gb=e.get("crit_gb", 8))
|
||||
return {"key": "infra-disk-data", "status": r["status"],
|
||||
"expected": f">={e['warn_gb']}G", "actual": f"{r['free_gb']}G",
|
||||
"evidence": r}
|
||||
@@ -530,7 +544,7 @@ def _run_shift(args, db_path):
|
||||
try:
|
||||
age_h = (now - dt.datetime.fromisoformat(nas_ts)
|
||||
).total_seconds() / 3600
|
||||
except ValueError:
|
||||
except (ValueError, TypeError): # D-9(10-05): TypeError 同防
|
||||
age_h = None
|
||||
if age_h is not None and age_h > 30:
|
||||
events.append(_event(nas_silent_key, "yellow",
|
||||
|
||||
@@ -10,6 +10,12 @@ $ts = Get-Date -Format 'yyyyMMdd_HHmmss'
|
||||
$logDir = 'C:\sanguo_vnpy_v2\data\migration_logs'
|
||||
if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null }
|
||||
$log = Join-Path $logDir "data_monitor_$ts.txt"
|
||||
# migration_logs 保洁(10-05 infra 插单): 13 个 wrapper 各自追加、无任何清理机制
|
||||
# =无限增长; 本 wrapper 双班跑必经, 顺手剪 14 天前旧件(PS 重定向副本无解析消费,
|
||||
# 监控日志层真身=static/logs/, 诊断残留而已)。
|
||||
Get-ChildItem -Path $logDir -File -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.LastWriteTime -lt (Get-Date).AddDays(-14) } |
|
||||
Remove-Item -ErrorAction SilentlyContinue
|
||||
C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\data_monitor.py --shift $Shift *>> $log
|
||||
$rc = $LASTEXITCODE
|
||||
Add-Content -Path $log -Value "data_monitor shift=$Shift rc=$rc"
|
||||
|
||||
@@ -147,13 +147,15 @@ def dir_latest_mtime(directory):
|
||||
return latest
|
||||
|
||||
|
||||
def check_disk(path):
|
||||
def check_disk(path, warn_gb=20, crit_gb=8):
|
||||
"""磁盘剩余水位; 阈值参数化(D-8 10-05: 注册表 warn_gb/crit_gb 消费——旧
|
||||
硬编码 8/20 与注册表脱钩, crit_gb 曾全仓无消费=改表不改行为的校准陷阱)."""
|
||||
target = path if os.path.exists(path) else os.path.dirname(os.path.abspath(path))
|
||||
usage = shutil.disk_usage(target)
|
||||
free_gb = usage.free / 1024 ** 3
|
||||
if free_gb < 8:
|
||||
if free_gb < crit_gb:
|
||||
status = "red"
|
||||
elif free_gb < 20:
|
||||
elif free_gb < warn_gb:
|
||||
status = "yellow"
|
||||
else:
|
||||
status = "green"
|
||||
@@ -195,7 +197,10 @@ def scan_signatures(text):
|
||||
"""10-05 三修: ①只扫 ERROR/[FATAL] 行——'Expecting value' 出现在重试
|
||||
WARNING 行(financial_abstract/000963 实证), 良性重试不再误中
|
||||
source_changed; ②'418' 针收紧为 '418 Client Error'——裸 418 子串误中
|
||||
rows=41823/ok=19418 进度行(anti_crawl 假阳性实证)。"""
|
||||
rows=41823/ok=19418 进度行(anti_crawl 假阳性实证)。
|
||||
**注(10-05 补充审计)**: 生产路径已无调用方(签名分诊走 signatures_by_type,
|
||||
data_monitor.log_layer_results/log_only_results)——本函数仅测试消费,
|
||||
勿再接线; 与 signatures_by_type 的行为差异=不按 [type] 归因。"""
|
||||
lines = _error_lines(text)
|
||||
seen, out = set(), []
|
||||
for needle, sig, severity in SIGNATURES:
|
||||
@@ -211,10 +216,15 @@ _TYPE_TAG = re.compile(r"\[(\w+)\]")
|
||||
def signatures_by_type(text):
|
||||
"""签名按日志行 [type] 归因(10-05): 'ERROR [share_capital] ...' 行的
|
||||
签名只归 share_capital——防跨类型连坐(xueqiu 的 418 把 share_capital
|
||||
的 failed 抬红实证)。断路器行归触发类型, data_monitor 侧全局取。"""
|
||||
的 failed 抬红实证)。断路器行归触发类型, data_monitor 侧全局取。
|
||||
[FATAL] 是级别标记不是类型(P3 补充审计): 裸 FATAL 行
|
||||
(akshare_static_download.py:2721 实格式)归 _global, [type] [FATAL]
|
||||
双标记行(:2160 实格式)归 [type]。"""
|
||||
out = {}
|
||||
for ln in _error_lines(text):
|
||||
m = _TYPE_TAG.search(ln)
|
||||
while m and m.group(1) == "FATAL":
|
||||
m = _TYPE_TAG.search(ln, m.end())
|
||||
t = m.group(1) if m else "_global"
|
||||
sigs = out.setdefault(t, [])
|
||||
for needle, sig, severity in SIGNATURES:
|
||||
|
||||
@@ -21,13 +21,34 @@ XCHECK_REPORTS = os.path.join(CORPUS_ROOT, "xcheck", "reports")
|
||||
OUT_DIR = "/volume1/stock/monitor_nas/out"
|
||||
DISK_WARN_GB = 100 # 初值, 首周观察校准(spec §20.15.1 磁盘水位 NAS 侧)
|
||||
|
||||
# 沪深北休市表镜像(N-2 10-05)——=monitor_registry.EXCHANGE_HOLIDAYS, CI 同步测
|
||||
# (test_holiday_table_in_sync_with_vps_registry)钉两表相等, 单边改表即红(同
|
||||
# 7f9b413 strategy 侧镜像纪律); 年更义务经 VPS 侧年更闸+同步测传导到本表。
|
||||
# 旧 last_weekday 只剥周末: 0925 中秋/国庆连休窗把假日当「昨交易日」——当前被
|
||||
# hot_rank producer 的日历日节奏掩住(files 假期照落), 语义仍错。表缺年=不扣
|
||||
# (fail-safe, 黄照旧绝不静默转绿)。
|
||||
EXCHANGE_HOLIDAYS = {
|
||||
2026: {
|
||||
"20260101", "20260102", # 元旦
|
||||
"20260216", "20260217", "20260218", "20260219", "20260220", "20260223", # 春节
|
||||
"20260406", # 清明
|
||||
"20260501", "20260504", "20260505", # 劳动节
|
||||
"20260619", # 端午
|
||||
"20260925", # 中秋
|
||||
"20261001", "20261002", "20261005", "20261006", "20261007", # 国庆(1008 复市)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def last_weekday(before):
|
||||
"""strictly before 且为周一~五的最近一天(YYYYMMDD)——16:00 查时当日 17:35 班未采."""
|
||||
"""strictly before 且为交易日(周一~五且非休市表日)的最近一天(YYYYMMDD)
|
||||
——16:00 查时当日 17:35 班未采; N-2: 接休市表镜像, 跳过连休假日."""
|
||||
d = before - dt.timedelta(days=1)
|
||||
while d.weekday() >= 5:
|
||||
while True:
|
||||
hol = EXCHANGE_HOLIDAYS.get(d.year)
|
||||
if d.weekday() < 5 and (hol is None or d.strftime("%Y%m%d") not in hol):
|
||||
return d.strftime("%Y%m%d")
|
||||
d -= dt.timedelta(days=1)
|
||||
return d.strftime("%Y%m%d")
|
||||
|
||||
|
||||
def mtime_age_hours(path, now):
|
||||
|
||||
@@ -21,7 +21,10 @@ mkdir -p "$ROOT/out"
|
||||
{
|
||||
echo "=== $(date '+%F %T') nas monitor start ==="
|
||||
"$PY" "$ROOT/nas_monitor_check.py"
|
||||
echo "=== $(date '+%F %T') nas monitor exit=$? ==="
|
||||
rc_nas=$?
|
||||
# N-1(10-05): rc 先存后拼——旧版 $(date) 在同一 echo 词内先展开把 $? 重置
|
||||
# 为 0, exit= 恒 0; rc 隔离下这是 NAS 体检崩溃在 cron.log 的唯一现场线索
|
||||
echo "=== $(date '+%F %T') nas monitor exit=$rc_nas ==="
|
||||
# 远端 inbox 目录幂等建好; ssh 必带 -n(vps-mirror 坑: 不带会吃 stdin)
|
||||
ssh -n "${SCP_OPTS[@]}" "$VPS" \
|
||||
"powershell -Command \"New-Item -ItemType Directory -Force -Path '$VPS_INBOX' | Out-Null\"" \
|
||||
|
||||
Reference in New Issue
Block a user