sanguo/sanguo_vnpy_v2
3
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
075052da546ad66630c5cc1c0a77342fa2e2f465
sanguo_vnpy_v2/scripts/data_platform/monitor_registry.py
T
claude_dev 075052da54
CI/CD / test (push) Successful in 38s
Details
CI/CD / nas-deploy (push) Successful in 3s
Details
CI/CD / nas-verify (push) Successful in 12s
Details
fix(data): 10-01 二轮审计吸收——白名单补四类+三注册表钉死测试; data-monitor 两班入自监; 旧名注册脚本根治; manifest 补齐; spec 三勘正 [vps] [nas] 
- P1-5 读写注册表脱节: EVENT_PANEL_TYPES 补 margin/sw_industry/dragon_tiger_seats/
  nb_daily(15→19, 09-12 四域注册采集时漏同步); 新增 test_whitelist_sync 钉死
  「采集 PER_DATE_TYPES 全族+gdhs == 白名单 == manifest domains ⊇」三同步义务;
  nb_hold 季频 asof 语义不进 exact-date 底座(钉死边界, 零消费方)
- P1-6 manifest 脱节: data_manifest.yaml 8 域→27 域(补事件面板族 19 类),
  消 data_gap_check 对真实供给误开缺口 issue 的潜伏面
- P1-10 监控自监: SCHTASKS +sanguo-data-monitor-morn/eve(17→19); 两 wrapper 本就
  恒 exit 0(设计预留), 非零=真崩, 零假红; 双班全死盲区待 strategy 域 health
  mtime 反盯(已转呈)
- docs_audit 旧名根治: register_schtasks.ps1 重写为现役 bs-daily/xt-daily
  (旧版重跑会复活死任务双开, 09-01 事故同源); bs_eod_wrapper/xt_eod_wrapper
  头注勘正
- docs_audit spec 三勘正: §16.1 白名单 14→19 类; §20.1「零改动自动可读」落空
  改「白名单一行即读+钉死测试」; §18.8 自称权威降级+ak-daily 07:35→19:00
  (VPS 实查)+xt 行名归一+ak-events 指针化; §20.10.2 硬计数指针化;
  §20.15.1/runbook 在岗 17→19
- docs_audit README §4 调度表整表换现役 10 任务; data-monitoring 计划加三行
  状态头注(初值退出权威位)

tests: data_platform+portfolio 1397 passed

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-01 09:19:29 +08:00

158 lines
7.2 KiB
Python
Raw Blame History

"""被监控对象注册表 — spec §20.15.1 权威导出(2026-09-25 初值, 行数带宽跑两周后校准).
kind 语义(监控检查分发依据, 见 monitor_checks/data_monitor):
per_date static/<name>/YYYYMMDD_<name>.parquet 日频/快照: 新鲜度(红)+行数带宽(黄)
weekly_friday 周五锚 YYYYMMDD_<name>.parquet: 周六 ready 后新鲜度(红)
month_bucket YYYYMM_<name>.parquet: 最新桶距今<=max_age_days(黄)+带宽(黄)
year_file data/valuation_baostock/<YYYY>.parquet 当年文件 mtime>=期望日
dir_mtime 目录内最新 mtime 距今<=max_age_days(黄)
db_sample quant_trading.db 抽样 MAX(datetime)>=期望交易日(红)
vintage_json 读 data/static/vintage_status.json 结论字段
log_only 仅过程层: 统计行存在且 failed=0(ak 逐股族)
static_exists one-shot 文件存在性
disk 数据根剩余空间
"""
import re
def _pd(name, band, ready="20:30", lag=0, severity="red", producer="vps"):
return {"name": name, "kind": "per_date", "root": "static", "subdir": name,
"pattern": rf"^(\d{{8}})_{re.escape(name)}$", "row_band": band,
"ready": ready, "lag": lag, "severity": severity,
"producer": producer, "enabled": True}
_PER_DATE = [
_pd("dragon_tiger", (0, 400)),
_pd("block_trade", (30, 300)),
_pd("restricted", (0, 60)),
_pd("zt_pool", (15, 150)),
_pd("zt_pool_zbgc", (1, 60)),
_pd("zt_pool_dtgc", (1, 60)),
_pd("fund_flow_industry", (60, 120)),
_pd("fund_flow_concept", (200, 600)),
_pd("ths_industry", (60, 120)),
_pd("ths_concept", (200, 600)),
_pd("sina_sector", (30, 80)),
_pd("sw_industry", (4500, 6000)), # 申万成分股粒度(09-26 实测 5220, spec 的 31=行业数)
_pd("xueqiu_hot", (5000, 6500)),
_pd("hot_rank", (99, 101), ready="18:30", producer="nas"),
_pd("margin", (200, 8000), ready="21:00", lag=1), # 实测 6051(0928 文件, 0929 班采); 两融标的扩容涨破旧 4000 上限致 0929/30 连黄
_pd("margin_sse", (200, 8000), ready="21:00", lag=1), # SSE 子集 ~3300 贴脸旧 4000 上限(1010 二审 P2-2, 与全市场同受扩容驱动)
_pd("dragon_tiger_seats", (200, 2000)), # 实测 670~810(0923~28); 旧(0,400)上限低致真交易日必黄(0928 670 实案)
_pd("nb_daily", (30, 150)),
]
_WEEKLY = [
{"name": "pledge_ratio", "kind": "weekly_friday", "root": "static",
"subdir": "pledge_ratio", "pattern": r"^(\d{8})_pledge_ratio$",
"row_band": (1500, 3000), "ready": "06:00", "severity": "red",
"producer": "vps", "enabled": True},
{"name": "research_report", "kind": "weekly_friday", "root": "static",
"subdir": "research_report", "pattern": r"^(\d{8})_research_report$",
"row_band": (50, 1500), "ready": "06:00", "severity": "red",
"producer": "vps", "enabled": True},
]
_MONTH = [
{"name": "pledge_detail", "kind": "month_bucket", "root": "static",
"subdir": "pledge_detail", "pattern": r"^(\d{6})_pledge_detail$",
"row_band": (40000, 250000), "max_age_days": 45, "severity": "yellow",
"producer": "vps", "enabled": True},
]
_YEAR = [
{"name": "valuation_baostock", "kind": "year_file", "root": "data",
"subdir": "valuation_baostock", "ready": "22:30", "severity": "yellow",
"producer": "vps", "enabled": True},
]
_DIR_MTIME = [
# (name, root, subdir, max_age_days, cadence 注释)
("gdhs", "static", "gdhs", 10), # ak-weekly 增量
("share_capital", "static", "share_capital", 10),
("top_holders", "static", "top_holders", 45),
("financial_abstract", "static", "financial_abstract", 45),
("nb_hold", "static", "nb_hold", 120), # 季频快照
("balance", "static", "balance", 130), # 三表 marathon 季频
("income", "static", "income", 130),
("cashflow", "static", "cashflow", 130),
("forecast", "static", "forecast", 130),
("express", "static", "express", 130),
("fundamentals_baostock", "data", "fundamentals_baostock", 4),
("index_const_hist", "data", "index_const_hist", 40), # idx-monthly 16日
]
_DB_SAMPLE = [
{"name": "dbbardata_daily", "kind": "db_sample", "root": "data",
"samples": [["600519", "SSE"], ["000001", "SZSE"], ["510300", "SSE"],
["159915", "SZSE"]],
"ready": "22:30", "lag": 0, "severity": "red", "producer": "vps",
"enabled": True}, # bs-daily 马拉松+xt 腿混合写入
{"name": "dbbardata_index", "kind": "db_sample", "root": "data",
"samples": [["000300", "SSE"], ["000905", "SSE"]],
"ready": "19:00", "lag": 0, "severity": "red", "producer": "vps",
"enabled": True}, # idx 18:30 9 指数
]
_MISC = [
{"name": "valuation", "kind": "log_only", "log_cadence": "daily",
"enabled": True, "producer": "vps", "severity": "red"},
{"name": "northbound", "kind": "log_only", "log_cadence": "weekly",
"enabled": False, "producer": "vps", "severity": "yellow",
"note": "§20.4 backlog 域(2024-08 停发), 只登记防漏, 不检查"},
{"name": "index_const", "kind": "static_exists", "root": "static",
"subdir": "index_const", "enabled": True, "producer": "vps",
"severity": "yellow"},
{"name": "industry", "kind": "static_exists", "root": "static",
"subdir": "industry", "enabled": True, "producer": "vps",
"severity": "yellow"},
{"name": "vintage", "kind": "vintage_json", "root": "static",
"subdir": "vintage_status.json", "enabled": True, "producer": "vps",
"severity": "yellow"},
{"name": "disk", "kind": "disk", "warn_gb": 20, "crit_gb": 8,
"enabled": True, "producer": "vps", "severity": "red"},
]
def _dir_entries():
out = []
for name, root, subdir, days in _DIR_MTIME:
out.append({"name": name, "kind": "dir_mtime", "root": root,
"subdir": subdir, "max_age_days": days,
"severity": "yellow", "producer": "vps", "enabled": True})
return out
REGISTRY = (_PER_DATE + _WEEKLY + _MONTH + _YEAR + _dir_entries()
+ _DB_SAMPLE + _MISC)
# 调度层在岗检查(spec §20.15.1 表后注: 19 个 schtask; retry 一次性死案即此类)
# 1010 二审 P1-2: strategy-monitor 两班补进名单——否则监控自身被禁用/损坏时
# alerts/health 停更且全链静默(在岗检查正是为此立的设计)
# 10-01 二轮审计 P1-10: data-monitor 两班(eve 23:50/morn 08:05)同款入岗——
# 两班互盯盖单班机械故障(wrapper 恒 exit 0,红态不污染 last_result,非零=真崩);
# 双班全死仍有盲区,根治=外部反盯(strategy_monitor 侧 health 文件 mtime 检查,
# 归 strategy 域)
SCHTASKS = [
("sanguo-qmt-relogin", "daily"),
("sanguo-qmt-sentinel", "always"),
("sanguo-qmt-probe0915", "weekday"),
("sanguo-live-supervisor", "always"),
("sanguo-shadow-desk", "always"),
("sanguo-api", "always"),
("sanguo-bs-daily", "daily"),
("sanguo-xt-daily", "daily"),
("sanguo-ak-daily", "daily"),
("sanguo-ak-events", "daily"),
("sanguo-ak-events-retry", "weekday"),
("sanguo-ak-weekly", "weekly"),
("sanguo-ak-vintage", "daily"),
("sanguo-index", "monthly"),
("sanguo-idx-daily", "daily"),
("sanguo-strategy-monitor-morn", "daily"),
("sanguo-strategy-monitor-eve", "daily"),
("sanguo-data-monitor-morn", "daily"),
("sanguo-data-monitor-eve", "daily"),
]
Reference in New Issue View Git Blame Copy Permalink