feat(data): 财务136 W3' dmsk双lane落地+spec§19.8实施记录 [nas]
- fundamentals_dmsk_download.py 单脚本(2009→今backfill+近3期daily NOTIC_DATE>=T-7),corpus骨架六修正全带入 - marker修正: daily按日折叠(§19.4原文「一次永逸」=day-2同款坑,52756e3教训);backfill键=域+期(域缺席连带跳过,首测抓出) - 次新期金丝雀防「filter失效vs空期」不分导致的整段静默空跑 - run_dmsk_standalone.sh(DSM 07:10 sanguo-fundamentals-daily,一次性容器同corpus款)+verify_dmsk.py(键唯一硬门) - 21测全mock绿,全量295绿;W1'镜像对账全绿+vb2026 H1缺口入档
This commit is contained in:
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/bash
|
||||
# run_dmsk_standalone.sh — 财务三表 dmsk 补缺层 runner(2026-09-08 spec §19.4)
|
||||
#
|
||||
# 模式: 与 corpus/5m 完全同款的「DSM 任务(admin) → bash wrapper → 一次性容器」
|
||||
# (docker run --rm 跑 lock-aligned 镜像,主容器重启/重建永不波及本进程)。
|
||||
#
|
||||
# 落库: /volume1/stock/fundamentals/dmsk/{balance,income,cashflow}/
|
||||
# dt=<REPORT_DATE>/part-*.parquet —— **绝不写 phase3 镜像树**
|
||||
# (/volume1/stock/sanguo_vnpy_v2/data/static 归 04:00 同步链所有, 写=被冲掉
|
||||
# +破坏 ≡VPS 不变量; 双层读法见 spec §19.7)。
|
||||
#
|
||||
# 两 lane 串行(脚本内 flock 单实例锁天然互斥): daily --until 08:00
|
||||
# (近3期×3域=9 unit≈110发≈4min; NOTICE_DATE>=T-7 新披露, 披露次日可用);
|
||||
# rc=0|3 续走 backfill --until 17:30(2009→今≈2200发一次性回补, 完成后每日 no-op
|
||||
# 全 done 秒退); rc=1|2 跳过 backfill 次日自愈。rc 聚合同 corpus: 真失败(1/2)
|
||||
# 优先于让路(3)。
|
||||
#
|
||||
# 错峰: 07:10 晚于 corpus 06:00——东财端点分域(datacenter-web vs
|
||||
# search-api/np-listapi)+各自限速 1.5s/1.0s, 并发重叠安全依据同 corpus(spec §19.3)。
|
||||
#
|
||||
# 部署位: NAS /volume1/stock/fundamentals/run_dsmk.sh(宿主文件,CI 不覆盖;
|
||||
# 改本 repo 副本后 scp -O 手动同步, 同 corpus/5m 纪律):
|
||||
# scp -O scripts/nas_sync/run_dmsk_standalone.sh \
|
||||
# sanguo-nas:/volume1/stock/fundamentals/run_dsmk.sh
|
||||
# DSM 任务(sanguo-fundamentals-daily, 每日 07:10, 用户=admin)命令:
|
||||
# bash /volume1/stock/fundamentals/run_dsmk.sh
|
||||
#
|
||||
# --no-healthcheck: 镜像 HEALTHCHECK=curl :8000/health 是 web 服务专用(同 corpus)。
|
||||
set -u
|
||||
DOCKER=/var/packages/Docker/target/usr/bin/docker
|
||||
ROOT=/volume1/stock/fundamentals
|
||||
LOG=$ROOT/cron.log
|
||||
APP=/volume1/homes/admin/.sanguo_projects/sanguo_vnpy_v2
|
||||
UID_ADMIN=1024 # id admin (2026-09-06 实测)
|
||||
GID_ADMIN=100 # users
|
||||
GID_ADMINS=101 # administrators(Synology ACL 授权组, corpus 金丝雀实测必带)
|
||||
|
||||
mkdir -p "$ROOT"
|
||||
{
|
||||
echo "=== $(date '+%F %T') standalone-container dmsk run start ==="
|
||||
"$DOCKER" run --rm --name sanguo-fund-dmsk --user "${UID_ADMIN}:${GID_ADMIN}" \
|
||||
--group-add "${GID_ADMINS}" --no-healthcheck --entrypoint python \
|
||||
-v /volume1/stock:/volume1/stock \
|
||||
-v "$APP":/app:ro \
|
||||
sanguo_vnpy_v2:lock-aligned \
|
||||
/app/scripts/data_platform/fundamentals_dmsk_download.py --lane daily --until 08:00
|
||||
rc_daily=$?
|
||||
echo "=== $(date '+%F %T') dmsk daily lane exit=$rc_daily ==="
|
||||
rc_back=0
|
||||
if [ "$rc_daily" -eq 0 ] || [ "$rc_daily" -eq 3 ]; then
|
||||
"$DOCKER" run --rm --name sanguo-fund-dmsk --user "${UID_ADMIN}:${GID_ADMIN}" \
|
||||
--group-add "${GID_ADMINS}" --no-healthcheck --entrypoint python \
|
||||
-v /volume1/stock:/volume1/stock \
|
||||
-v "$APP":/app:ro \
|
||||
sanguo_vnpy_v2:lock-aligned \
|
||||
/app/scripts/data_platform/fundamentals_dmsk_download.py --lane backfill --until 17:30
|
||||
rc_back=$?
|
||||
echo "=== $(date '+%F %T') dmsk backfill lane exit=$rc_back ==="
|
||||
else
|
||||
echo "=== daily 非零($rc_daily), 跳过 backfill lane(次日自愈) ==="
|
||||
fi
|
||||
} >> "$LOG" 2>&1
|
||||
# rc 聚合: 真失败(1/2)优先于让路(3);全让路=3;全零=0
|
||||
if [ "${rc_back:-0}" -ne 0 ] && [ "${rc_back:-0}" -ne 3 ]; then
|
||||
exit "$rc_back"
|
||||
fi
|
||||
if [ "${rc_daily:-1}" -ne 0 ] && [ "${rc_daily:-1}" -ne 3 ]; then
|
||||
exit "$rc_daily"
|
||||
fi
|
||||
if [ "${rc_daily:-1}" -ne 0 ] || [ "${rc_back:-0}" -ne 0 ]; then
|
||||
exit 3
|
||||
fi
|
||||
exit 0
|
||||
Reference in New Issue
Block a user