feat(data): T1 语料库 wrapper run_corpus.sh+连通金丝雀全绿 [nas]
spec §18.7 T1(sanguo-corpus 一次性容器,统一 5m 模式): - repo 规范副本 scripts/nas_sync/run_corpus_standalone.sh(同 run_5m_standalone.sh 先例),部署位=/volume1/stock/corpus/run_corpus.sh(宿主文件 CI 不覆盖,scp -O 同步) - 两改良落地:--name sanguo-corpus(杀=docker stop)+--user 1024:100(admin 属主) - 新发现硬需求 --group-add 101:/volume1/stock Synology ACL 只授权 group:administrators,容器同 uid 无组 101 读文件 Permission denied (POSIX rwxrwxrwx 被 NFSv4 ACL 覆盖);补组后读/写/继承全绿 - 两 lane 串行(daily 成功才跑 backfill --until 17:30),rc 聚合 daily 优先 - 金丝雀(logs/canary_t1.log):python 3.10.20+pandas/pyarrow/requests 齐备; --user 写=uid1024 属主;四域出网 200=巨潮 query/巨潮 PDF/np-listapi/ search-api(content 摘要字段在,akshare 同款端点契约验证) - 目录骨架 admin 属主:news_meta/news_fulltext/ann_meta/ann_pdf/state/logs
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
# run_corpus_standalone.sh — 基本面语料库(新闻+公告)采集 runner(2026-09-06 spec §18)
|
||||
#
|
||||
# 模式: 与 sanguo-5m 完全同款的「bash wrapper → 一次性容器」(docker run --rm 跑
|
||||
# lock-aligned 镜像,主容器重启/重建永不波及本进程——08-22 5m 被 CI 杀三案后的
|
||||
# 同款根治)。
|
||||
#
|
||||
# corpus 两改良(spec §18.1,用户「统一到5min吧」终版):
|
||||
# ① DSM 任务以 admin 跑(admin 在 docker 组,run/stop 实测可用);
|
||||
# ② 容器加 --user 1024:100(admin)→ 输出文件 admin 属主(root 文件坑不复发),
|
||||
# 杀进程 = docker stop sanguo-corpus(admin 一条命令)。
|
||||
#
|
||||
# --group-add 101(administrators, 09-06 金丝雀实测发现的硬需求):
|
||||
# /volume1/stock 的 Synology ACL(NFSv4 风格,synoacltool)只授权
|
||||
# group:administrators——宿主机 admin 靠组成员身份通过;容器内同 uid 1024
|
||||
# 但无附属组 101 会被 Permission denied(POSIX rwxrwxrwx 位被 ACL 覆盖)。
|
||||
# 补 --group-add 101 后读/写/新文件继承全绿(admin 属主+administrators ACL)。
|
||||
#
|
||||
# 两 lane 串行(脚本内单实例锁天然互斥,这里顺序执行): daily 成功后才跑
|
||||
# backfill(--until 17:30 墙钟自停);rc 聚合=任一非零即整体非零,daily 优先报。
|
||||
#
|
||||
# 部署位: NAS /volume1/stock/corpus/run_corpus.sh(宿主文件,CI 不覆盖;
|
||||
# 改本 repo 副本后需 scp -O 手动同步——DSM 无 SFTP 子系统,同 5m 纪律):
|
||||
# scp -O scripts/nas_sync/run_corpus_standalone.sh \
|
||||
# sanguo-nas:/volume1/stock/corpus/run_corpus.sh
|
||||
# DSM 任务(sanguo-corpus-daily, 每日 06:00, 用户=admin)命令:
|
||||
# bash /volume1/stock/corpus/run_corpus.sh
|
||||
#
|
||||
# 注意: 镜像 tag 若将来基建重建改名,这里要跟着改(与主容器 compose 同源)。
|
||||
set -u
|
||||
DOCKER=/var/packages/Docker/target/usr/bin/docker
|
||||
LOG=/volume1/stock/corpus/cron.log
|
||||
APP=/volume1/homes/admin/.sanguo_projects/sanguo_vnpy_v2
|
||||
UID_ADMIN=1024 # id admin (2026-09-06 实测)
|
||||
GID_ADMIN=100 # users
|
||||
GID_ADMINS=101 # administrators(Synology ACL 授权组,金丝雀实测必带)
|
||||
|
||||
{
|
||||
echo "=== $(date '+%F %T') standalone-container corpus run start ==="
|
||||
"$DOCKER" run --rm --name sanguo-corpus --user "${UID_ADMIN}:${GID_ADMIN}" \
|
||||
--group-add "${GID_ADMINS}" --entrypoint python \
|
||||
-v /volume1/stock:/volume1/stock \
|
||||
-v "$APP":/app:ro \
|
||||
sanguo_vnpy_v2:lock-aligned \
|
||||
/app/scripts/data_platform/corpus_download.py --lane daily
|
||||
rc_daily=$?
|
||||
echo "=== $(date '+%F %T') corpus daily lane exit=$rc_daily ==="
|
||||
rc_back=0
|
||||
if [ "$rc_daily" -eq 0 ]; then
|
||||
"$DOCKER" run --rm --name sanguo-corpus --user "${UID_ADMIN}:${GID_ADMIN}" \
|
||||
--group-add "${GID_ADMINS}" --entrypoint python \
|
||||
-v /volume1/stock:/volume1/stock \
|
||||
-v "$APP":/app:ro \
|
||||
sanguo_vnpy_v2:lock-aligned \
|
||||
/app/scripts/data_platform/corpus_download.py --lane backfill --until 17:30
|
||||
rc_back=$?
|
||||
echo "=== $(date '+%F %T') corpus backfill lane exit=$rc_back ==="
|
||||
else
|
||||
echo "=== daily 非零($rc_daily), 跳过 backfill lane(次日自愈) ==="
|
||||
fi
|
||||
} >> "$LOG" 2>&1
|
||||
if [ "${rc_daily:-1}" -ne 0 ]; then
|
||||
exit "$rc_daily"
|
||||
fi
|
||||
exit "${rc_back:-0}"
|
||||
Reference in New Issue
Block a user