feat(data): run_daily_update加raw增量—每日15:30拉最近5天raw推NAS(C-S3实走数据管道)

This commit is contained in:
2026-07-08 06:51:00 +08:00
parent 6931a7b541
commit 674cfadba7
+11 -1
View File
@@ -1,10 +1,11 @@
#!/bin/bash
# v2 数据下载包装(SSH 模式:rsync NAS↔本地 + 跑 v1 daily_all_update
# v2 数据下载包装(SSH 模式:rsync NAS↔本地 + 跑 v1 daily_all_update + raw 增量
# 不依赖 SMB 挂载,用 sanguo-nas SSH key 免密。
# 用法:./run_daily_update.sh [--skip-daily|--skip-15min]
# env:
# STOCK_LIMIT=N 限制股票数(验证用,默认 0=全市场)
# SKIP_PULL=1 跳过 rsync 拉(验证用;生产必须拉,v1 才能增量)
# SKIP_RAW=1 跳过 raw 增量(默认跑,C-S3 实走需当日 raw)
set -e
NAS=sanguo-nas
LOCAL=${STOCK_MOUNT:-/tmp/stock_dl}
@@ -13,6 +14,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
mkdir -p "$LOCAL/logs/daily_update/progress" \
"$LOCAL/A股数据/日线数据/daily" \
"$LOCAL/A股数据/日线数据/raw" \
"$LOCAL/minute_kline/15min" \
"$LOCAL/A股数据/stock_info" \
"$LOCAL/sanguo_vnpy/data"
@@ -21,6 +23,7 @@ if [ "${SKIP_PULL:-0}" != "1" ]; then
echo "=== 1. rsync 拉 NAS 现有 → 本地(增量,首次慢后续快)==="
rsync -az "$NAS:/volume1/stock/A股数据/stock_info/" "$LOCAL/A股数据/stock_info/" || echo " stock_info pull warn"
rsync -az "$NAS:/volume1/stock/A股数据/日线数据/daily/" "$LOCAL/A股数据/日线数据/daily/" || echo " daily pull warn"
rsync -az "$NAS:/volume1/stock/A股数据/日线数据/raw/" "$LOCAL/A股数据/日线数据/raw/" || echo " raw pull warn"
rsync -az "$NAS:/volume1/stock/minute_kline/15min/" "$LOCAL/minute_kline/15min/" || echo " 15min pull warn"
rsync -az "$NAS:/volume1/stock/sanguo_vnpy/data/" "$LOCAL/sanguo_vnpy/data/" || echo " vnpy data pull warn"
fi
@@ -29,8 +32,15 @@ echo "=== 2. 跑 v1 增量更新(STOCK_MOUNT=$LOCAL STOCK_LIMIT=${STOCK_LIMIT:
cd "$SCRIPT_DIR"
python3 daily_all_update.py "$@"
if [ "${SKIP_RAW:-0}" != "1" ]; then
echo "=== 2.5 raw 增量(最近 5 天覆盖 raw_dirC-S3 实走需当日 raw==="
RAW_START=$(python3 -c "import datetime;print((datetime.date.today()-datetime.timedelta(days=5)).isoformat())")
SLEEP=0.5 python3 raw_redownload.py --all --start "$RAW_START" || echo " raw 增量 warn"
fi
echo "=== 3. rsync 推 本地 → NAS ==="
rsync -az "$LOCAL/A股数据/日线数据/daily/" "$NAS:/volume1/stock/A股数据/日线数据/daily/"
rsync -az "$LOCAL/A股数据/日线数据/raw/" "$NAS:/volume1/stock/A股数据/日线数据/raw/"
rsync -az "$LOCAL/minute_kline/15min/" "$NAS:/volume1/stock/minute_kline/15min/"
rsync -az "$LOCAL/sanguo_vnpy/data/" "$NAS:/volume1/stock/sanguo_vnpy/data/"
echo "=== 完成 $(date) ==="