docs+fix(data): run_daily_update重写raw+qfq增量 + daily-update-design v3双源

- run_daily_update.sh: 重写raw+qfq增量(跳daily_all_update新浪坏接口KeyError:date,
  持久路径data_cache/daily_update不进tmp, set -uo pipefail不-e); C-S3每日raw(撮合)+qfq(warmup)
- daily-update-design.md: v3.0变更记录+十六节(双源raw/qfq架构/脚本更新清单/launchd15:30部署/准确性验证/与v2关系)
- config/data_platform.yaml: minute_15_qfq_dir/minute_15_raw_dir(15min双源目录)
This commit is contained in:
2026-07-10 08:16:41 +08:00
parent 917d5bca2a
commit 86a7f8ef2d
2 changed files with 80 additions and 36 deletions
+41 -36
View File
@@ -1,46 +1,51 @@
#!/bin/bash
# v2 数据下载包装(SSH 模式:rsync NAS↔本地 + 跑 v1 daily_all_update + raw 增量)
# 不依赖 SMB 挂载,用 sanguo-nas SSH key 免密。
# 用法:./run_daily_update.sh [--skip-daily|--skip-15min]
# 每日数据增量(C-S3 实走用):raw 真实价 + qfq 前复权,最近 N 天 → NAS。
#
# 跳过 v1 daily_all_update(新浪源接口坏 KeyError:dateC-S3 不用 daily mixed)。
# raw/qfq 用 raw_redownload.pyakshare 新浪 stock_zh_a_daily,全量 29600 已验证)。
# 15min baostock 增量 = 分期项(默认跳,日内策略落地时加)。
#
# env:
# STOCK_LIMIT=N 限制股票数(验证用,默认 0=全市场
# SKIP_PULL=1 跳过 rsync 拉(验证用;生产必须拉,v1 才能增量
# SKIP_RAW=1 跳 raw 增量(默认跑,C-S3 实走需当日 raw)
set -e
# DAYS=5 增量回看天数(C-S3 实走需当日,5 天兜底停牌/补缺
# STOCK_LIST=... 全市场 csv(默认 data_cache/stock_info
# SKIP_RAW=1 跳 raw 增量
# SKIP_QFQ=1 跳 qfq 增量
set -uo pipefail # 不用 -e:单只失败不退(raw_redownload 内部已容错记 fail
NAS=sanguo-nas
LOCAL=${STOCK_MOUNT:-/tmp/stock_dl}
export STOCK_MOUNT=$LOCAL
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
LOCAL=${STOCK_MOUNT:-$ROOT/data_cache/daily_update} # 持久目录(不进 /tmp,重启不丢)
SL=${STOCK_LIST:-$ROOT/data_cache/stock_info/stock_basic_info_raw_20260326_113530.csv}
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
DAYS=${DAYS:-5}
START=$(python3 -c "import datetime;print((datetime.date.today()-datetime.timedelta(days=$DAYS)).isoformat())")
mkdir -p "$LOCAL/logs/daily_update/progress" \
"$LOCAL/A股数据/日线数据/daily" \
"$LOCAL/A股数据/日线数据/raw" \
"$LOCAL/minute_kline/15min" \
"$LOCAL/A股数据/stock_info" \
"$LOCAL/sanguo_vnpy/data"
if [ "${SKIP_PULL:-0}" != "1" ]; then
echo "=== 1. rsync 拉 NAS 现有 → 本地(增量,首次慢后续快)==="
rsync -az "$NAS:/volume1/stock/A股数据/stock_info/" "$LOCAL/A股数据/stock_info/" || echo " stock_info pull warn"
rsync -az "$NAS:/volume1/stock/A股数据/日线数据/daily/" "$LOCAL/A股数据/日线数据/daily/" || echo " daily pull warn"
rsync -az "$NAS:/volume1/stock/A股数据/日线数据/raw/" "$LOCAL/A股数据/日线数据/raw/" || echo " raw pull warn"
rsync -az "$NAS:/volume1/stock/minute_kline/15min/" "$LOCAL/minute_kline/15min/" || echo " 15min pull warn"
rsync -az "$NAS:/volume1/stock/sanguo_vnpy/data/" "$LOCAL/sanguo_vnpy/data/" || echo " vnpy data pull warn"
fi
echo "=== 2. 跑 v1 增量更新(STOCK_MOUNT=$LOCAL STOCK_LIMIT=${STOCK_LIMIT:-0}==="
mkdir -p "$LOCAL/raw" "$LOCAL/qfq"
echo "=== $(date) 每日增量 start=$START DAYS=$DAYS ==="
cd "$SCRIPT_DIR"
python3 daily_all_update.py "$@"
if [ "${SKIP_RAW:-0}" != "1" ]; then
echo "=== 2.5 raw 增量(最近 5 天覆盖 raw_dirC-S3 实走需当日 raw==="
RAW_START=$(python3 -c "import datetime;print((datetime.date.today()-datetime.timedelta(days=5)).isoformat())")
SLEEP=0.5 python3 raw_redownload.py --all --start "$RAW_START" || echo " raw 增量 warn"
# 拉 stock_info(代码列表)若本地缺
if [ ! -f "$SL" ]; then
mkdir -p "$(dirname "$SL")"
rsync -az "$NAS:/volume1/stock/A股数据/stock_info/" "$(dirname "$SL")/" || true
fi
echo "=== 3. rsync 推 本地 → NAS ==="
rsync -az "$LOCAL/A股数据/日线数据/daily/" "$NAS:/volume1/stock/A股数据/日线数据/daily/"
rsync -az "$LOCAL/A股数据/日线数据/raw/" "$NAS:/volume1/stock/A股数据/日线数据/raw/"
rsync -az "$LOCAL/minute_kline/15min/" "$NAS:/volume1/stock/minute_kline/15min/"
rsync -az "$LOCAL/sanguo_vnpy/data/" "$NAS:/volume1/stock/sanguo_vnpy/data/"
# 1. raw 增量(C-S3 撮合真实价,adjustflag=3 / akshare adjust=""
if [ "${SKIP_RAW:-0}" != "1" ]; then
echo "=== 1. raw 增量(最近 $DAYS 天 → $LOCAL/raw==="
STOCK_LIST="$SL" RAW_DIR="$LOCAL/raw" SLEEP=0.5 \
python3 raw_redownload.py --all --start "$START" --adjust "" || echo " raw warn(单只失败已记)"
fi
# 2. qfq 增量(C-S3 warmup 信号,无除权缺口)
if [ "${SKIP_QFQ:-0}" != "1" ]; then
echo "=== 2. qfq 增量(最近 $DAYS 天 → $LOCAL/qfq==="
STOCK_LIST="$SL" RAW_DIR="$LOCAL/qfq" SLEEP=0.5 \
python3 raw_redownload.py --all --start "$START" --adjust qfq || echo " qfq warn(单只失败已记)"
fi
# 3. rsync 推 NASraw_dir + qfq_dir
echo "=== 3. rsync → NAS ==="
rsync -az "$LOCAL/raw/" "$NAS:/volume1/stock/A股数据/日线数据/raw/" || echo " raw push warn"
rsync -az "$LOCAL/qfq/" "$NAS:/volume1/stock/A股数据/日线数据/qfq/" || echo " qfq push warn"
echo "=== 完成 $(date) ==="