feat(data): 双源5yr全市场部署 + baostock 15min + 断点续传

- raw_redownload 加断点续传(exists/skip已存在, 扩范围重下覆盖)
- baostock_download: 15min双源(qfq+raw)下载器, NAS容器跑, 限速防封
- full_deploy_5yr.sh: 无人值守日线双源pipeline(qfq→raw→rsync→验证)
- verify_dual_source: 容器内双源部署验证(fetch_day/iter_bars/除权日)

实测: 日线双源5yr 29600文件×2(rsync NAS), 浦发除权日 raw-5.9%/qfq-0.7%,
容器内 fetch_day/iter_bars/全市场抽检5只全通过. 15min沪深300 baostock限流121只.
This commit is contained in:
2026-07-09 19:22:36 +08:00
parent d5d3eea236
commit a372544045
4 changed files with 288 additions and 2 deletions
+38
View File
@@ -0,0 +1,38 @@
#!/bin/bash
# 无人值守日线双源 5yr 全市场部署 pipeline(本机串行,qfq→raw→rsync→验证)。
# 不并发打新浪源(封IP):qfq 进程结束才下 raw。
# 15min baostock 走 NAS 容器,与本机并行(不同源不同机)。
set -uo pipefail
cd /Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2
LOG=/tmp/full_deploy.log
SL=data_cache/stock_info/stock_basic_info_raw_20260326_113530.csv
log(){ echo "[$(date '+%m-%d %H:%M:%S')] $*" | tee -a "$LOG"; }
log "=== STAGE1 等 qfq 5yr 进程结束 ==="
while pgrep -f "raw_redownload.py --all --adjust qfq" >/dev/null; do sleep 60; done
log "qfq 完成: $(find data_cache/qfq -name '*.parquet'|wc -l) 文件"
log "=== STAGE2 raw 5yr 全市场(断点续传,单线程限速)==="
STOCK_LIST="$SL" RAW_DIR=data_cache/raw SLEEP=1.0 \
python3 scripts/data_platform/raw_redownload.py --all --start 2021-01-01 --adjust "" >> "$LOG" 2>&1
log "raw 完成: $(find data_cache/raw -name '*.parquet'|wc -l) 文件"
log "=== STAGE3 rsync qfq 全量(--delete覆盖NAS旧5验证股) → NAS ==="
rsync -az --delete \
data_cache/qfq/ \
sanguo-nas:/volume1/stock/A股数据/日线数据/qfq/ >> "$LOG" 2>&1
log "qfq rsync done"
log "=== STAGE4 rsync raw 2021-2023 → NAS(不覆盖NAS已有24-26==="
for y in 2021 2022 2023; do
rsync -az \
"data_cache/raw/$y/" \
"sanguo-nas:/volume1/stock/A股数据/日线数据/raw/$y/" >> "$LOG" 2>&1
log " raw/$y done"
done
log "=== STAGE5 验证 NAS 文件数 ==="
ssh sanguo-nas 'for d in qfq raw; do for y in 2021 2022 2023 2024 2025 2026; do echo "$d/$y=$(ls /volume1/stock/A股数据/日线数据/$d/$y/*.parquet 2>/dev/null|wc -l)"; done; done' >> "$LOG" 2>&1
log "=== ALL DONE ==="
echo "ALL_DONE_MARKER" >> "$LOG"