claude_dev
a372544045
- raw_redownload 加断点续传(exists/skip已存在, 扩范围重下覆盖) - baostock_download: 15min双源(qfq+raw)下载器, NAS容器跑, 限速防封 - full_deploy_5yr.sh: 无人值守日线双源pipeline(qfq→raw→rsync→验证) - verify_dual_source: 容器内双源部署验证(fetch_day/iter_bars/除权日) 实测: 日线双源5yr 29600文件×2(rsync NAS), 浦发除权日 raw-5.9%/qfq-0.7%, 容器内 fetch_day/iter_bars/全市场抽检5只全通过. 15min沪深300 baostock限流121只.
39 lines
1.7 KiB
Bash
Executable File
39 lines
1.7 KiB
Bash
Executable File
#!/bin/bash
|
||
# 无人值守日线双源 5yr 全市场部署 pipeline（本机串行，qfq→raw→rsync→验证）。
|
||
# 不并发打新浪源（封IP）：qfq 进程结束才下 raw。
|
||
# 15min baostock 走 NAS 容器，与本机并行（不同源不同机）。
|
||
set -uo pipefail
|
||
cd /Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2
|
||
LOG=/tmp/full_deploy.log
|
||
SL=data_cache/stock_info/stock_basic_info_raw_20260326_113530.csv
|
||
log(){ echo "[$(date '+%m-%d %H:%M:%S')] $*" | tee -a "$LOG"; }
|
||
|
||
log "=== STAGE1 等 qfq 5yr 进程结束 ==="
|
||
while pgrep -f "raw_redownload.py --all --adjust qfq" >/dev/null; do sleep 60; done
|
||
log "qfq 完成: $(find data_cache/qfq -name '*.parquet'|wc -l) 文件"
|
||
|
||
log "=== STAGE2 raw 5yr 全市场（断点续传，单线程限速）==="
|
||
STOCK_LIST="$SL" RAW_DIR=data_cache/raw SLEEP=1.0 \
|
||
python3 scripts/data_platform/raw_redownload.py --all --start 2021-01-01 --adjust "" >> "$LOG" 2>&1
|
||
log "raw 完成: $(find data_cache/raw -name '*.parquet'|wc -l) 文件"
|
||
|
||
log "=== STAGE3 rsync qfq 全量(--delete覆盖NAS旧5验证股) → NAS ==="
|
||
rsync -az --delete \
|
||
data_cache/qfq/ \
|
||
sanguo-nas:/volume1/stock/A股数据/日线数据/qfq/ >> "$LOG" 2>&1
|
||
log "qfq rsync done"
|
||
|
||
log "=== STAGE4 rsync raw 2021-2023 → NAS（不覆盖NAS已有24-26）==="
|
||
for y in 2021 2022 2023; do
|
||
rsync -az \
|
||
"data_cache/raw/$y/" \
|
||
"sanguo-nas:/volume1/stock/A股数据/日线数据/raw/$y/" >> "$LOG" 2>&1
|
||
log " raw/$y done"
|
||
done
|
||
|
||
log "=== STAGE5 验证 NAS 文件数 ==="
|
||
ssh sanguo-nas 'for d in qfq raw; do for y in 2021 2022 2023 2024 2025 2026; do echo "$d/$y=$(ls /volume1/stock/A股数据/日线数据/$d/$y/*.parquet 2>/dev/null|wc -l)"; done; done' >> "$LOG" 2>&1
|
||
|
||
log "=== ALL DONE ==="
|
||
echo "ALL_DONE_MARKER" >> "$LOG"
|