From 674cfadba7e6693a08373df38c028038cc0c1747 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 8 Jul 2026 06:51:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(data):=20run=5Fdaily=5Fupdate=E5=8A=A0raw?= =?UTF-8?q?=E5=A2=9E=E9=87=8F=E2=80=94=E6=AF=8F=E6=97=A515:30=E6=8B=89?= =?UTF-8?q?=E6=9C=80=E8=BF=915=E5=A4=A9raw=E6=8E=A8NAS(C-S3=E5=AE=9E?= =?UTF-8?q?=E8=B5=B0=E6=95=B0=E6=8D=AE=E7=AE=A1=E9=81=93)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/data_platform/run_daily_update.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/data_platform/run_daily_update.sh b/scripts/data_platform/run_daily_update.sh index c2e042d..319b455 100755 --- a/scripts/data_platform/run_daily_update.sh +++ b/scripts/data_platform/run_daily_update.sh @@ -1,10 +1,11 @@ #!/bin/bash -# v2 数据下载包装(SSH 模式:rsync NAS↔本地 + 跑 v1 daily_all_update) +# v2 数据下载包装(SSH 模式:rsync NAS↔本地 + 跑 v1 daily_all_update + raw 增量) # 不依赖 SMB 挂载,用 sanguo-nas SSH key 免密。 # 用法:./run_daily_update.sh [--skip-daily|--skip-15min] # env: # STOCK_LIMIT=N 限制股票数(验证用,默认 0=全市场) # SKIP_PULL=1 跳过 rsync 拉(验证用;生产必须拉,v1 才能增量) +# SKIP_RAW=1 跳过 raw 增量(默认跑,C-S3 实走需当日 raw) set -e NAS=sanguo-nas LOCAL=${STOCK_MOUNT:-/tmp/stock_dl} @@ -13,6 +14,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" mkdir -p "$LOCAL/logs/daily_update/progress" \ "$LOCAL/A股数据/日线数据/daily" \ + "$LOCAL/A股数据/日线数据/raw" \ "$LOCAL/minute_kline/15min" \ "$LOCAL/A股数据/stock_info" \ "$LOCAL/sanguo_vnpy/data" @@ -21,6 +23,7 @@ if [ "${SKIP_PULL:-0}" != "1" ]; then echo "=== 1. rsync 拉 NAS 现有 → 本地(增量,首次慢后续快)===" rsync -az "$NAS:/volume1/stock/A股数据/stock_info/" "$LOCAL/A股数据/stock_info/" || echo " stock_info pull warn" rsync -az "$NAS:/volume1/stock/A股数据/日线数据/daily/" "$LOCAL/A股数据/日线数据/daily/" || echo " daily pull warn" + rsync -az "$NAS:/volume1/stock/A股数据/日线数据/raw/" "$LOCAL/A股数据/日线数据/raw/" || echo " raw pull warn" rsync -az "$NAS:/volume1/stock/minute_kline/15min/" "$LOCAL/minute_kline/15min/" || echo " 15min pull warn" rsync -az "$NAS:/volume1/stock/sanguo_vnpy/data/" "$LOCAL/sanguo_vnpy/data/" || echo " vnpy data pull warn" fi @@ -29,8 +32,15 @@ echo "=== 2. 跑 v1 增量更新(STOCK_MOUNT=$LOCAL STOCK_LIMIT=${STOCK_LIMIT: cd "$SCRIPT_DIR" python3 daily_all_update.py "$@" +if [ "${SKIP_RAW:-0}" != "1" ]; then + echo "=== 2.5 raw 增量(最近 5 天覆盖 raw_dir,C-S3 实走需当日 raw)===" + RAW_START=$(python3 -c "import datetime;print((datetime.date.today()-datetime.timedelta(days=5)).isoformat())") + SLEEP=0.5 python3 raw_redownload.py --all --start "$RAW_START" || echo " raw 增量 warn" +fi + echo "=== 3. rsync 推 本地 → NAS ===" rsync -az "$LOCAL/A股数据/日线数据/daily/" "$NAS:/volume1/stock/A股数据/日线数据/daily/" +rsync -az "$LOCAL/A股数据/日线数据/raw/" "$NAS:/volume1/stock/A股数据/日线数据/raw/" rsync -az "$LOCAL/minute_kline/15min/" "$NAS:/volume1/stock/minute_kline/15min/" rsync -az "$LOCAL/sanguo_vnpy/data/" "$NAS:/volume1/stock/sanguo_vnpy/data/" echo "=== 完成 $(date) ==="