From ab8f4e3782c9b7f7bb25905778284360a52dfb98 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 2 Sep 2026 22:31:23 +0800 Subject: [PATCH] =?UTF-8?q?fix(nas):=20sync=5Fparquet=5Fdaily=20=E6=89=A9?= =?UTF-8?q?=E5=88=B0=20A=20=E6=A1=A3=2013=20=E7=B1=BB+gdhs=20=E5=85=A8?= =?UTF-8?q?=E9=87=8F=E6=8B=89=E2=80=94=E2=80=94=E5=B7=A1=E6=9F=A5=E5=AE=9E?= =?UTF-8?q?=E9=94=A4=20TYPES=20=E5=86=99=E6=AD=BB=E6=97=A7=204=20=E7=B1=BB?= =?UTF-8?q?,=E6=98=8E=E6=99=A8=2003:00=20=E5=BF=AB=E7=85=A7=E6=97=8F/?= =?UTF-8?q?=E6=B6=A8=E5=81=9C=E6=B1=A0=E5=88=B0=E4=B8=8D=E4=BA=86=20NAS=20?= =?UTF-8?q?[nas]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 09-02 巡查发现: sync_parquet_daily.sh TYPES 硬编码 dragon_tiger/block_trade/ margin_sse/restricted,今晚 19:30 A 档首夜产出(快照族 6 类+涨停池×3)不在拉取 清单,明晨同步后 NAS 回测侧依旧零文件(provider §16 出口在 NAS 断供)。 修: TYPES 扩到 13 类(+zt_pool×3/fund_flow×2/ths×2/xueqiu_hot/sina_sector, 7 天滚动窗逻辑不变);gdhs=per-period(季度末日期永不落 7 天窗)单独全量拉 (dir /b 枚举后逐文件 scp,≤10 文件幂等,周六 ak-weekly 产出)。 生效位=/volume1/stock(scripts/nas_sync/),CI 不覆盖,需手动部署 NAS;部署后 手动跑一轮验证=顺带把今晚首夜数据提前送 NAS(不等明晨 03:00)。 --- scripts/nas_sync/sync_parquet_daily.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/nas_sync/sync_parquet_daily.sh b/scripts/nas_sync/sync_parquet_daily.sh index 114764f..5ed71ca 100644 --- a/scripts/nas_sync/sync_parquet_daily.sh +++ b/scripts/nas_sync/sync_parquet_daily.sh @@ -7,8 +7,10 @@ # - per-date 类文件名含日期(YYYYMMDD_type.parquet),可按日期精确拉(scp 单文件,不扫目录)。 # - per-stock 类(valuation/三表/northbound 等)文件名无日期,走 sync_parquet.sh 周度全量。 # -# 覆盖:ak-events 每日产出(dragon_tiger/block_trade/margin_sse/restricted)。 +# 覆盖:ak-events 每日产出。2026-09-02 起扩到 13 类(原 4 类事件 + A 档 panel 9 类: +# zt_pool×3 + fund_flow×2 + ths×2 + xueqiu_hot + sina_sector,见活文档 §15)。 # 每日拉最近 7 天窗口(容错漏跑几天;节假日无文件=scp 失败=skip 正常)。 +# gdhs 为 per-period(季度末日期,永不落 7 天窗),单独全量拉(≤10 个文件,幂等)。 set -u KEY=/var/services/homes/admin/.ssh/id_ed25519_nas @@ -17,7 +19,7 @@ ROOT=/volume1/stock/sanguo_vnpy_v2 DATA=$ROOT/data LOG=$ROOT/data_backup/parquet_daily.log -TYPES="dragon_tiger block_trade margin_sse restricted" +TYPES="dragon_tiger block_trade margin_sse restricted zt_pool zt_pool_zbgc zt_pool_dtgc fund_flow_industry fund_flow_concept ths_industry ths_concept xueqiu_hot sina_sector" WINDOW_DAYS=7 TS=$(date '+%F %T') @@ -40,4 +42,17 @@ for type in $TYPES; do done done +# gdhs per-period 全量拉(季度末文件名不在滚动窗口内;周六 ak-weekly 产出,≤10 文件) +mkdir -p "$DATA/static/gdhs" +for f in $(ssh -i "$KEY" -o StrictHostKeyChecking=no \ + "$VPS" "dir /b C:\\sanguo_vnpy_v2\\data\\static\\gdhs\\*.parquet" 2>/dev/null); do + if scp -q -i "$KEY" -o StrictHostKeyChecking=no \ + "$VPS:C:/sanguo_vnpy_v2/data/static/gdhs/$f" \ + "$DATA/static/gdhs/" 2>/dev/null; then + PULLED=$((PULLED + 1)) + else + SKIPPED=$((SKIPPED + 1)) + fi +done + echo "[$(date '+%T')] DAILY DONE pulled=$PULLED skipped=$SKIPPED (skip=节假日/未出,正常)" >> "$LOG"