65 lines
2.8 KiB
Bash
65 lines
2.8 KiB
Bash
#!/bin/bash
|
||
# NAS 端：VPS parquet 每日增量同步（仅 per-date 类，按日期精确拉，不扫 mtime）。
|
||
#
|
||
# 设计背景（2026-07-31 实测）：
|
||
# - VPS 机械盘扫 parquet 小文件 mtime 极慢（valuation+financial_abstract 11070 文件 >60s），
|
||
# bsdtar --newer 增量不可行（印证 phase3 当年弃 rsync 改 SQLite 增量的同一坑）。
|
||
# - per-date 类文件名含日期（YYYYMMDD_type.parquet），可按日期精确拉（scp 单文件，不扫目录）。
|
||
# - per-stock 类（valuation/三表/northbound 等）文件名无日期，走 sync_parquet.sh 周度全量。
|
||
#
|
||
# 覆盖：ak-events 每日产出。2026-09-02 起扩到 13 类(原 4 类事件 + A 档 panel 9 类:
|
||
# zt_pool×3 + fund_flow×2 + ths×2 + xueqiu_hot + sina_sector,见活文档 §15)。
|
||
# 每日拉最近 7 天窗口（容错漏跑几天；节假日无文件=scp 失败=skip 正常）。
|
||
# gdhs 为 per-period(季度末日期,永不落 7 天窗),单独全量拉(≤10 个文件,幂等)。
|
||
set -u
|
||
|
||
KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
|
||
VPS=Administrator@49.232.102.198
|
||
ROOT=/volume1/stock/sanguo_vnpy_v2
|
||
DATA=$ROOT/data
|
||
LOG=$ROOT/data_backup/parquet_daily.log
|
||
|
||
TYPES="dragon_tiger block_trade margin_sse restricted zt_pool zt_pool_zbgc zt_pool_dtgc fund_flow_industry fund_flow_concept ths_industry ths_concept xueqiu_hot sina_sector"
|
||
WINDOW_DAYS=7
|
||
|
||
TS=$(date '+%F %T')
|
||
echo "=== $TS parquet daily (per-date, ${WINDOW_DAYS}day window) ===" >> "$LOG"
|
||
|
||
PULLED=0
|
||
SKIPPED=0
|
||
for type in $TYPES; do
|
||
mkdir -p "$DATA/static/$type"
|
||
for i in $(seq 0 $((WINDOW_DAYS - 1))); do
|
||
d=$(date -d "$i days ago" +%Y%m%d)
|
||
file="${d}_${type}.parquet"
|
||
if scp -q -i "$KEY" -o StrictHostKeyChecking=no \
|
||
"$VPS:C:/sanguo_vnpy_v2/data/static/$type/$file" \
|
||
"$DATA/static/$type/" 2>/dev/null; then
|
||
PULLED=$((PULLED + 1))
|
||
else
|
||
SKIPPED=$((SKIPPED + 1))
|
||
fi
|
||
done
|
||
done
|
||
|
||
# gdhs per-period 全量拉(季度末文件名不在滚动窗口内;周六 ak-weekly 产出,≤10 文件)
|
||
mkdir -p "$DATA/static/gdhs"
|
||
# dir /b 输出带 CRLF:
|
||
粘在文件名尾 → scp 远端路径不存在 → 静默失败 6 天
|
||
# (09-02~09-08 NAS gdhs 空)。tr -d '
|
||
' 根治。
|
||
for f in $(ssh -i "$KEY" -o StrictHostKeyChecking=no \
|
||
"$VPS" "dir /b C:\\sanguo_vnpy_v2\\data\\static\\gdhs\\*.parquet" 2>/dev/null | tr -d '\r'); do
|
||
# 期文件 write-once(marker 锁定): 已存在即跳过, 补史后 53 文件每日全量重拉是浪费
|
||
[ -f "$DATA/static/gdhs/$f" ] && continue
|
||
if scp -q -i "$KEY" -o StrictHostKeyChecking=no \
|
||
"$VPS:C:/sanguo_vnpy_v2/data/static/gdhs/$f" \
|
||
"$DATA/static/gdhs/" 2>/dev/null; then
|
||
PULLED=$((PULLED + 1))
|
||
else
|
||
SKIPPED=$((SKIPPED + 1))
|
||
fi
|
||
done
|
||
|
||
echo "[$(date '+%T')] DAILY DONE pulled=$PULLED skipped=$SKIPPED (skip=节假日/未出,正常)" >> "$LOG"
|