feat(ops): NAS→VPS 数据同步脚本(Mac中继,符dev→prod)
sync_nas_to_vps.sh: 同步质量验证日线(raw+qfq parquet)NAS→VPS via Mac (光猫拦直连)。符号可配置(默认模拟账户持仓600000/000001),幂等(rsync --update), 前缀规则对齐 bridge_client.to_bridge_code。已验证:24文件落地VPS结构正确。 验证通过→Mac cron每日收盘后跑=生产数据管道。
This commit is contained in:
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
# sync_nas_to_vps.sh — 同步质量验证的日线数据(raw+qfq parquet)NAS→VPS,经 Mac 中继。
|
||||
#
|
||||
# 为什么走 Mac:光猫拦 NAS→VPS 直连;NAS 无公网,VPS 拉不到 NAS。
|
||||
# Mac 同处 LAN(可达 NAS)+ 可达 VPS(公网),做中继。脚本在 Mac Mini 跑。
|
||||
#
|
||||
# NAS = 数据权威(staging→verify→merge 后的 raw/qfq);VPS = 实盘生产,只取策略标的。
|
||||
# 数据布局:{raw,qfq}/{year}/{sh|sz}{symbol}_daily.parquet
|
||||
#
|
||||
# 用法: ./sync_nas_to_vps.sh [symbol ...] 默认: 模拟账户持仓 600000 000001
|
||||
# 幂等:rsync --update 仅传变化;scp 覆盖(年度 parquet 文件稳定)。
|
||||
# 验证通过后:Mac cron 每日跑(收盘后)→ VPS 数据每日更新。
|
||||
set -euo pipefail
|
||||
|
||||
NAS_HOST="sanguo-nas"
|
||||
NAS_BASE="/volume1/stock/A股数据/日线数据"
|
||||
VPS_HOST="Administrator@49.232.102.198"
|
||||
VPS_KEY="${VPS_KEY:-$HOME/.ssh/id_ed25519}"
|
||||
VPS_DATA="C:/sanguo_vnpy_v2/data"
|
||||
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# 前缀规则(对齐 sanguo_trader.bridge_client.to_bridge_code,避免在脚本里 import vnpy):
|
||||
# 沪 sh: 60/68/51/56/58 开头 深 sz: 00/30/15 开头 默认 sh
|
||||
prefix_for() {
|
||||
local s="$1"
|
||||
case "${s:0:2}" in 60|68|51|56|58) echo sh;; 00|30|15) echo sz;; *) echo sh;; esac
|
||||
}
|
||||
|
||||
SYMBOLS=("$@"); [[ ${#SYMBOLS[@]} -eq 0 ]] && SYMBOLS=(600000 000001)
|
||||
|
||||
echo "[sync] symbols: ${SYMBOLS[*]} NAS=${NAS_HOST} VPS=${VPS_HOST}"
|
||||
total=0
|
||||
for sym in "${SYMBOLS[@]}"; do
|
||||
pre=$(prefix_for "$sym")
|
||||
for kind in raw qfq; do
|
||||
mkdir -p "$TMP/$kind"
|
||||
if ! rsync -az --include='*/' --include="${pre}${sym}_daily.parquet" --exclude='*' \
|
||||
"${NAS_HOST}:${NAS_BASE}/${kind}/" "$TMP/$kind/" >/dev/null 2>&1; then
|
||||
echo "[sync] $sym $kind: NAS rsync FAIL"; continue
|
||||
fi
|
||||
n=$(find "$TMP/$kind" -name '*.parquet' | wc -l | tr -d ' ')
|
||||
if [[ "$n" -eq 0 ]]; then echo "[sync] $sym $kind: NAS 无文件(前缀或标的错?)"; rm -rf "$TMP/$kind"; continue; fi
|
||||
if scp -r -q -i "$VPS_KEY" "$TMP/$kind/"* "$VPS_HOST:${VPS_DATA}/${kind}/" >/dev/null 2>&1; then
|
||||
echo "[sync] $sym $kind: $n files OK"; total=$((total+n))
|
||||
else
|
||||
echo "[sync] $sym $kind: VPS scp FAIL"
|
||||
fi
|
||||
rm -rf "$TMP/$kind"
|
||||
done
|
||||
done
|
||||
echo "[sync] DONE, total $total files → ${VPS_DATA}/{raw,qfq}/"
|
||||
Reference in New Issue
Block a user