claude_dev
b2aab85105
新增 export_table.py/merge_table.py/sync_tables.sh: VPS stdin喂python全量dump → scp → NAS全量替换(DROP+CREATE+INSERT,幂等可重跑)。sync_dbbardata.sh 每日increment末尾连带跑 sync_tables,保持NAS副本成份股/复权因子不陈旧。修复NAS组合回测选股恒空(NAS副本缺constituent_unified表,phase3只导dbbardata)。
98 lines
3.5 KiB
Bash
98 lines
3.5 KiB
Bash
#!/bin/bash
|
||
# NAS 端：VPS dbbardata 增量/全量同步（id 增量，rowid 索引快；VPS pull 方向）。
|
||
#
|
||
# 用法：
|
||
# 首次全量(分片,断点续传): bash sync_dbbardata.sh full
|
||
# 每日增量(定时任务): bash sync_dbbardata.sh increment
|
||
#
|
||
# 设计：
|
||
# - VPS 端脚本不落盘，通过 ssh stdin 喂 export_increment.py 执行（绕 Windows 部署）。
|
||
# - 全量按 id 分片(STEP 行/片)逐片导出→pull→merge，since.txt 记进度，中断重跑 full 自动续。
|
||
# - 增量：since=since.txt 的 max_id，导出 id>since，更新 since.txt。
|
||
# - merge 用 INSERT OR IGNORE，UNIQUE(symbol,exchange,interval,datetime) 去重，重跑安全。
|
||
set -eu
|
||
|
||
KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
|
||
VPS=Administrator@49.232.102.198
|
||
VPS_DB='C:\sanguo_vnpy_v2\data\quant_trading.db'
|
||
VPS_OUT='C:\sanguo_vnpy_v2\data\_sync_inc.db'
|
||
VPS_OUT_SCP='C:/sanguo_vnpy_v2/data/_sync_inc.db'
|
||
VPS_PY='C:\Python310\python.exe -X utf8 -'
|
||
|
||
ROOT=/volume1/stock/sanguo_vnpy_v2
|
||
EXP=$ROOT/scripts/nas_sync/export_increment.py
|
||
MERGE=$ROOT/scripts/nas_sync/merge_increment.py
|
||
DB=$ROOT/data_backup/quant_trading.db
|
||
STAGE=$ROOT/data_backup/_staging
|
||
STATE=$ROOT/data_backup/since.txt
|
||
LOG=$ROOT/data_backup/sync.log
|
||
STEP=2000000 # 全量分片步长（行/片，~200MB/片；首版2000万/2GB致首片export+scp跨公网挂,降至此）
|
||
|
||
mkdir -p "$STAGE"
|
||
LOG_TS="$(date '+%F %T')"
|
||
echo "=== $LOG_TS MODE=${1:-increment} ===" >> "$LOG"
|
||
|
||
get_max() {
|
||
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" "$VPS_PY --db $VPS_DB --max-id" < "$EXP" \
|
||
| grep -oE 'MAX_ID=[0-9]+' | cut -d= -f2
|
||
}
|
||
|
||
export_slice() {
|
||
# $1=id_start $2=id_end(空=不限)
|
||
if [ -n "$2" ]; then
|
||
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" \
|
||
"$VPS_PY --db $VPS_DB --out $VPS_OUT --id-start $1 --id-end $2" < "$EXP"
|
||
else
|
||
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" \
|
||
"$VPS_PY --db $VPS_DB --out $VPS_OUT --id-start $1" < "$EXP"
|
||
fi
|
||
}
|
||
|
||
pull_merge() {
|
||
scp -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$VPS_OUT_SCP" "$STAGE/inc.db" >> "$LOG" 2>&1
|
||
python3 "$MERGE" --db "$DB" --inc "$STAGE/inc.db" >> "$LOG" 2>&1
|
||
rm -f "$STAGE/inc.db"
|
||
}
|
||
|
||
CUR_MAX=$(get_max)
|
||
echo "[$LOG_TS] VPS max_id=$CUR_MAX" >> "$LOG"
|
||
|
||
case "${1:-increment}" in
|
||
full)
|
||
SINCE=$(cat "$STATE" 2>/dev/null || echo 0)
|
||
echo "[$LOG_TS] full since=$SINCE cur_max=$CUR_MAX step=$STEP" >> "$LOG"
|
||
while [ "$SINCE" -lt "$CUR_MAX" ]; do
|
||
END=$((SINCE + STEP))
|
||
[ "$END" -gt "$CUR_MAX" ] && END=$CUR_MAX
|
||
echo "[$(date '+%T')] slice id($SINCE,$END]" >> "$LOG"
|
||
export_slice "$SINCE" "$END"
|
||
pull_merge
|
||
echo "$END" > "$STATE"
|
||
SINCE=$END
|
||
done
|
||
echo "[$(date '+%T')] FULL DONE since=$SINCE" >> "$LOG"
|
||
;;
|
||
increment)
|
||
SINCE=$(cat "$STATE" 2>/dev/null || echo 0)
|
||
echo "[$LOG_TS] increment since=$SINCE -> $CUR_MAX" >> "$LOG"
|
||
if [ "$SINCE" -ge "$CUR_MAX" ]; then
|
||
echo "[$LOG_TS] up-to-date" >> "$LOG"
|
||
exit 0
|
||
fi
|
||
export_slice "$SINCE" ""
|
||
pull_merge
|
||
echo "$CUR_MAX" > "$STATE"
|
||
echo "[$(date '+%T')] INCREMENT DONE since=$CUR_MAX" >> "$LOG"
|
||
;;
|
||
*)
|
||
echo "usage: $0 {full|increment}" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
|
||
# 小表全量同步(constituent_unified 成份股 / bs_adjust_factor 复权因子:
|
||
# 月度或除权日才变,万级行,全量几秒)。独立 log(sync_tables.log),
|
||
# 失败不影响 dbbardata 主流程。每日 increment 连带跑,保持 NAS 副本不陈旧。
|
||
bash "$(dirname "$0")/sync_tables.sh" >> "$LOG" 2>&1 \
|
||
|| echo "[$(date '+%T')] WARN sync_tables 非致命" >> "$LOG"
|