claude_dev
d76171434b
- sync_dbbardata.sh: STEP 2000万→200万/片(首片2GB export+scp跨公网挂) - merge_increment.py: 加 PRAGMA cache=500MB+WAL+NORMAL+synchronous (replica 2亿行 UNIQUE索引数GB,默认2MB cache致索引IO merge>120s超时) - 全量完成: replica 2.019亿行(=dbbardata全量日线+15min+ETF) - 踩坑链: ssh reset中断→timeout杀ssh留NAS merge残留持锁→since落后数据已在库 - 教训: timeout杀ssh不杀远程python;merge大表必加PRAGMA;断点续传since.txt救命 Co-Authored-By: Claude <noreply@anthropic.com>
92 lines
3.1 KiB
Bash
92 lines
3.1 KiB
Bash
#!/bin/bash
|
||
# NAS 端：VPS dbbardata 增量/全量同步（id 增量，rowid 索引快；VPS pull 方向）。
|
||
#
|
||
# 用法：
|
||
# 首次全量(分片,断点续传): bash sync_dbbardata.sh full
|
||
# 每日增量(定时任务): bash sync_dbbardata.sh increment
|
||
#
|
||
# 设计：
|
||
# - VPS 端脚本不落盘，通过 ssh stdin 喂 export_increment.py 执行（绕 Windows 部署）。
|
||
# - 全量按 id 分片(STEP 行/片)逐片导出→pull→merge，since.txt 记进度，中断重跑 full 自动续。
|
||
# - 增量：since=since.txt 的 max_id，导出 id>since，更新 since.txt。
|
||
# - merge 用 INSERT OR IGNORE，UNIQUE(symbol,exchange,interval,datetime) 去重，重跑安全。
|
||
set -eu
|
||
|
||
KEY=/var/services/homes/admin/.ssh/id_ed25519_nas
|
||
VPS=Administrator@49.232.102.198
|
||
VPS_DB='C:\sanguo_vnpy_v2\data\quant_trading.db'
|
||
VPS_OUT='C:\sanguo_vnpy_v2\data\_sync_inc.db'
|
||
VPS_OUT_SCP='C:/sanguo_vnpy_v2/data/_sync_inc.db'
|
||
VPS_PY='C:\Python310\python.exe -X utf8 -'
|
||
|
||
ROOT=/volume1/stock/sanguo_vnpy_v2
|
||
EXP=$ROOT/scripts/nas_sync/export_increment.py
|
||
MERGE=$ROOT/scripts/nas_sync/merge_increment.py
|
||
DB=$ROOT/data_backup/quant_trading.db
|
||
STAGE=$ROOT/data_backup/_staging
|
||
STATE=$ROOT/data_backup/since.txt
|
||
LOG=$ROOT/data_backup/sync.log
|
||
STEP=2000000 # 全量分片步长（行/片，~200MB/片；首版2000万/2GB致首片export+scp跨公网挂,降至此）
|
||
|
||
mkdir -p "$STAGE"
|
||
LOG_TS="$(date '+%F %T')"
|
||
echo "=== $LOG_TS MODE=${1:-increment} ===" >> "$LOG"
|
||
|
||
get_max() {
|
||
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" "$VPS_PY --db $VPS_DB --max-id" < "$EXP" \
|
||
| grep -oE 'MAX_ID=[0-9]+' | cut -d= -f2
|
||
}
|
||
|
||
export_slice() {
|
||
# $1=id_start $2=id_end(空=不限)
|
||
if [ -n "$2" ]; then
|
||
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" \
|
||
"$VPS_PY --db $VPS_DB --out $VPS_OUT --id-start $1 --id-end $2" < "$EXP"
|
||
else
|
||
ssh -i "$KEY" -o StrictHostKeyChecking=no "$VPS" \
|
||
"$VPS_PY --db $VPS_DB --out $VPS_OUT --id-start $1" < "$EXP"
|
||
fi
|
||
}
|
||
|
||
pull_merge() {
|
||
scp -i "$KEY" -o StrictHostKeyChecking=no "$VPS:$VPS_OUT_SCP" "$STAGE/inc.db" >> "$LOG" 2>&1
|
||
python3 "$MERGE" --db "$DB" --inc "$STAGE/inc.db" >> "$LOG" 2>&1
|
||
rm -f "$STAGE/inc.db"
|
||
}
|
||
|
||
CUR_MAX=$(get_max)
|
||
echo "[$LOG_TS] VPS max_id=$CUR_MAX" >> "$LOG"
|
||
|
||
case "${1:-increment}" in
|
||
full)
|
||
SINCE=$(cat "$STATE" 2>/dev/null || echo 0)
|
||
echo "[$LOG_TS] full since=$SINCE cur_max=$CUR_MAX step=$STEP" >> "$LOG"
|
||
while [ "$SINCE" -lt "$CUR_MAX" ]; do
|
||
END=$((SINCE + STEP))
|
||
[ "$END" -gt "$CUR_MAX" ] && END=$CUR_MAX
|
||
echo "[$(date '+%T')] slice id($SINCE,$END]" >> "$LOG"
|
||
export_slice "$SINCE" "$END"
|
||
pull_merge
|
||
echo "$END" > "$STATE"
|
||
SINCE=$END
|
||
done
|
||
echo "[$(date '+%T')] FULL DONE since=$SINCE" >> "$LOG"
|
||
;;
|
||
increment)
|
||
SINCE=$(cat "$STATE" 2>/dev/null || echo 0)
|
||
echo "[$LOG_TS] increment since=$SINCE -> $CUR_MAX" >> "$LOG"
|
||
if [ "$SINCE" -ge "$CUR_MAX" ]; then
|
||
echo "[$LOG_TS] up-to-date" >> "$LOG"
|
||
exit 0
|
||
fi
|
||
export_slice "$SINCE" ""
|
||
pull_merge
|
||
echo "$CUR_MAX" > "$STATE"
|
||
echo "[$(date '+%T')] INCREMENT DONE since=$CUR_MAX" >> "$LOG"
|
||
;;
|
||
*)
|
||
echo "usage: $0 {full|increment}" >&2
|
||
exit 1
|
||
;;
|
||
esac
|