claude_dev
774170ec05
采集层(多源各下): - baostock: 日线全字段全量(baostock_daily_fullmarket) + 15min全市场 + 静态(基础/复权/分红/季频/三表) + 成份股 - akshare: 静态(估值/龙虎榜/大宗/融资融券/北向/指数成分/行业/股本/解禁/业绩预告) - xtdata(miniQMT): build_daily_from_xtdata + daily_update_xtdata 数据补全 P0: - ETF全市场: universe 扩展 沪深A股∪ETF∪基金(7414), dividend_type='front' 前复权 - 历史成份股(治幸存者偏差): index_const_hist_download 深证/国证 adjust_cni 4指数 + 中证1000/2000快照 + 新浪交叉校验 - 退市K线: baostock_delisted_download + import_delisted_to_db(实证 Day1 fetch_all_stocks 已含退市) 灌库: - import_baostock_to_db: daily_baostock_full(5537股/1826万行,18字段)+ bs_index_constituent + bs_adjust_factor - INSERT OR REPLACE 幂等, WAL+busy_timeout, dbbardata 不碰 每日增量 #7(用户决策A: VPS直跑): - daily_update_static: login探针防黑名单graceful skip + LOOKBACK7 + query_stock_basic含退市 + INSERT OR REPLACE + QUERY_COUNT守48000/天 设计文档: spec(13节三层融合) + P0 plan + 数据gap设计
36 lines
1.7 KiB
PowerShell
36 lines
1.7 KiB
PowerShell
# baostock Day1 全量下载 wrapper（成份股 → 日K线 串行，单进程单登录）
|
||
# 环境变量覆盖脚本默认 Mac 路径 → Windows 输出路径
|
||
$ErrorActionPreference = 'Continue'
|
||
$ProgressPreference = 'SilentlyContinue'
|
||
$py = 'C:\Python310\python.exe'
|
||
$dir = 'C:\sanguo_vnpy_v2\scripts\data_platform'
|
||
$env:BS_DAILY_OUT_DIR = 'C:\sanguo_vnpy_v2\data\daily_baostock'
|
||
$env:BS_CONSTITUENT_OUT_DIR = 'C:\sanguo_vnpy_v2\data\constituent_baostock'
|
||
$env:PYTHONIOENCODING = 'utf-8'
|
||
$wrapper_log = 'C:\sanguo_vnpy_v2\data\baostock_day1_wrapper.log'
|
||
$const_log = 'C:\sanguo_vnpy_v2\data\constituent_baostock_download.log'
|
||
$daily_log = 'C:\sanguo_vnpy_v2\data\daily_baostock_download.log'
|
||
|
||
function Log-W($m) {
|
||
"$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $m" | Out-File -FilePath $wrapper_log -Append -Encoding UTF8
|
||
}
|
||
|
||
Log-W "=== WRAPPER START ==="
|
||
Log-W "BS_DAILY_OUT_DIR=$($env:BS_DAILY_OUT_DIR)"
|
||
Log-W "DiskFree_GB_start=$((Get-PSDrive C).Free / 1GB)"
|
||
|
||
# PHASE1 成份股历史（快，~20min，~1200 query）— 同时验证 VPS 环境(login/网络/parquet写)
|
||
Log-W "PHASE1 constituent START"
|
||
& $py -X utf8 "$dir\baostock_constituent_download.py" *>&1 | Tee-Object -FilePath $const_log
|
||
$const_exit = $LASTEXITCODE
|
||
Log-W "PHASE1 constituent DONE exit=$const_exit"
|
||
|
||
# PHASE2 日K线全市场全字段（慢，~15h，~5537 query）— 主线，过夜跑
|
||
Log-W "PHASE2 daily_kline START"
|
||
& $py -X utf8 "$dir\baostock_daily_fullmarket_download.py" *>&1 | Tee-Object -FilePath $daily_log
|
||
$daily_exit = $LASTEXITCODE
|
||
Log-W "PHASE2 daily_kline DONE exit=$daily_exit"
|
||
|
||
Log-W "DiskFree_GB_end=$((Get-PSDrive C).Free / 1GB)"
|
||
Log-W "=== WRAPPER EXIT const_exit=$const_exit daily_exit=$daily_exit ==="
|