From 774170ec05c56f5a53bdbeda0e64cd5f91093987 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 22 Jul 2026 10:34:22 +0800 Subject: [PATCH] =?UTF-8?q?feat(data):=20=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E8=9E=8D=E5=90=88=20P0=20=E8=A1=A5=E5=85=A8=20+=20=E6=AF=8F?= =?UTF-8?q?=E6=97=A5=E5=A2=9E=E9=87=8F=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 采集层(多源各下): - baostock: 日线全字段全量(baostock_daily_fullmarket) + 15min全市场 + 静态(基础/复权/分红/季频/三表) + 成份股 - akshare: 静态(估值/龙虎榜/大宗/融资融券/北向/指数成分/行业/股本/解禁/业绩预告) - xtdata(miniQMT): build_daily_from_xtdata + daily_update_xtdata 数据补全 P0: - ETF全市场: universe 扩展 沪深A股∪ETF∪基金(7414), dividend_type='front' 前复权 - 历史成份股(治幸存者偏差): index_const_hist_download 深证/国证 adjust_cni 4指数 + 中证1000/2000快照 + 新浪交叉校验 - 退市K线: baostock_delisted_download + import_delisted_to_db(实证 Day1 fetch_all_stocks 已含退市) 灌库: - import_baostock_to_db: daily_baostock_full(5537股/1826万行,18字段)+ bs_index_constituent + bs_adjust_factor - INSERT OR REPLACE 幂等, WAL+busy_timeout, dbbardata 不碰 每日增量 #7(用户决策A: VPS直跑): - daily_update_static: login探针防黑名单graceful skip + LOOKBACK7 + query_stock_basic含退市 + INSERT OR REPLACE + QUERY_COUNT守48000/天 设计文档: spec(13节三层融合) + P0 plan + 数据gap设计 --- docs/portfolio_local_data_gaps.md | 82 ++ docs/static_data_cache_plan.md | 105 ++ docs/static_data_gaps_design.md | 46 + .../plans/2026-07-21-data-fusion-p0.md | 221 ++++ .../2026-07-21-data-source-fusion-design.md | 243 +++++ .../data_platform/akshare_static_download.py | 983 ++++++++++++++++++ .../baostock_15min_fullmarket_download.py | 576 ++++++++++ .../baostock_15min_sample_to_db.py | 551 ++++++++++ .../baostock_constituent_download.py | 496 +++++++++ .../baostock_daily_fullmarket_download.py | 582 +++++++++++ .../data_platform/baostock_day1_wrapper.ps1 | 35 + .../data_platform/baostock_day2a_wrapper.ps1 | 24 + .../data_platform/baostock_day2b_wrapper.ps1 | 23 + .../baostock_delisted_download.py | 443 ++++++++ .../data_platform/baostock_static_download.py | 795 ++++++++++++++ .../data_platform/build_daily_from_xtdata.py | 6 +- scripts/data_platform/daily_update_static.py | 333 ++++++ .../daily_update_static_wrapper.ps1 | 12 + scripts/data_platform/daily_update_xtdata.py | 6 +- .../data_platform/import_baostock_to_db.py | 387 +++++++ scripts/data_platform/import_bs_wrapper.ps1 | 14 + .../data_platform/import_delisted_to_db.py | 187 ++++ .../data_platform/import_vnpy_minute_fast.py | 3 +- .../index_const_hist_download.py | 627 +++++++++++ .../index_const_hist_wrapper.ps1 | 31 + .../data_platform/merge_staging_to_data.py | 61 ++ .../data_platform/probe_delisted_baostock.py | 185 ++++ scripts/data_platform/probe_delisted_ext.py | 91 ++ .../probe_early_delisted_kline.py | 41 + scripts/data_platform/probe_etf.py | 37 + scripts/data_platform/probe_etf_v2.py | 57 + .../test_baostock_daily_constituent_sample.py | 215 ++++ scripts/data_platform/test_mootdx_depth.py | 124 +++ .../data_platform/verify_delisted_in_db.py | 18 + scripts/data_platform/verify_etf_data.py | 50 + scripts/data_platform/verify_etf_staging.py | 75 ++ 36 files changed, 7762 insertions(+), 3 deletions(-) create mode 100644 docs/portfolio_local_data_gaps.md create mode 100644 docs/static_data_cache_plan.md create mode 100644 docs/static_data_gaps_design.md create mode 100644 docs/superpowers/plans/2026-07-21-data-fusion-p0.md create mode 100644 docs/superpowers/specs/2026-07-21-data-source-fusion-design.md create mode 100644 scripts/data_platform/akshare_static_download.py create mode 100644 scripts/data_platform/baostock_15min_fullmarket_download.py create mode 100644 scripts/data_platform/baostock_15min_sample_to_db.py create mode 100755 scripts/data_platform/baostock_constituent_download.py create mode 100755 scripts/data_platform/baostock_daily_fullmarket_download.py create mode 100644 scripts/data_platform/baostock_day1_wrapper.ps1 create mode 100644 scripts/data_platform/baostock_day2a_wrapper.ps1 create mode 100644 scripts/data_platform/baostock_day2b_wrapper.ps1 create mode 100644 scripts/data_platform/baostock_delisted_download.py create mode 100644 scripts/data_platform/baostock_static_download.py create mode 100644 scripts/data_platform/daily_update_static.py create mode 100644 scripts/data_platform/daily_update_static_wrapper.ps1 create mode 100644 scripts/data_platform/import_baostock_to_db.py create mode 100644 scripts/data_platform/import_bs_wrapper.ps1 create mode 100644 scripts/data_platform/import_delisted_to_db.py create mode 100644 scripts/data_platform/index_const_hist_download.py create mode 100644 scripts/data_platform/index_const_hist_wrapper.ps1 create mode 100644 scripts/data_platform/merge_staging_to_data.py create mode 100644 scripts/data_platform/probe_delisted_baostock.py create mode 100644 scripts/data_platform/probe_delisted_ext.py create mode 100644 scripts/data_platform/probe_early_delisted_kline.py create mode 100644 scripts/data_platform/probe_etf.py create mode 100644 scripts/data_platform/probe_etf_v2.py create mode 100755 scripts/data_platform/test_baostock_daily_constituent_sample.py create mode 100644 scripts/data_platform/test_mootdx_depth.py create mode 100644 scripts/data_platform/verify_delisted_in_db.py create mode 100644 scripts/data_platform/verify_etf_data.py create mode 100644 scripts/data_platform/verify_etf_staging.py diff --git a/docs/portfolio_local_data_gaps.md b/docs/portfolio_local_data_gaps.md new file mode 100644 index 0000000..8bb5b49 --- /dev/null +++ b/docs/portfolio_local_data_gaps.md @@ -0,0 +1,82 @@ +# LocalParquetProvider V1 数据缺口记录 + +> V1 已通过 VPS 真实数据验证(2026-07-21): +> fundamentals 字段值合理(茅台市值 18433亿/PE 23.6/ROE 0.19/净利率 0.51)、 +> get_price/get_index_stocks/trade_days/all_securities 全通、B_mean 趋势信号正常、 +> 回测出完整 JSON(117 交易日, 0.4s/月, 无 baostock 卡死)。 +> +> **0 交易根因(非 provider bug)**: `_pick_big_universe` 选股 target=[] +> = `big` filter 8 条件 AND 过严 + `roic_big` 用 roic(V1 NaN) + bm market_cap 100-900亿 +> 不匹配 hs300 大盘 + B_mean<0 时兜底海外 ETF(无 K 线)。补 roic + 调 filter 阈值即出交易。 +> +> 以下缺口不阻塞 MVP 链路验证,但全市场正式回测前需补齐。 + +## 缺口 1: 历史成分股(治幸存者偏差,重要 ⚠️) + +**现状**: VPS `static/index_const/index_const.parquet` 仅 **2026-07-17 最新一期**快照。 +`get_index_stocks(index, date)` 的 `date` 参数当前被忽略(无历史数据可读)。 + +**影响**: 回测 2020 年选股池 = "现在还在 hs300/zz500 里的股票" → 幸存者偏差(结果虚高)。 +`max_pool` 小范围验证影响相对小(只取前 N 只),**全市场轮动回测前必须补**。 + +**补齐方案**(任选,不用 baostock online): +- akshare `index_stock_cons_csindex(symbol="000300")` 按调仓日拉历史成分(csindex 源) +- 中证指数官网 csindex.com.cn 历史成分下载 +- 用户侧(数据补全 session)补到 `static/index_const_history/` 多期 parquet, provider 加日期过滤 + +## 缺口 2: gross_profit_margin(V1 NaN) + +**现状**: akshare income 表无明确"营业成本(COGS)"列(有 OPERATE_INCOME 营收、OPERATE_EXPENSE 营业总成本,但非纯 COGS)。 +V1 `gross_profit_margin` 置 NaN,策略 filter 该阈值失效(不过滤毛利率)。 + +**补齐方案**: 从 `static/financial_abstract/{code}_*.parquet` 读现成"销售毛利率" +(宽表 指标×季度,含 1990-2026)。解析:找指标行"销售毛利率",取最新季度列。 + +## 缺口 3: roic(V1 NaN) + +**现状**: ROIC = NOPAT / (权益 + 有息负债 - 现金),需有息负债拆分。 +V1 置 NaN。balance 表有 BORROW_FUND/BOND_PAYABLE 等字段可算。 + +**补齐方案**: balance 读 BORROW_FUND(短期借款) + BOND_PAYABLE(应付债券) + SUBBOND_PAYABLE ++ 现金(CASH_DEPOSIT_PBC 附近字段),算 roic。NOPAT = 营业利润 ×(1 - 税率)。 + +## V1 单位口径备忘(VPS 实测验证合理 ✅) + +| 字段 | VPS 源单位 | 转换 | 验证值(2024-06) | +|---|---|---|---| +| market_cap | 总市值(元) | /1e8 转亿 | 茅台 18433 亿 ✅ | +| circulating_market_cap | 流通市值(元) | /1e8 | ✅ | +| pe_ratio | PE(TTM) 数值 | 直接 | 茅台 23.6 ✅ | +| pb_ratio | 市净率 数值 | 直接 | 茅台 7.69 ✅ | +| ps_ratio/pcf_ratio | 市销率/市现率 | 直接 | ✅ | +| eps | BASIC_EPS 元 | 直接 | 茅台 33.19 ✅ | +| roe | 归母净利润/归母权益 | 小数(单期非TTM) | 茅台 0.19 ✅ | +| roa | 净利润/总资产 | 小数 | ✅ | +| net_profit_margin | 归母净利润/营收 | 小数 | 茅台 0.51 ✅ | +| inc_revenue_yoy | OPERATE_INCOME_YOY 百分数 | /100 | 茅台 +0.18 ✅ | +| total_liability | TOTAL_LIABILITIES 元 | /1e8 | 浦发 85000 亿 ✅ | +| total_sheet_owner_equities | TOTAL_PARENT_EQUITY 元 | /1e8 | ✅ | +| retained_profit | SURPLUS_RESERVE+UNASSIGN_RPOFIT | /1e8 | ✅ | + +## VPS 两种代码格式(已适配,备忘) + +VPS `data/` 下代码格式**不统一**: +- **K 线** `qfq/{年}/` `raw/{年}/`: baostock 风格 `sh600000_daily.parquet`(sh/sz 前缀无点) + → `jq_to_kline_code("600000.XSHG") = "sh600000"` +- **三表/估值** `static/{table}/`: jq 后缀 `000001.SZ_balance.parquet` + → `jq_to_file_code("000001.XSHE") = "000001.SZ"` + +`get_price` 用 `jq_to_kline_code`,`get_fundamentals_df` 用 `jq_to_file_code`。 + +## V1 已验证可用的接口 + +| 方法 | 状态 | 备注 | +|---|---|---| +| get_price | ✅ | qfq 日线, 单股 index=date / 多股 panel | +| get_fundamentals_df | ✅ | 19 列对齐 _FUNDAMENTAL_COLUMNS, 字段值合理 | +| get_security_info | ✅ | valuation 最新行 | +| get_trade_days | ✅ | sh600000 K 线 date 列 | +| get_all_securities | ✅ | 5528 股 | +| get_index_stocks | ⚠️ | 仅当前快照(缺口 1) | +| get_current_tick | ✅ | valuation 推算 close + 涨跌停(主板±10%) | +| get_split_dividend | ✅ | 占位返空(qfq 已复权) | diff --git a/docs/static_data_cache_plan.md b/docs/static_data_cache_plan.md new file mode 100644 index 0000000..d5daded --- /dev/null +++ b/docs/static_data_cache_plan.md @@ -0,0 +1,105 @@ +# A股静态数据全量缓存到 VPS — 设计与采集计划 + +> 2026-07-19 立。目标:全市场 A股静态/基本面/参考数据全量缓存到 VPS 本地(parquet),作选股(基本面)与回测数据源。 + +## 原则(用户钦定) +1. **尽量多缓存**——能下的全下,避免限流/网络依赖。 +2. **串行可,等待长可接受**——不追求并发速度,稳定性优先。 +3. **准确性第一**——下错不如不下,每类数据必须验证。 +4. **建立每日自动更新**——历史一次灌满 + 每日增量。 +5. 用途:基本面选股 + 回测。盘中实时当日数据(实盘)未来再做。 + +## 为什么缓存优于实时取(背景) +历史静态数据(含"日频"的历史部分)永不改变。本地缓存:秒级读盘/零网络依赖/可复现快照/不触发限流。实时取历史:慢/不稳/不可复现/反复触发封 IP。**唯一非静态是"今天未收盘/未公布"的部分,日终收盘后即变静态。** + +## 范围(全量 ~2GB) +| 组 | 类别 | 频率 | 源 | 估算 | +|---|---|---|---|---| +| A 基础元数据 | 基础信息(代码/名称/交易所/板块/上市退市/状态) | 静态 | baostock | 1MB | +| | 行业分类(申万/中信/概念) | 静态 | akshare | 30MB | +| | 指数成分+权重(300/500/1000/国证2000) | 月 | akshare | 50MB | +| B 财务 | 三大报表(资产/利润/现金流) | 季 | baostock | 150MB | +| | 季频衍生指标(ROE/EPS/毛利率/净利增速/负债率/杜邦) | 季 | baostock | 200MB | +| | 业绩预告/快报 | 季事件 | akshare | 30MB | +| C 股本/公司行为 | 股本结构变动 | 事件 | akshare | 50MB | +| | 十大股东+十大流通股东 | 季 | akshare | 250MB | +| | 分红送转配股 | 事件 | baostock | 40MB | +| | 限售解禁 | 事件 | akshare | 20MB | +| D 估值/复权(日频) | 估值快照(PE/PB/PS/PF/股息率/市值/流通市值) | 日 | akshare | 300MB | +| | 复权因子(qfq/hfq) | 日 | baostock | 150MB | +| E 市场参考(日频,可选) | 龙虎榜/大宗交易/融资融券/北向/ST停复牌 | 日 | akshare | ~360MB | + +## 存储格式与目录 +**parquet(每类一个目录)**,VPS `C:\sanguo_vnpy_v2\data\static\\`。匹配现有 15min/daily parquet 模式,回测 pandas 直读,增量 append/overwrite 幂等。 +``` +data/static/ + basic/ 基础信息(全量刷新,1文件 or per-stock) + industry/ 行业分类 + index_const/ 指数成分 + balance/ 资产负债表(per-stock parquet) + income/ 利润表 + cashflow/ 现金流量表 + indicator/ 季频财务指标(ROE/EPS/...) + forecast/ 业绩预告/快报 + share_capital/ 股本结构 + top_holders/ 十大股东 + dividend/ 分红送转 + lockup_release/ 限售解禁 + valuation/ 估值日频(PE/PB/市值) + adjust_factor/ 复权因子 + dragon_tiger/ 龙虎榜 + block_trade/ 大宗交易 + margin/ 融资融券 + northbound/ 北向资金 +``` + +## 数据源映射 + 串行约束(关键) +| 源 | 数据 | 并发约束 | +|---|---|---| +| **baostock** | 基础信息/复权因子/分红/季频指标/三表 | **单登录串行,跟15min共用登录→必须等15min跑完才能开**(并发=IP封6-24h) | +| **akshare** | 估值/龙虎榜/大宗/融资融券/北向/指数成分/行业/股本/十大股东/解禁/业绩预告 | 不同源,可与baostock错峰;东财源要限速防反爬 | + +## 准确性协议(每类数据强制) +1. **断点续传 marker + 失败/空数据区分**(复用 15min 的 empty-vs-failed 修复) +2. **下载后抽样验证(≥10只)**:字段完整 / 日期覆盖(回溯到2020) / 值合理性(PE>0、ROE合理区间、volume≥0、OHLC 自洽) +3. **行数 + 覆盖率统计**写入日志 +4. **幂等写入**(per-stock parquet overwrite;INSERT OR REPLACE 若入 DB),staging 隔离→验证→合并(用户铁律:绝不直写主库/主目录未验) +5. (可选)跨源抽检:baostock 季频财务 vs miniQMT PershareIndex 抽几只对一对 + +## 每日自动更新机制 +Windows schtask `sanguo-static-daily`,每日盘后 **16:30** 跑 `daily_update_static.py`: +- **日频类(估值/龙虎榜/大宗/融资融券/北向)**:追加当日(或近N日补漏) +- **小表全量刷新**:基础信息/行业/指数成分/分红(事件少,全量省得算增量) +- **财报季(4/8/10月底后)**:追加新季报(三表/季频/十大股东) +- **复权因子**:每日刷新(除权事件会改累计因子) +- 失败告警 + 断点续传 + **绝不破坏既有数据**(只 append/replace 单日单股) +- 串行 baostock 部分 + 限速 akshare 部分,单进程跑完 + +## 执行阶段 +- **Phase 0(进行中)**:15min baostock,ETA 07-19 ~22:30 +- **Phase 1**:baostock 静态下载脚本(基础/复权/分红/季频/三表)— **构建 now,运行须等15min完** +- **Phase 2**:akshare 静态下载脚本(估值/龙虎榜/大宗/融资融券/北向/指数成分/行业/股本/十大股东/解禁/业绩预告)— **构建+可now起**(不同源) +- **Phase 3**:每类抽样验证 → 修问题 +- **Phase 4**:每日更新 schtask + 验证增量 + +## akshare 调查结果(2026-07-19 确认,15类端点实证) +**15类中 11 类端点直接可用,4 类有替代。财务类全部"单股一次拉全历史"(完美绕开 baostock per-quarter 百万调用)。全量 ~2.5-3GB,单线程 17-22h(可挂机,不同源可与 baostock 并行)。** + +确认端点(4种模式): +- **per-stock(5500股×1调用)**:估值`stock_value_em` / 北向`stock_hsgt_individual_em` / 股本`stock_share_change_cninfo` / 十大流通`stock_gdfx_free_top_10_em`(×20报告期) / **三大报表`stock_balance/profit/cash_flow_sheet_by_report_em`(319/203/254列全历史)** / 财务摘要`stock_financial_abstract` +- **per-date(交易日×1调用)**:龙虎榜`stock_lhb_detail_em` / 大宗`stock_dzjy_mrmx(symbol="A股")` / 融资融券沪`stock_margin_detail_sse` / 解禁`stock_restricted_release_detail_em` +- **per-period(报告期×1调用)**:业绩预告`stock_yjyg_em` / 业绩快报`stock_yjkb_em` +- **one-shot**:指数成分`index_stock_cons_csindex`(300/500/1000) / 行业`sw_index_first_info`(申万,东财`stock_board_industry_name_em`ConnectionError 弃用) + +**有问题/替代**:`stock_margin_detail_szse`(深融资融券)超时频繁→先跳过;`stock_gdfx_holding_detail_em(date)`按日全市场超时→改个股循环;`stock_a_indicator_lg`新版删→用`stock_value_em`。 + +**大小明细**:估值150MB / 三大报表1.2GB / 财务摘要300MB / 北向120MB / 融资融券500MB / 十大流通60MB / 其余<100MB各。**总~2.5-3GB**。 + +**优先级**:P0 三大报表+财务摘要(~1.5GB,~10h,核心财务)→ P1 估值+北向+融资融券(~770MB,~3h)→ P2 龙虎榜/大宗/解禁/业绩预告/股本/指数/行业(~300MB,~1h)。 + +## 部署架构(自愈链,2026-07-19) +- **15min baostock**:schtask `sanguo-bs15min` + 自愈.bat(ping-sleep 30min重试)。2026-07-19 12:00 baostock全球故障(Mac+VPS同挂10002007),自愈中,恢复即续 from marker 2847。 +- **baostock静态**:schtask `sanguo-bs-static` + 自愈.bat(**wait15**等15min "ALL DONE" → 自动接力 → ping-sleep自愈)。脚本`baostock_static_download.py`已部署(basic/adjust_factor/dividend,rs.fields动态取字段)。 +- **akshare静态**:schtask `sanguo-bs-akshare`(待建)+ 自愈.bat。脚本构建中。**不同源,可与baostock并行**。 +- 监控:cron 779cbb71 每30min probe_all + 异常自修 + 完成报告。caffeinate防睡眠。 +- **.bat sleep 用 `ping -n N 127.0.0.1`**(timeout.exe在SYSTEM schtask下失效,见 memory schtasks-system-bat-gotchas)。 diff --git a/docs/static_data_gaps_design.md b/docs/static_data_gaps_design.md new file mode 100644 index 0000000..bef75c1 --- /dev/null +++ b/docs/static_data_gaps_design.md @@ -0,0 +1,46 @@ +# 静态数据 3 个真缺口 — 补充设计(2026-07-19 记录,待当前下载完再动手) + +> 当前下载(15min Mac补全 + akshare static + baostock static)跑完后再继续本设计。先记录存底。 + +## 数据现状实测(VPS quant_trading.db + data/ 目录,非推理) +- **DB dbbardata**: 15m(2025-07~2026-07,1年,xt_tacitdata源)/5m(1年)/**d日线(2010~2026,16年,5205 symbols,OHLCV+amount)**。dbbardata schema 有 turnover(=成交额amount),**无换手率/涨跌幅列**。 +- **parquet data/raw + data/qfq**: 各 59816,O H L C V 6列,16年(xtdata建,build_daily_from_xtdata)。 +- **data/static/**: akshare balance 跑着(4300+ parquet)。 +- **实测缺口**: + 1. 日线**缺换手率+涨跌幅**(amount已在DB 16年) + 2. **ETF不在daily universe**(5205 symbols大概率纯股票,518880等海外ETF缺) + 3. **指数成分历史(含被踢)完全无**(全项目无脚本,akshare只当前快照) + +## 3 缺口设计(派生方案,避开数据混乱+接口限流) + +### 缺口1:日线换手/涨跌 → 派生,不新下载 +- **pct_chg** = (close今 - close昨)/close昨,**从 qfq close 算**(16年,避免除权跳空)。 +- **换手率** = volume / 流通股本。volume在DB;**流通股本在 akshare valuation(stock_value_em,排队P1,8.5年)**。 +- amount:DB已有(16年)。 +- **不开 daily_extra 新目录**,读取层派生 或 DB加列 → 不加剧"四套口径分裂" + **零新东财负载**。 + +### 缺口2:ETF日线 → 加进现有 xtdata universe +- ETF清单:海外 518880(黄金)/513100(纳指)/513030(德国)/164824(石油)/159866(有色) + 主 510300/510500/159915等,maintain成config。 +- 加进 `build_daily_from_xtdata` 的 universe → 走现有 xtdata 本地管线(miniQMT),**不碰东财,无限流**。 + +### 缺口3:指数成分历史(含被踢) → csindex 抓取,先 spike +- 范围:hs300(000300)/zz500(000905)/zz50(000016)/中小综指(399101)/创业板指(399006)。 +- **先派 agent spike 调研源**:akshare 有无历史成分API(index_stock_cons_weight_csindex带日期?fund_portfolio_hold_em?)、csindex.cn 历史成分xlsx URL规律+反爬、深证399101/399006 巨潮/szse 源。 +- **B档(务实,先行)**:抓全部历史调仓成分→并集(曾经入选集),消灭幸存者偏差。output `data/index_const_hist/.parquet`。 +- A档(精确,后做):时间序列(指数,生效日,成分,加/剔)。 +- csindex 独立源,串行单线程抓,**限流风险低**。 + +## 风险结论(为何用派生方案) +- **原设计(akshare daily_extra)** 有双风险:① 数据混乱——日线口径第四处(DB/parquet raw/parquet qfq/daily_extra),回测不知读哪;② 接口限流——daily_fields_akshare 并发 akshare_static = 第二股东财流量→东财封(同 baostock 黑名单原理)。 +- **派生方案**:缺口1 两风险全消(不下载/不开新目录);ETF走xtdata无限流;csindex独立源串行低风险。 +- 唯一仍调外部API:缺口3(csindex)+ 已排队的 akshare static 本身——保持串行+限速,不新增并发。 + +## 决策与顺序(下载完后) +1. **派生换手/涨跌**(读取层工具 或 DB加列)——缺口1 +2. **ETF 入 xtdata daily universe**——缺口2 +3. **csindex spike 调研** → 定 B档抓取脚本——缺口3 +4. (远期)A档精确成分时间序列 + +## 关联 +- 主计划:`docs/static_data_cache_plan.md` +- 现状memory:`baostock-15min-vps-deploy-plan` / `db-primary-parquet-fallback` / `data-download-architecture` diff --git a/docs/superpowers/plans/2026-07-21-data-fusion-p0.md b/docs/superpowers/plans/2026-07-21-data-fusion-p0.md new file mode 100644 index 0000000..b5cd70d --- /dev/null +++ b/docs/superpowers/plans/2026-07-21-data-fusion-p0.md @@ -0,0 +1,221 @@ +# P0 数据补全实现计划(历史成份股 + ETF 全市场 + 退市 K 线) + +> **For agentic workers:** 用 superpowers:subagent-driven-development 或 executing-plans 执行。Steps 用 `[ ]` 跟踪。 + +**Goal:** 补齐治幸存者偏差 + 策略核心缺口三类数据,落到 VPS 本地。 + +**Architecture:** 各源采集脚本 → staging parquet → 验证探针 → 合并主库;baostock 单登录守 48000/天;dbbardata 不动。 + +**Tech Stack:** python3.10 / akshare / baostock / xtquant(xtdata)/ pandas / pyarrow / sqlite3 + +--- + +## Global Constraints(所有 task 隐含) + +- **baostock 单进程单登录**,不并发(防黑名单,日 ≤48000 query) +- **直连不走代理**:`$env:http_proxy=''; $env:https_proxy=''; $env:all_proxy=''` +- **dbbardata 不破坏**:只 INSERT OR REPLACE `daily_baostock_full` / 新表,不动 dbbardata 既有行 +- **优先 baostock + miniQMT(xtdata)** +- **staging → 验证探针 → 合并主库**(用户铁律,不直接写主库) +- Windows VPS 49.232.102.198,`C:\Python310\python.exe -X utf8`,schtasks `/ru SYSTEM` +- 输出根:`C:\sanguo_vnpy_v2\data\` + +--- + +## File Structure + +| 文件 | 责任 | +|---|---| +| `scripts/data_platform/index_const_hist_download.py`(新) | 历史成份股采集(akshare 国证 + 新浪 + baostock 补时点) | +| `scripts/data_platform/build_daily_from_xtdata.py`(改 :40) | ETF universe 扩展(一次性全量) | +| `scripts/data_platform/daily_update_xtdata.py`(改 :114) | ETF 每日增量 universe | +| `scripts/data_platform/baostock_delisted_download.py`(新) | 退市股列表 + K 线采集 | +| `scripts/data_platform/import_delisted_to_db.py`(新) | 退市 K 线灌 `daily_baostock_full` | +| 各 `*_wrapper.ps1` + schtask | 部署 | + +--- + +## Task 1: 历史成份股采集(治幸存者偏差) + +**Files:** Create `scripts/data_platform/index_const_hist_download.py`;Output `data/index_const_hist/{code}.parquet` + +**Interfaces:** +- Consumes: akshare `index_detail_hist_cni(symbol)` + `index_detail_hist_adjust_cni(symbol)`(国证源);新浪 `vII_HistoryComponent`(pandas.read_html, gb2312);baostock `query_hs300/zz500/sz50_stocks(date)` +- Produces: `data/index_const_hist/{code}.parquet`(列:`updateDate/index_code/code/code_name/adjust_type`);并集 = 曾经入选集 + +**指数清单:** +- 深证/国证(akshare 国证源):399001 / 399006 / 399101 / 399005 / 399330 +- 中证(新浪):000852(中证1000)/ 932000(中证2000)/ 000300(交叉校验)/ 000016(上证50) +- baostock 已有(300/500/50 在 `bs_index_constituent`):Task1 补时点序列到同 schema + +- [ ] **1.1 探针:akshare 国证源 hist 版** +```python +import akshare as ak +df = ak.index_detail_hist_cni(symbol="399101") # 历史样本(日期/样本代码/权重) +print(df.columns.tolist(), len(df), df.head(3)) +adj = ak.index_detail_hist_adjust_cni(symbol="399101") # 调样记录(调整类型 OLD/+/-) +print(adj.columns.tolist(), len(adj)) +``` +预期:hist 有日期+样本+权重;adjust 有调整类型。**陷阱:必须 hist 版**(`index_detail_cni` 非 hist 版 2025-11-25 起只近期);`ak.index_stock_hist` 已下线别用。 + +- [ ] **1.2 探针:新浪中证历史成份** +```python +import pandas as pd +url = "http://vip.stock.finance.sina.com.cn/corp/go.php/vII_HistoryComponent/indexid/000852.phtml" +df = pd.read_html(url, encoding="gb2312")[0] +print(df.columns.tolist(), len(df), df.head(3)) +``` +预期:品种代码/品种名称/纳入日期/剔除日期(空=至今在列),含 *ST/退市股。 + +- [ ] **1.3 实现 `index_const_hist_download.py`**:三路采集 → 统一 schema(`updateDate/index_code/code/code_name/adjust_type`)→ 写 `data/index_const_hist/{code}.parquet`。串行 `time.sleep(0.8)`(akshare/新浪防封),单进程。环境变量 `BS_INDEX_HIST_OUT_DIR` 覆盖默认 Mac 路径(同 Day1 wrapper 模式)。 + +- [ ] **1.4 验证探针**:每指数 parquet 行数 + 抽样 3 行;**幸存者偏差校验** = 并集 `distinct code` 数 > 当前成份股数(证明含被踢股,例如 399101 并集 > 958 当前)。 + +- [ ] **1.5 wrapper + schtask**:`index_const_hist_wrapper.ps1`(设 OUT_DIR + utf8 + unset proxy + log);schtask `sanguo-index-hist` `/sc monthly /mo 2`(半年度调样后,6/12 月)`/ru SYSTEM`。 + +- [ ] **1.6 commit**:`git add scripts/data_platform/index_const_hist_download.py scripts/data_platform/index_const_hist_wrapper.ps1 && git commit -m "feat(data): 历史成份股采集(治幸存者偏差,国证+新浪+baostock)"` + +--- + +## Task 2: ETF 全市场日线 + +**Files:** Modify `scripts/data_platform/build_daily_from_xtdata.py:40` + `daily_update_xtdata.py:114` + +**Interfaces:** +- Consumes: xtdata `get_stock_list_in_sector('沪深A股'/'沪深ETF'/'沪深基金')` + `get_market_data_ex(dividend_type='front')` +- Produces: 全市场 ETF(~1000 只)日线**前复权**,落 parquet/dbbardata(复用现有 xtdata 管线) + +- [ ] **2.1 探针:ETF universe + 1 只 K 线** +```python +from xtquant import xtdata as xd +etf = xd.get_stock_list_in_sector('沪深ETF') or [] +fund = xd.get_stock_list_in_sector('沪深基金') or [] +a = xd.get_stock_list_in_sector('沪深A股') or [] +u = list(set(a + etf + fund)) +print(f"A={len(a)} ETF={len(etf)} fund={len(fund)} union={len(u)}") +r = xd.get_market_data_ex([], ['510300.SH'], period='1d', + start_time='20240101', end_time='20260721', dividend_type='front') +df = r.get('510300.SH') +print('510300 bars:', 0 if df is None else len(df), '| tail close:', None if df is None else df['close'].iloc[-1]) +``` +预期:ETF ~1000,union > A 股数;510300 前复权日线有值,close 非 NaN。 + +- [ ] **2.2 改 universe**:`build_daily_from_xtdata.py:40` 和 `daily_update_xtdata.py:114` 把 +```python +u = xd.get_stock_list_in_sector("沪深A股") or [] +``` +改为 +```python +u = list(set( + (xd.get_stock_list_in_sector("沪深A股") or []) + + (xd.get_stock_list_in_sector("沪深ETF") or []) + + (xd.get_stock_list_in_sector("沪深基金") or []) +)) +``` +保留 `dividend_type='front'`(前复权,§13 默认)。 + +- [ ] **2.3 全量下载 ETF**:跑改后的 `build_daily_from_xtdata.py`(走现有 xtdata 管线,**无限流**)→ parquet。 + +- [ ] **2.4 验证**:ETF 数 + 抽样(510300/513050/159919)+ 前复权 close 非 NaN + 日期范围。 + +- [ ] **2.5 schtask**:复用 `sanguo-daily-update`(universe 扩展后自动含 ETF,无需新 schtask)。 + +- [ ] **2.6 commit**:`git commit -m "feat(data): ETF 全市场日线(xtdata universe 扩展+前复权)"` + +--- + +## Task 3: 退市股 K 线(反幸存者偏差核心) + +**Files:** Create `scripts/data_platform/baostock_delisted_download.py` + `import_delisted_to_db.py`;Output → `daily_baostock_full` + +**Interfaces:** +- Consumes: baostock `query_all_stock(day)` + `query_stock_basic(code)`(status + 退市日期)+ `query_history_k_data_plus(code, fields, adjustflag=3)` +- Produces: 退市股 K 线 INSERT OR REPLACE `daily_baostock_full`(18 列,复用 `parse_baostock_code`) + +**范围:** 近 5 年退市(退市日期 ≥ 2021;守 48000/天;退市股分天跑) + +- [ ] **3.1 探针:退市股列表字段** +```python +import baostock as bs, pandas as pd +bs.login() +rs = bs.query_all_stock(day="2026-07-18") +rows = [] +while (rs.error_code == '0') & rs.next(): + rows.append(rs.get_row_data()) +df = pd.DataFrame(rows, columns=rs.fields) +print('query_all_stock fields:', rs.fields, '| rows:', len(df)) +rs2 = bs.query_stock_basic(code="sh.600000") +b = [] +while (rs2.error_code == '0') & rs2.next(): + b.append(rs2.get_row_data()) +print('query_stock_basic fields:', rs2.fields, '| sample:', b[0] if b else None) +bs.logout() +``` +预期:`query_stock_basic` 含 `type`(1股)/`status`(1上市 0退市)/`outDate`(退市日期)。筛 `status=0 & outDate>='2021-01-01'`。 + +- [ ] **3.2 实现 `baostock_delisted_download.py`**: + - 遍历全 code(或 `query_all_stock` 多日并集)→ `query_stock_basic` 筛 `status=0 & outDate>='2021-01-01'` → 退市股列表 + - 逐只 `query_history_k_data_plus(code, start_date='1990-01-01', end_date=outDate, fields=18字段, adjustflag=3)` → staging `data/delisted_kline/{code}.parquet` + - 单进程单登录,`time.sleep` 守预算,marker 断点续传(复用 Day1 模板),DAILY_LIMIT 计数器 + +- [ ] **3.3 `import_delisted_to_db.py`**:staging → INSERT OR REPLACE `daily_baostock_full`(复用 `parse_baostock_code` sh.600000→600000+SH + `executemany`,WAL + busy_timeout=60000,同 `import_baostock_to_db.py`)。**dbbardata 不碰**。 + +- [ ] **3.4 验证探针**:退市股数 + 抽样(某退市股 K 线行数 + max(date) ≤ 退市日)+ `daily_baostock_full` 行数增量 + distinct symbol 增量。 + +- [ ] **3.5 wrapper + schtask**:`baostock_delisted_wrapper.ps1`;schtask `sanguo-delisted` `/sc monthly /ru SYSTEM`(月度,守 48000,错开 day2b 02:00 + bs-daily-increment 17:00)。 + +- [ ] **3.6 commit**:`git commit -m "feat(data): 退市股 K 线采集(baostock,反幸存者偏差)"` + +--- + +## Task 4: baostock 日增量 → daily_baostock_full(#7 daily_update_static) + +> **串行约束**:本 task 与 Task3 都用 baostock 长会话,**必须串行**(Task3 probe → Task3 执行 → Task4),不可并发(防黑名单)。 + +**Files:** Create `scripts/data_platform/daily_update_static.py` + `daily_update_static_wrapper.ps1` + +**背景:** 现有 `daily_update_xtdata.py` 只产 parquet 不灌 `daily_baostock_full`(已知 gap,memory `db-primary-parquet-fallback` 记录)。本 task 补 baostock 日线的**每日增量灌库**。 + +**Interfaces:** +- Consumes: baostock `query_stock_basic`(全 A,type=1 含退市,复用 `baostock_daily_fullmarket_download.py:fetch_all_stocks`)+ `query_history_k_data_plus`(LOOKBACK 窗口,adjustflag=3 raw,18 字段同 `BS_FIELDS`) +- Produces: staging `data/daily_baostock_increment/{YYYYMMDD}/{code}.{exc}_daily.parquet`(审计)→ 同进程 INSERT OR REPLACE `daily_baostock_full`(复用 `parse_baostock_code`+executemany+WAL+busy_timeout,同 `import_baostock_to_db.py`) + +**设计(LOOKBACK 窗口 + 幂等,不同于全量 marker 模式):** +- **不用 marker 断点续传**(全量才需要;增量每日全量重拉最近 N 天) +- `LOOKBACK_DAYS=7`(覆盖周末/节假日;baostock 日终更新,17:00 跑时当日 bar 已就绪) +- 每只 1 query → 5537 query/run ≪ 48000/天 ✅(留足余量给 day2b/Task3) +- `sleep 0.4s × 5537 ≈ 37min`(17:00 schtask 可接受) +- `QUERY_COUNT` 计数器 + `DAILY_LIMIT=40000` 防御(复用全量脚本模式) +- **一脚本贯通**:download LOOKBACK → staging parquet(审计)→ in-memory df → executemany INSERT OR REPLACE(幂等,重复跑同一天安全,`drop_duplicates keep last` 不需要因 PK+OR REPLACE 天然去重) + +**Steps:** +- [ ] **4.1 探针(可选,Day1 已实证 query_history_k_data_plus 可用)**:ssh VPS 跑 1 只近 7 天确认接口 + 当日 bar 就绪 +- [ ] **4.2 写 `daily_update_static.py`**:自包含,结构 + - `unset proxy` + `socket.setdefaulttimeout(30)`(同全量脚本,baostock 坑) + - `_login_once`/`_relogin`/`fetch_all_stocks`/`fetch_one_daily`/`parse_baostock_code` 复用(可 import 或复制;优先 from `baostock_daily_fullmarket_download import ...`,注意 `QUERY_COUNT` global 需在同进程) + - `LOOKBACK` 窗口:`start=today-7, end=today` + - 主循环:逐只 `fetch_one_daily` → staging parquet → 累积 df → 每 100 只 `executemany INSERT OR REPLACE`(WAL+busy_timeout=60000) + - `QUERY_COUNT`/`DAILY_LIMIT`/断路器/定期重登 复用 + - 结束 verify:抽样 3 只 `max(date) ≈ today`、当日新增行数 + - 环境变量 `BS_INCREMENT_OUT_DIR`/`DB_PATH` 覆盖默认(同 Day1 wrapper 模式适配 Win) +- [ ] **4.3 小样本**:`--limit 10` 跑 10 只,确认 staging 有行 + DB 抽样 max(date)≈today +- [ ] **4.4 全量跑**:5537 只,守预算 +- [ ] **4.5 wrapper + schtask**:`daily_update_static_wrapper.ps1`(unset proxy+utf8+OUT_DIR+log);schtask `sanguo-bs-daily-increment` `/sc daily /st 17:00 /ru SYSTEM`(错开 daily-update 16:30 + day2b 02:00 + Task3 月度) +- [ ] **4.6 commit**:`git commit -m "feat(data): baostock 日增量灌库 daily_update_static(#7 gap 补)"` + +--- + +## Self-Review + +- **Spec 覆盖**:Task1→spec §4 成份股行 + §8 P0.1;Task2→§4 ETF 行 + §8 P0.2;Task3→§4 退市行 + §8 P0.3 ✅ +- **Placeholder 扫描**:无 TBD/TODO;采集脚本给接口+探针+schema,实现者按骨架写完整(采集脚本完整代码由执行 agent 基于 接口/schema/陷阱 产出)✅ +- **类型一致**:`index_const_hist` schema 各源统一;`daily_baostock_full` 18 列复用 `import_baostock_to_db.py` 的 `parse_baostock_code`+executemany ✅ +- **陷阱纳入**:`ak.index_stock_hist` 下线(1.1 标注)/ csindex SPA 无历史(用国证+新浪)/ 新浪 gb2312(1.2)/ hist 版必须(1.1)✅ + +--- + +## Execution Handoff + +计划存 `docs/superpowers/plans/2026-07-21-data-fusion-p0.md`。执行方式: +1. **Subagent-Driven**(推荐):每 Task 派 fresh agent + task 间 review +2. **Inline**:本 session 批量执行 + checkpoint diff --git a/docs/superpowers/specs/2026-07-21-data-source-fusion-design.md b/docs/superpowers/specs/2026-07-21-data-source-fusion-design.md new file mode 100644 index 0000000..bfd7252 --- /dev/null +++ b/docs/superpowers/specs/2026-07-21-data-source-fusion-design.md @@ -0,0 +1,243 @@ +# A 股多数据源融合层设计 + +> 日期:2026-07-21 | 基于 brainstorming + 4 源全能力调查(akshare / baostock / miniQMT / csindex) +> 状态:设计草案,待用户评审 → writing-plans + +--- + +## 1. 背景与痛点 + +多数据源(akshare / baostock / xtdata / miniQMT)各不全,整合时格式有偏差: +- **symbol 格式**:`600519` vs `sh.600000` vs `600519.SH` +- **exchange 命名**:`SSE` vs `SH` +- **复权口径**:raw vs qfq +- **volume 单位**:xtdata÷100 vs 原值 +- **价格源间漂移**:同股同日不同源 close 微差 + +### 用户约束(明确) +- ❌ 不要查询时网络源切换(源变化/限流不可控) +- ✅ VPS 本地一份稳定数据,日常只读本地 +- ✅ 网络源只用于「采集时拼凑完整本地」 +- ✅ 使用层无感(本地缺才网络兜底,罕见) +- ✅ 优先 miniQMT + baostock + +--- + +## 2. 设计原则(三层) + +| 层 | 职责 | 原则 | +|---|---|---| +| **采集层** | 多网络源 → 拼凑完整本地 | 各源 adapter + 定时 schtask;源不可控隔离在采集(失败重试,不影响使用层) | +| **数据层** | 整理(重叠定权威,特定保留) | **不强合物理表**(vnpy 回归风险);每类定权威源(优先 baostock+miniqmt) | +| **使用层** | `LocalUnifiedProvider` 逻辑融合 | 读权威表 + 归一化 + 本地缺网络兜底;策略无感 | + +**核心**:网络源不稳的风险只影响采集层(定时跑、可重试),使用层永远读本地 —— 风险隔离。 + +--- + +## 3. 数据源全能力盘点(综合调查) + +### 3.1 已下(稳定源) +| 表/源 | 内容 | 范围 | 增量 | +|---|---|---|---| +| `dbbardata` | 日线+5m/15m(xtdata/akshare/baostock) | 日线2010+/分钟2020+ | ✅ 16:30 | +| `daily_baostock_full` | 日线 18 字段(pe/pb/turn/pctChg) | 1990–2026 | 待 #7 | +| `bs_index_constituent` | 成份股 300/500/50(含退市) | 2006+ | 待增量 | +| `bs_adjust_factor` | 复权因子 | 全史 | 待增量 | +| akshare 静态表 | 三表/估值/北向/龙虎榜/融资融券/股本/解禁/业绩预告 | - | 部分增量 | +| miniQMT xtdata | 日线+5m/15m 全周期(实时 T+0,零漂移) | 16年 | 按需 | +| miniQMT PershareIndex | ROE/毛利率/EPS | 季频 | 按需 | +| parquet | data/raw、daily_baostock、minute_5/15、static/index_const、qfq+raw ETF(5只) | - | 部分 | + +### 3.2 重叠(5 处) +1. 日线 OHLCV:dbbardata ∩ daily_baostock_full ∩ parquet raw(三处) +2. 估值 PE/PB:akshare valuation ∩ daily_baostock_full(peTTM/pbMRQ) +3. 15min:dbbardata ∩ parquet minute_15 +4. 基本面:akshare 三表 ∩ miniQMT PershareIndex ∩ baostock 季频 +5. 成份股:bs_index_constituent ∩ parquet index_const + +### 3.3 新发现缺口(本次调查) +- **ETF 全市场日线**(现仅 5 只,策略资产类别缺口) +- **深证/中证历史成份股**(治幸存者偏差,当前仅最新快照 = latent bug) +- **退市股 K 线**(baostock 有,从未提取;反幸存者偏差核心) +- 申万行业 SW1/2/3 + 历史变动 +- 龙虎榜 / 合约信息(涨跌停/ST)/ 可转债 / 研报一致预期 / 股东户数 / 除权明细 + +--- + +## 4. 权威源地图(数据层整理) + +| 数据类 | 权威源 | 物理存储 | 重叠处理 / 备注 | +|---|---|---|---| +| 日线 OHLCV(个股,历史) | **baostock** | `daily_baostock_full` | dbbardata 日线保留(vnpy 回测硬依赖);parquet raw 冗余 | +| 日线(盘中实时) | **miniQMT xtdata** | xtdata API | 独有(T+0 实时) | +| **日线 ETF(全市场)** | **miniQMT xtdata** | parquet/dbbardata | universe 加 `沪深ETF∪沪深基金`,`dividend_type='front'` 自动复权 | +| 估值 PE/PB/turn | **baostock** | `daily_baostock_full` | akshare valuation 兜底/校验 | +| 基本面指标 ROE/毛利率 | **miniQMT PershareIndex** | miniQMT API | akshare 三表补原始报表 | +| 基本面三表(原始) | akshare | akshare 表 | miniQMT Balance/Income/CashFlow + baostock 季频交叉 | +| 15min | **baostock** | `dbbardata` | parquet minute_15 冗余可清 | +| 成份股 300/500/50(含退市) | **baostock** | `bs_index_constituent` | `query_*_stocks(date)` 任意时点 | +| **成份股 深证/国证(399xxx)** | **akshare(国证源)** | parquet | `index_detail_hist_cni` + `index_detail_hist_adjust_cni` | +| **成份股 中证1000/2000(000852/932000)** | **新浪** | parquet | `vII_HistoryComponent`(gb2312,含退市) | +| **退市股 K 线** | **baostock** | `daily_baostock_full` | `query_all_stock` status=0 + `query_stock_basic` 退市日期 + 逐只 K | +| 复权因子 | **baostock** | `bs_adjust_factor` | 独有 | +| 申万行业 SW1/2/3 | **miniQMT xtdata** | parquet | `get_sector_list` + `get_stock_list_in_sector`;akshare 补历史变动 | +| 龙虎榜 | **miniQMT xtdata** | parquet | `get_longhubang`;akshare 兜底 | +| 合约信息(涨跌停/ST/上市日) | **miniQMT xtdata** | parquet | `get_instrument_detail` 全 A 一入库 | +| 可转债 | akshare | parquet | `bond_zh_hs_cov_min` + `bond_cb_adj_logs_jsl`(转股价) | +| 研报/一致预期 EPS | akshare | parquet | `stock_research_info_em` | +| 股东户数 | miniQMT/akshare | parquet | `Holdernum` 表 / `stock_zh_a_gdhs_detail` | +| 除权明细 | miniQMT xtdata | parquet | `get_divid_factors` | +| 龙虎榜/北向/融资融券/解禁 | akshare | akshare 表 | 独有(保留) | + +--- + +## 5. 归一化规则 + +| 维度 | 统一标准 | 源映射 | +|---|---|---| +| symbol | `600519.SH`(数字+交易所后缀) | baostock `sh.600000`→`600000.SH`;dbbardata 纯数字+exchange 字段 | +| exchange | `SH`/`SZ` | dbbardata `SSE`/`SZSE` → `SH`/`SZ` | +| 日期 | ISO `2026-07-21` | 各源统一 | +| 复权 | raw 存储 + `factor` 字段(QLib:`factor=adj/raw`) | 查询时按需 qfq(`$close/$factor`);治 raw/qfq 冲突 | +| volume | 原值(股) | xtdata ÷100 还原 | +| 停牌 | OHLCV 全 NaN | QLib 约定 | +| 溯源 | `source` 字段 | 每行标来源 + 主源/补丁标记 | + +--- + +## 6. 使用层:`LocalUnifiedProvider`(逻辑融合) + +**接口**: +```python +get_daily(symbol, start, end, adjust='raw') # 个股日线 +get_etf_daily(symbol, ...) # ETF 日线 +get_fundamentals(symbol, fields, date) # 财务指标/三表 +get_constituent(index, date) # 成份股(含历史,治幸存者偏差) +get_industry(symbol, date) # 申万行业(含历史变动) +get_longhubang(symbol, start, end) # 龙虎榜 +get_instrument_detail(symbol) # 涨跌停/ST +get_delisted_kline(...) # 退市股 K 线 +``` + +**职责**: +- 按数据类路由到权威表(§4 地图) +- 归一化(§5 规则):源格式 → 统一 vt_symbol/exchange/复权/单位 +- 本地缺 → `network_fetcher` 透明兜底(罕见,如新股未及采集)→ 写本地 → 返回 +- `source` 字段溯源;可选多源交叉校验 +- 使用层 API 不变,不知数据来自哪个源 + +**实现**:扩展现有 `sanguo_portfolio/providers/` 的 `DataProvider` 接口(BaostockProvider/LocalParquetProvider 已有)。 + +--- + +## 7. 增量 schtask 清单 + +| schtask | 数据 | 源 | 时间 | +|---|---|---|---| +| `sanguo-daily-update`(已有) | 日线+分钟→dbbardata | xtdata | 16:30 | +| `sanguo-bs-daily-increment`(#7 待建) | 日线→daily_baostock_full | baostock | 17:00 | +| `sanguo-etf-daily-increment`(新) | ETF 全市场日线 | xtdata(universe 沪深ETF) | 17:30 | +| `sanguo-akshare-static-increment`(新) | 估值/龙虎榜/三表/北向 | akshare | 18:00 | +| `sanguo-index-hist`(新,半年度) | 历史成份股调样 | akshare 国证 + 新浪 | 调样后(6/12 月) | +| `sanguo-delisted`(新,月度) | 退市股列表+K 线 | baostock | 月初 | + +**query 预算守 48000/天/IP**(baostock 硬限):各 baostock schtask 错开 + 日计数器。 + +--- + +## 8. 分阶段实现 + +### P0 — 治幸存者偏差 + 策略核心缺口 +1. **历史成份股**:深证/国证(`index_detail_hist_cni`)+ 中证1000/2000(新浪 `vII_HistoryComponent`)+ 300/500/50(baostock 已有) +2. **ETF 全市场日线**:xtdata universe 改 `沪深A股∪沪深ETF∪沪深基金` + `dividend_type='front'`(改 `build_daily_from_xtdata.py:40` + `daily_update_xtdata.py:114`) +3. **退市股 K 线**:baostock `query_all_stock` 筛 status=0 + `query_stock_basic` 退市日期 + 逐只 K → `daily_baostock_full` + +### P1 — 策略增强 +4. 申万行业 SW1/2/3(xtdata `get_sector_list`)+ 历史变动 +5. 龙虎榜(xtdata `get_longhubang`) +6. 合约信息涨跌停/ST(xtdata `get_instrument_detail`)全 A 入库 +7. 可转债(akshare `bond_zh_hs_cov_min` + 转股价调整) +8. 研报/一致预期 EPS(akshare `stock_research_info_em`) + +### P2 — 按需 +9. 股东户数 / 除权明细 / 业绩快报 / 大宗交易 / 宏观 / 期货 + +### 融合层(贯穿) +10. 归一化库(vt_symbol/exchange/factor/volume 映射) +11. `LocalUnifiedProvider`(读权威表 + 归一化 + 网络兜底) +12. 完整度监控报表(每类覆盖率/缺口,源退化早发现) + +--- + +## 9. 陷阱清单(实证) + +- `ak.index_stock_hist` **已下线**(akshare 1.10.37,2024 初)—— 别抄 2022-23 旧博客 +- `ak.index_stock_cons` 的"纳入日期"字段有迷惑性 —— 只是当前 300 只各自最初纳入日,不含被剔除,治不了幸存者偏差 +- csindex.com.cn 是 Vue SPA —— `requests.get` 拿空壳,官网只当前 Excel 无历史 +- 新浪 `fund_etf_hist_sina` **不复权** —— 不适合回测;ETF 复权走 xtdata `dividend_type='front'` +- 东财 `fund_etf_hist_em` **封 IP** —— 单线程限速或避用 +- baostock `query_all_stock` 不列退市日期 —— 配合 `query_stock_basic` +- 申万历史板块有变更 —— 回测用当时分类 +- `ak.index_detail_cni`(非 hist 版)2025-11-25 起只近期 —— 必须用 hist 版 + +--- + +## 10. YAGNI(不做) + +- ❌ 不强合物理表(冲突解决/历史一致性/vnpy 回归风险,代价大) +- ❌ 不引 QLib/OpenBB 框架(几百 MB,只摘模式:factor/source/归一) +- ❌ CS 截面归一(中期按需,先解决不全+格式) +- ❌ 实时 tick/盘口(非日终策略才需) + +--- + +## 11. 风险与对策 + +| 风险 | 对策 | +|---|---| +| vnpy 回测硬读 dbbardata | 不动 dbbardata,provider 层 SSE↔SH 映射 | +| baostock query 预算 48000/天 | 增量 schtask 错开 + 日计数器(#7+day2b 同天 44296<48000) | +| 东财封 IP | 避用东财,优先 baostock/xtdata/新浪/国证 | +| 接口下线(如 index_stock_hist) | 调查实证,不抄旧文 | +| 数据源漂移/幽灵尖峰 | source 溯源 + 涨跌停/量异常校验 | +| 历史成份股缺口致回测幸存者偏差 | P0 优先补(深证+中证1000/2000+退市) | + +--- + +## 12. 实现路径(writing-plans 拆) + +- **Phase 1(P0 数据补全)**:历史成份股 + ETF 全市场 + 退市 K 线(3 个采集脚本 + 灌库) +- **Phase 2(融合层)**:归一化库 + `LocalUnifiedProvider`(读现有+新表) +- **Phase 3(P1 增强)**:板块/龙虎榜/合约/可转债/研报 +- **Phase 4(运维)**:增量 schtask 全套 + 完整度监控 + +--- + +## 13. 开放问题与默认决策(自行决策,你可推翻) + +| # | 问题 | 默认(我定) | 备选 | 理由 | +|---|---|---|---|---| +| 1 | exchange 统一格式 | **SH/SZ**(provider 映射 dbbardata SSE→SH) | 保留 SSE/SZSE | baostock/xtdata 都用 SH/SZ,主流;vnpy SSE 在 provider 层映射 | +| 2 | P0 三项优先级 | **全做**(历史成份股 + ETF + 退市 K 线) | 先 ETF(策略即用) | 三项都治幸存者偏差/核心缺口,并行不冲突 | +| 3 | ETF 复权方式 | **xtdata `dividend_type='front'`**(前复权) | raw + factor(精确还原) | 前复权够策略用;raw+factor 中期按需 | +| 4 | 退市股 K 线范围 | **近 5 年退市**(守 baostock 48000/天预算) | 全退市(几千只,慢) | 近 5 年覆盖绝大多数回测;全量可后补 | +| 5 | `LocalUnifiedProvider` 接口 | §6 签名(get_daily/get_etf/get_fundamentals/get_constituent/get_industry/...) | 精简 | 覆盖全天候 + CTA 需求 | +| 6 | 增量 schtask 时间 | 17:00(baostock 日线)/ 17:30(ETF)/ 18:00(akshare 静态) | 调整 | 错开 sanguo-daily-update 16:30 + day2b 02:00 | +| 7 | 物理表 | **不新建统一表**,provider 读现有(dbbardata/daily_baostock_full/各 parquet) | 建 daily_unified | 避免 vnpy 回归 + 数据迁移(§10 YAGNI) | + +**默认推进路径**:按以上默认 → spec 定稿 → 转 writing-plans(P0 拆 3 个采集脚本实现计划)。你审 spec 时可推翻任一项,我改。 + +--- + +## 参考(调查来源) + +- xtdata 官方:https://dict.thinktrader.net/nativeApi/xtdata.html +- akshare 指数:https://akshare.akfamily.xyz/data/index/index.html +- akshare 基金:https://akshare.akfamily.xyz/data/fund/fund_public.html +- akshare 债券:https://akshare.akfamily.xyz/data/bond/bond.html +- baostock API:https://www.baostock.com/mainContent?file=pythonAPI.md +- 国证指数网(深证历史):http://www.cnindex.com.cn/module/index-detail.html?indexCode=399001 +- 新浪历史成份:http://vip.stock.finance.sina.com.cn/corp/go.php/vII_HistoryComponent/indexid/000852.phtml +- QLib 数据层:https://qlib.readthedocs.io/en/latest/component/data.html +- 现有缺口设计:`docs/static_data_gaps_design.md` diff --git a/scripts/data_platform/akshare_static_download.py b/scripts/data_platform/akshare_static_download.py new file mode 100644 index 0000000..0c512af --- /dev/null +++ b/scripts/data_platform/akshare_static_download.py @@ -0,0 +1,983 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""AKShare A 股静态数据全量下载 (16 类, 四种模式, 只产 parquet) + +目标: + 在 Windows Server 2022 VPS (49.232.102.198) 上用 akshare 全量拉取 A 股 + 静态数据 (估值/北向/股本/十大股东/三大报表/财务摘要/龙虎榜/大宗/融资融券/ + 解禁/业绩预告/业绩快报/指数成分/申万行业), 输出 import-ready parquet. + +四种模式: + - 模式A per-stock (5500股循环, 每股1调用) → ._.parquet + - 模式B per-date (交易日循环, 每日1调用) → _.parquet + - 模式C per-period (报告期循环, 每期1调用) → _.parquet + - 模式D one-shot (单次调用) → .parquet + +硬约束 (踩过的坑 / 用户铁律): + 1. 单线程串行, 严禁并发 (东财限流严, 并发会封 IP) + 2. 每次 akshare 调用后 sleep AK_INTERVAL=0.8s (防封) + 3. 每次 akshare 调用 30s 硬超时 (akshare 经常挂死, 用 ThreadPoolExecutor + 单线程 + future.result(timeout=30) 保护, 超时计 failed) + 4. 重试退避: 东财 ConnectionError 常见, 3 次重试, 指数退避 (2s/4s/8s) + 5. 断路器: 连续 30 个单位 (stock/date/period) failed → exit 2 + 6. empty vs failed 区分: + - 空 df (无北向持仓 / 退市 / 节假日无龙虎榜) → status='empty' 中性 + - 异常 / 超时 → status='failed' 计断路器 + 7. marker 断点续传 (per-unit): 只在成功写 parquet 后写 marker + 8. 开头 unset proxy (akshare 底层 requests 读 proxy 环境变量) + +与 baostock_static_download.py 差异: + - timeout 机制不同: akshare 用 ThreadPoolExecutor + future.result(timeout) + (baostock 用 socket.setdefaulttimeout, 对 akshare 不够: akshare 内部 + requests 会重试很久, 必须 future-level kill) + - 四种模式 (baostock 只有 per-stock / per-year / one-shot 三种) + - symbol 格式适配 code_to_symbol(): 同一 code 在不同端点格式不同 + (stock_value_em="600519", balance_sheet="SH600519", + top_holders="sh600519", share_change="600519") + - 重试退避指数 backoff (baostock 重试即重登, akshare 重试即等 2/4/8s) + +⚠️ 本脚本只依赖 akshare, 不碰 baostock (可与 baostock 15min 下载并行)。 + +用法: + # 全跑 (16 类) + python akshare_static_download.py + # 只跑指定类型 (逗号分隔) + python akshare_static_download.py --types valuation,index_const + # 小样测试 (per-stock 类前 3 股) + python akshare_static_download.py --types valuation --limit 3 + # 指定股票 (6 位 code, 自动猜交易所) + python akshare_static_download.py --types balance --codes 600519,000001 + # 日期范围 (per-date 类用) + python akshare_static_download.py --types dragon_tiger --start 20260101 --end 20260715 + # 强制重下 + python akshare_static_download.py --types valuation --force + +输出目录结构 (OUT_DIR/data/static): + {OUT_DIR}/ + ├── valuation/ (per-stock) + │ ├── 600519.SH_valuation.parquet + │ └── .600519.SH_valuation.akshare + ├── balance/ (per-stock, 带交易所前缀调用) + │ ├── 600519.SH_balance.parquet + │ └── .600519.SH_balance.akshare + ├── top_holders/ (per-stock × per-period) + │ ├── 600519.SH_2020930_top_holders.parquet + │ └── ... + ├── dragon_tiger/ (per-date) + │ ├── 20260715_dragon_tiger.parquet + │ └── .20260715_dragon_tiger.akshare + ├── forecast/ (per-period) + │ ├── 20251231_forecast.parquet + │ └── ... + ├── index_const/ (one-shot) + │ ├── index_const.parquet + │ └── .index_const.akshare + └── logs/akshare_static_YYYYMMDD_HHMMSS.log + +退出码: 0=完成, 1=致命错误 (akshare 装载/列表拉取失败), 2=断路器触发 (可重试) +""" + +import argparse +import concurrent.futures +import datetime +import json +import logging +import os +import sys +import time +from functools import partial +from pathlib import Path +from typing import Any, Callable, List, Optional, Tuple + +# ======================== 硬约束: unset proxy ======================== +# 必须在 import akshare 之前清理 (akshare 底层 requests 读 proxy 环境变量) +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(_k, None) + +# stdout 行缓冲 (SSH-detached / pipe 重定向时也能看到实时进度) +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass + +import akshare as ak # noqa: E402 +import pandas as pd # noqa: E402 + + +# ======================== 配置 (环境变量可覆盖) ======================== + +DEFAULT_OUT_DIR = r"C:\sanguo_vnpy_v2\data\static" +OUT_DIR = Path(os.environ.get("AK_STATIC_OUT_DIR", DEFAULT_OUT_DIR)) +DEFAULT_LOG_DIR = OUT_DIR / "logs" +_log_dir_env = os.environ.get("AK_STATIC_LOG_DIR") +LOG_DIR = Path(_log_dir_env) if _log_dir_env else DEFAULT_LOG_DIR + +START_DATE_DEFAULT = "2020-01-01" +# 报告期循环范围 (近5年×4季, forecast/express/top_holders 用). +# top_holders per-stock × per-period 会有股票×20期组合. +REPORT_PERIODS = [ + f"{y}{m:02d}{d:02d}" + for y in range(2020, datetime.date.today().year + 1) + for (m, d) in [(3, 31), (6, 30), (9, 30), (12, 31)] + if datetime.date(y, m, d) <= datetime.date.today() +] + +AK_INTERVAL = 0.8 # 每次调用后间隔秒 (防封 IP) +AK_TIMEOUT = 30.0 # 单次调用硬超时秒 (akshare 经常挂死) +AK_MAX_RETRIES = 3 # 单次下载重试次数 (含指数退避) +RETRY_BACKOFF = [2, 4, 8] # 重试间隔秒 (指数退避) +PROGRESS_LOG_EVERY = 100 # 每 N 个单位打一次进度日志 +CIRCUIT_BREAKER = 30 # 连续失败 N 个 → 断路退出 + +# 模式 A: per-stock 类型 (8 类) +PER_STOCK_TYPES = ( + "valuation", # stock_value_em(symbol="600519") + "northbound", # stock_hsgt_individual_em(symbol="600519") + "share_capital", # stock_share_change_cninfo(symbol="600519") + "balance", # stock_balance_sheet_by_report_em(symbol="SH600519") + "income", # stock_profit_sheet_by_report_em(symbol="SH600519") + "cashflow", # stock_cash_flow_sheet_by_report_em(symbol="SH600519") + "financial_abstract", # stock_financial_abstract(symbol="600519") + # top_holders 单独 (per-stock × per-period) +) +# 模式 B: per-date 类型 (4 类, margin_szse 跳过) +PER_DATE_TYPES = ( + "dragon_tiger", # stock_lhb_detail_em(start_date, end_date) + "block_trade", # stock_dzjy_mrmx(symbol="A股", start_date, end_date) + "margin_sse", # stock_margin_detail_sse(date) + "restricted", # stock_restricted_release_detail_em(start_date, end_date) +) +# 模式 C: per-period 类型 (2 类) +PER_PERIOD_TYPES = ( + "forecast", # stock_yjyg_em(date=period) + "express", # stock_yjkb_em(date=period) +) +# 模式 D: one-shot 类型 (2 类) +ONE_SHOT_TYPES = ( + "index_const", # index_stock_cons_csindex(symbol) × 3 合并 + "industry", # sw_index_first_info() +) +# top_holders 特殊: per-stock × per-period +TOP_HOLDERS = "top_holders" + +ALL_TYPES = ( + PER_STOCK_TYPES + + (TOP_HOLDERS,) + + PER_DATE_TYPES + + PER_PERIOD_TYPES + + ONE_SHOT_TYPES +) + + +# ======================== 日志 ======================== + +def setup_logging() -> Tuple[logging.Logger, Path]: + """配置 root logger: 同时写 stdout + 文件。返回 (logger, log_file)。""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = LOG_DIR / f"akshare_static_{ts}.log" + + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + root = logging.getLogger() + root.setLevel(logging.INFO) + for h in list(root.handlers): + root.removeHandler(h) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + root.addHandler(sh) + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + root.addHandler(fh) + + return logging.getLogger(__name__), log_file + + +logger, LOG_FILE = setup_logging() + + +# ======================== 工具函数 ======================== + +def normalize_date(s: str) -> str: + """接受 YYYYMMDD 或 YYYY-MM-DD, 返回 YYYYMMDD; 空串 → 今天。""" + s = (s or "").strip() + if not s: + return datetime.date.today().strftime("%Y%m%d") + s = s.replace("-", "") + if len(s) == 8 and s.isdigit(): + return s + raise ValueError(f"无效日期格式: {s} (期望 YYYYMMDD 或 YYYY-MM-DD)") + + +def guess_exchange_by_code(code: str) -> str: + """6/68/51/9 开头 → SH (含 9 开头 B 股), 其他 → SZ。 + 11/12 开头是可转债, 不应进入 (列表源 stock_info_a_code_name 只返股票)。""" + if code.startswith(("6", "68", "51", "9")): + return "SH" + return "SZ" + + +def code_to_symbol(code: str, exchange: str, endpoint: str) -> str: + """同 code 在不同 akshare 端点格式不同。 + + - stock_value_em / northbound / financial_abstract / share_change_cninfo: + 纯数字 "600519" + - balance/income/cashflow (三大报表): "SH600519" / "SZ000001" (大写前缀) + - top_holders (stock_gdfx_free_top_10_em): "sh600519" (小写前缀) + + endpoint 取端点函数名 (作为标识符), 不区分大小写比较。 + """ + e = endpoint.lower() + if e in ("balance_sheet", "income_sheet", "cashflow_sheet"): + # 三大报表: SH/SZ 大写前缀 + return f"{exchange}{code}" + if e == "top_holders": + # 十大股东: 小写前缀 + pfx = "sh" if exchange == "SH" else "sz" + return f"{pfx}{code}" + # 默认 (valuation / northbound / financial_abstract / share_capital): 纯数字 + return code + + +# ======================== 超时 + 重试 ======================== + +def call_ak_with_timeout( + fn: Callable[..., pd.DataFrame], + *args: Any, + timeout: float = AK_TIMEOUT, + **kwargs: Any, +) -> pd.DataFrame: + """单线程 ThreadPoolExecutor 包装 akshare 调用 + 硬超时。 + + akshare 内部 requests 在网络异常时会重试很久 (无总超时), 必须用 + future-level timeout 才能保证不挂死。超时抛 concurrent.futures.TimeoutError + (上层捕获并计 failed)。 + + 注意: 超时后 future 不可真正 kill (Python 线程不能强杀), 但 executor + 退出后工作线程仍会在后台跑完或最终超时; 对进程主流程无影响 (我们不 + 等 join)。每次新建 executor 避免线程累积。 + """ + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex: + future = ex.submit(fn, *args, **kwargs) + return future.result(timeout=timeout) + + +def call_ak_with_retry( + fn: Callable[..., pd.DataFrame], + unit_label: str, + *args: Any, + **kwargs: Any, +) -> Tuple[Optional[pd.DataFrame], str]: + """带重试退避的 akshare 调用。返 (df_or_None, status)。 + + status ∈ {'ok', 'failed'}: + - 调用成功 (df 可能空) → ('ok' or 'empty', df) + - 重试耗尽仍异常/超时 → ('failed', None) + + empty vs ok 的区分在调用方按 df.empty 判 (此处统一返 df, 调用方判 empty)。 + 重试退避: 第1次失败 sleep 2s, 第2次 4s, 第3次 8s (RETRY_BACKOFF)。 + """ + last_err: Optional[Exception] = None + for attempt in range(AK_MAX_RETRIES): + try: + df = call_ak_with_timeout(fn, *args, **kwargs) + return df, "ok" + except concurrent.futures.TimeoutError as e: + last_err = e + logger.warning( + "%s 超时重试 %d/%d: (>%ds)", + unit_label, attempt + 1, AK_MAX_RETRIES, int(AK_TIMEOUT), + ) + except Exception as e: + last_err = e + msg = str(e)[:200] + logger.warning( + "%s 异常重试 %d/%d: %s", + unit_label, attempt + 1, AK_MAX_RETRIES, msg, + ) + # 指数退避 + if attempt < AK_MAX_RETRIES - 1: + time.sleep(RETRY_BACKOFF[attempt]) + logger.error("%s 重试 %d 次仍失败: %s", unit_label, AK_MAX_RETRIES, last_err) + return None, "failed" + + +# ======================== 股票列表 (akshare 轻量端点) ======================== + +def fetch_all_stocks() -> List[Tuple[str, str]]: + """akshare stock_info_a_code_name() 拉全市场 A 股代码列表。 + + 返回 [(code, exchange), ...], 例 [('600519', 'SH'), ('000001', 'SZ')]。 + 端点轻量, 单次调用返 ~5500 行 (code + name 两列), 不取行情。 + exchange 按 code 开头猜 (6/68/51/9 → SH, 其他 → SZ)。 + """ + df, status = call_ak_with_retry( + ak.stock_info_a_code_name, "all_stocks", + ) + if status == "failed" or df is None: + raise RuntimeError("stock_info_a_code_name 拉取失败") + if df.empty: + raise RuntimeError("stock_info_a_code_name 返空 (异常)") + + # akshare 返回列名: code, name (stock_info_a_code_name) + out: List[Tuple[str, str]] = [] + n_skip = 0 + for code in df["code"].tolist(): + code = str(code).strip() + if len(code) != 6 or not code.isdigit(): + n_skip += 1 + continue + exchange = guess_exchange_by_code(code) + out.append((code, exchange)) + logger.info( + "股票列表: 总 %d 只 (跳过 %d 非法 code), 来自 stock_info_a_code_name", + len(out), n_skip, + ) + return out + + +# ======================== 路径 / marker ======================== + +def subdir_for(data_type: str) -> Path: + """data_type → OUT_DIR / 。""" + return OUT_DIR / data_type + + +def parquet_path_per_unit(data_type: str, unit_id: str) -> Path: + """通用 parquet 路径: /.parquet。 + + unit_id 由各模式自拼: + - per-stock: '600519.SH_valuation' + - per-stock × per-period (top_holders): '600519.SH_2020930_top_holders' + - per-date: '20260715_dragon_tiger' + - per-period: '20251231_forecast' + - one-shot: 'index_const' + """ + return subdir_for(data_type) / f"{unit_id}.parquet" + + +def marker_path_for(parquet_path: Path) -> Path: + """parquet → 同目录 .{stem}.akshare marker。""" + return parquet_path.parent / f".{parquet_path.stem}.akshare" + + +def load_done_units(data_type: str) -> set: + """扫子目录 marker 构造已完成 unit_id 集合 (真相源)。 + + marker 文件名格式: .{unit_id}.akshare + 直接用 stem (= unit_id) 作为 key, 不解析 (各模式 unit_id 规则不同, 统一字符串匹配)。 + """ + done: set = set() + d = subdir_for(data_type) + if not d.exists(): + return done + ext = ".akshare" + for marker in d.glob(f".*{ext}"): + name = marker.name + if not name.startswith(".") or not name.endswith(ext): + continue + unit_id = name[1:-len(ext)] + if unit_id: + done.add(unit_id) + return done + + +def write_parquet_and_marker( + df: pd.DataFrame, + parquet_path: Path, +) -> bool: + """写 parquet + marker。返 True 成功 / False 失败。""" + try: + parquet_path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path_for(parquet_path).write_text( + datetime.datetime.now().isoformat() + ) + return True + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return False + + +def _df_or_empty(result: Tuple[Optional[pd.DataFrame], str]) -> pd.DataFrame: + """把 call_ak_with_retry 的返回 (df_or_None, status) 转成非 None df。 + + 必须 None 显式判断 (不能 `df or pd.DataFrame()`, DataFrame 的 truth value + ambiguous, 会抛 "The truth value of a DataFrame is ambiguous"。 + """ + df = result[0] + return df if df is not None else pd.DataFrame() + + +# ======================== per-stock fetch 函数 (8 类) ======================== + +def fetch_valuation(symbol: str) -> pd.DataFrame: + """stock_value_em(symbol='600519') — 估值 (PE/PB/市值等13列, ~2000行/股)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_value_em, f"valuation/{symbol}", symbol=symbol, + )) + + +def fetch_northbound(symbol: str) -> pd.DataFrame: + """stock_hsgt_individual_em(symbol='600519') — 北向持股 (~1700行/股)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_hsgt_individual_em, f"northbound/{symbol}", symbol=symbol, + )) + + +def fetch_share_capital(symbol: str) -> pd.DataFrame: + """stock_share_change_cninfo(symbol='600519') — 股本变动 (44列)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_share_change_cninfo, f"share_capital/{symbol}", symbol=symbol, + )) + + +def fetch_balance_sheet(symbol: str) -> pd.DataFrame: + """stock_balance_sheet_by_report_em(symbol='SH600519') — 资产负债表 (319列)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_balance_sheet_by_report_em, f"balance/{symbol}", symbol=symbol, + )) + + +def fetch_income_sheet(symbol: str) -> pd.DataFrame: + """stock_profit_sheet_by_report_em(symbol='SH600519') — 利润表 (203列)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_profit_sheet_by_report_em, f"income/{symbol}", symbol=symbol, + )) + + +def fetch_cashflow_sheet(symbol: str) -> pd.DataFrame: + """stock_cash_flow_sheet_by_report_em(symbol='SH600519') — 现金流量表 (254列)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_cash_flow_sheet_by_report_em, f"cashflow/{symbol}", symbol=symbol, + )) + + +def fetch_financial_abstract(symbol: str) -> pd.DataFrame: + """stock_financial_abstract(symbol='600519') — 财务摘要 (80指标)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_financial_abstract, f"financial_abstract/{symbol}", symbol=symbol, + )) + + +# ======================== top_holders (per-stock × per-period) ======================== + +def fetch_top_holders_one_period( + symbol: str, period: str, +) -> pd.DataFrame: + """stock_gdfx_free_top_10_em(symbol='sh600519', date='20250930') 单期。 + symbol 小写前缀, period YYYYMMDD.""" + return _df_or_empty(call_ak_with_retry( + ak.stock_gdfx_free_top_10_em, + f"top_holders/{symbol}/{period}", + symbol=symbol, date=period, + )) + + +# ======================== per-date fetch 函数 (4 类) ======================== + +def fetch_dragon_tiger(date: str) -> pd.DataFrame: + """stock_lhb_detail_em(start_date=end_date=date) — 单日龙虎榜。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_lhb_detail_em, f"dragon_tiger/{date}", + start_date=date, end_date=date, + )) + + +def fetch_block_trade(date: str) -> pd.DataFrame: + """stock_dzjy_mrmx(symbol='A股', start_date=end_date=date) — 大宗交易明细。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_dzjy_mrmx, f"block_trade/{date}", + symbol="A股", start_date=date, end_date=date, + )) + + +def fetch_margin_sse(date: str) -> pd.DataFrame: + """stock_margin_detail_sse(date=date) — 沪市融资融券明细。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_margin_detail_sse, f"margin_sse/{date}", date=date, + )) + + +def fetch_restricted(date: str) -> pd.DataFrame: + """stock_restricted_release_detail_em(start_date=end_date=date) — 解禁明细。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_restricted_release_detail_em, f"restricted/{date}", + start_date=date, end_date=date, + )) + + +# ======================== per-period fetch 函数 (2 类) ======================== + +def fetch_forecast(period: str) -> pd.DataFrame: + """stock_yjyg_em(date=period) — 业绩预告 (全市场, 单期单调用)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_yjyg_em, f"forecast/{period}", date=period, + )) + + +def fetch_express(period: str) -> pd.DataFrame: + """stock_yjkb_em(date=period) — 业绩快报 (全市场, 单期单调用)。""" + return _df_or_empty(call_ak_with_retry( + ak.stock_yjkb_em, f"express/{period}", date=period, + )) + + +# ======================== one-shot fetch 函数 (2 类) ======================== + +def fetch_index_const() -> pd.DataFrame: + """index_stock_cons_csindex 循环 ["000300","000905","000852"] 合并。 + + 300/500/1000 三大指数成分, 合并返单 df, 加 index_code 列标识来源。 + """ + index_codes = ["000300", "000905", "000852"] + frames: List[pd.DataFrame] = [] + for idx in index_codes: + df, status = call_ak_with_retry( + ak.index_stock_cons_csindex, f"index_const/{idx}", symbol=idx, + ) + if status == "failed" or df is None: + logger.warning("index_const %s 失败, 跳过该指数", idx) + continue + if df.empty: + logger.warning("index_const %s 返空, 跳过", idx) + continue + df = df.copy() + df["index_code"] = idx + frames.append(df) + time.sleep(AK_INTERVAL) # 三次调用之间也限速 + if not frames: + return pd.DataFrame() + return pd.concat(frames, ignore_index=True) + + +def fetch_industry() -> pd.DataFrame: + """sw_index_first_info() — 申万一级行业列表 (东财接口 ConnectionError, 申万替代)。""" + return _df_or_empty(call_ak_with_retry( + ak.sw_index_first_info, "industry", + )) + + +# ======================== 通用下载单元 (写 parquet + marker) ======================== + +def download_one_unit( + data_type: str, + unit_id: str, + fetch_fn: Callable[[], pd.DataFrame], + force: bool, +) -> Tuple[str, int]: + """通用单 unit 下载: 拉 df → 写 parquet + marker。 + + fetch_fn() → df (可能空) 或 raise (call_ak_with_retry 已吞异常返 None, + 各 fetch_xxx 已把 None 转空 df; 这里 df 永远非 None 但可能空)。 + + 返 (status, rows), status ∈ {'ok', 'skipped', 'empty', 'failed'}。 + """ + parquet_path = parquet_path_per_unit(data_type, unit_id) + marker_path = marker_path_for(parquet_path) + + if not force and marker_path.exists(): + return "skipped", 0 + + try: + df = fetch_fn() + except Exception as e: + # 兜底: call_ak_with_retry 内部已重试, 这里理论上不应触发, + # 但保险 (各 fetch_xxx 转 None→空 df, 这里捕获意外异常) + logger.error("[%s] %s fetch 异常: %s", data_type, unit_id, e) + return "failed", 0 + + if df is None: + # fetch_xxx 保证返非 None, 但保险 + return "failed", 0 + + # 写 parquet + marker (空 df 也写, 静态语义: "查过了确实无数据") + if not write_parquet_and_marker(df, parquet_path): + return "failed", 0 + return ("ok" if not df.empty else "empty"), len(df) + + +# ======================== 主循环 (通用, 适用所有四种模式) ======================== + +def run_one_type( + data_type: str, + units: List[Tuple[str, Callable[[], pd.DataFrame]]], + args: argparse.Namespace, +) -> Tuple[dict, bool]: + """通用类型主循环: 按顺序处理每个 unit (unit_id, fetch_fn)。 + + 返 (stats, circuit_triggered)。 + + 每个 unit 之间 sleep AK_INTERVAL 限速 (最后一个不 sleep)。 + """ + # marker 断点续传 + done_set = load_done_units(data_type) + if args.force: + todo = [(uid, fn) for uid, fn in units] + else: + todo = [(uid, fn) for uid, fn in units if uid not in done_set] + logger.info( + "[%s] 待处理 %d (已完成 %d, 总 %d)", + data_type, len(todo), len(done_set), len(units), + ) + + stats = {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "rows": 0} + consec_fail = 0 + circuit_triggered = False + t_start = time.time() + total_bytes = 0 + + for i, (uid, fn) in enumerate(todo): + try: + status, rows = download_one_unit(data_type, uid, fn, args.force) + except Exception as e: + status, rows = "failed", 0 + logger.debug("[%s] %s 异常: %s", data_type, uid, e) + + stats[status] = stats.get(status, 0) + 1 + if status == "ok": + stats["rows"] += rows + consec_fail = 0 + # 统计 parquet 大小 + try: + total_bytes += parquet_path_per_unit(data_type, uid).stat().st_size + except OSError: + pass + elif status == "empty": + consec_fail = 0 + elif status == "failed": + consec_fail += 1 + # skipped 不重置也不递增 + + # 断路器 + if consec_fail >= CIRCUIT_BREAKER: + logger.error( + "[%s] [FATAL] 断路器触发: 连续 %d 个 unit 失败, " + "akshare 疑似不可达, 退出 (done_set 不含 failed, 复跑会重试)", + data_type, consec_fail, + ) + circuit_triggered = True + break + + # 进度日志 + if (i + 1) % PROGRESS_LOG_EVERY == 0 or (i + 1) == len(todo): + elapsed = time.time() - t_start + logger.info( + "[%s] 进度 %d/%d ok=%d empty=%d failed=%d skipped=%d " + "rows=%d size=%.1fMB (%.0f秒)", + data_type, i + 1, len(todo), + stats["ok"], stats["empty"], stats["failed"], stats["skipped"], + stats["rows"], total_bytes / 1024 / 1024, elapsed, + ) + + # 限速 + if i < len(todo) - 1: + time.sleep(AK_INTERVAL) + + elapsed = time.time() - t_start + logger.info( + "[%s] 完成, 耗时 %.1f 秒 (%.1f分), 统计: %s, 总 parquet 大小 %.1fMB", + data_type, elapsed, elapsed / 60, + json.dumps(stats, ensure_ascii=False), + total_bytes / 1024 / 1024, + ) + return stats, circuit_triggered + + +# ======================== 各模式 units 构造 ======================== + +def build_per_stock_units( + data_type: str, + endpoint: str, + fetch_binder: Callable[[str], pd.DataFrame], + all_codes: List[Tuple[str, str]], + args: argparse.Namespace, +) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: + """构造 per-stock units: [(unit_id, fetch_fn), ...]。 + + unit_id = '{code}.{EXC}_{data_type}', 例 '600519.SH_valuation' + fetch_fn 闭包绑定 symbol (按 endpoint 格式)。 + + endpoint 决定 symbol 格式 (见 code_to_symbol)。 + """ + # --codes 过滤 + if args.codes: + codes_set = {c.strip() for c in args.codes.split(",") if c.strip()} + todo_codes = [(c, guess_exchange_by_code(c)) for c in codes_set] + else: + todo_codes = list(all_codes) + # --limit 截断 + if args.limit > 0: + todo_codes = todo_codes[: args.limit] + + units: List[Tuple[str, Callable[[], pd.DataFrame]]] = [] + for code, exc in todo_codes: + symbol = code_to_symbol(code, exc, endpoint) + unit_id = f"{code}.{exc}_{data_type}" + # partial 绑定 symbol (lambda 闭包易 late-bind, 用 partial 安全) + fn = partial(fetch_binder, symbol) + units.append((unit_id, fn)) + return units + + +def build_top_holders_units( + all_codes: List[Tuple[str, str]], + args: argparse.Namespace, +) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: + """top_holders 特殊: per-stock × per-period (近5年×4季 = 20期). + + unit_id = '{code}.{EXC}_{period}_top_holders', 例 '600519.SH_2020930_top_holders' + """ + if args.codes: + codes_set = {c.strip() for c in args.codes.split(",") if c.strip()} + todo_codes = [(c, guess_exchange_by_code(c)) for c in codes_set] + else: + todo_codes = list(all_codes) + if args.limit > 0: + todo_codes = todo_codes[: args.limit] + + # 报告期取近 5 年×4季 (REPORT_PERIODS 已含至今所有) + periods = REPORT_PERIODS[-20:] if len(REPORT_PERIODS) >= 20 else REPORT_PERIODS + + units: List[Tuple[str, Callable[[], pd.DataFrame]]] = [] + for code, exc in todo_codes: + symbol = code_to_symbol(code, exc, "top_holders") + for period in periods: + unit_id = f"{code}.{exc}_{period}_{TOP_HOLDERS}" + fn = partial(fetch_top_holders_one_period, symbol, period) + units.append((unit_id, fn)) + logger.info( + "[top_holders] %d 票 × %d 期 = %d units", + len(todo_codes), len(periods), len(units), + ) + return units + + +def build_per_date_units( + data_type: str, + fetch_fn: Callable[[str], pd.DataFrame], + args: argparse.Namespace, +) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: + """构造 per-date units: 每个交易日 1 个 unit。 + + unit_id = '{date}_{data_type}', 例 '20260715_dragon_tiger' + + 交易日简单生成 (周一到周五), 排除节假日策略: 拉到空就 skip (计 empty, + 不计 failed); 不依赖节假日表 (节假日多日无数据, 拉空即合法)。 + """ + start = normalize_date(args.start) + end = normalize_date(args.end) + start_dt = datetime.datetime.strptime(start, "%Y%m%d").date() + end_dt = datetime.datetime.strptime(end, "%Y%m%d").date() + if start_dt > end_dt: + raise ValueError(f"--start {start} > --end {end}") + + units: List[Tuple[str, Callable[[], pd.DataFrame]]] = [] + d = start_dt + while d <= end_dt: + # 周一到周五 (周末无交易, 不入列) + if d.weekday() < 5: + date_str = d.strftime("%Y%m%d") + unit_id = f"{date_str}_{data_type}" + units.append((unit_id, partial(fetch_fn, date_str))) + d += datetime.timedelta(days=1) + logger.info( + "[%s] 日期范围 %s ~ %s, 工作日 %d 天 (节假日拉空计 empty)", + data_type, start, end, len(units), + ) + return units + + +def build_per_period_units( + data_type: str, + fetch_fn: Callable[[str], pd.DataFrame], +) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: + """构造 per-period units: 每个报告期 1 个 unit。 + + unit_id = '{period}_{data_type}', 例 '20251231_forecast' + 报告期取 REPORT_PERIODS (近 5 年×4季)。 + """ + units: List[Tuple[str, Callable[[], pd.DataFrame]]] = [] + for period in REPORT_PERIODS: + unit_id = f"{period}_{data_type}" + units.append((unit_id, partial(fetch_fn, period))) + logger.info( + "[%s] 报告期 %d 个 (近5年×4季, %s..%s)", + data_type, len(units), REPORT_PERIODS[0], REPORT_PERIODS[-1], + ) + return units + + +def build_one_shot_units( + data_type: str, + fetch_fn: Callable[[], pd.DataFrame], +) -> List[Tuple[str, Callable[[], pd.DataFrame]]]: + """构造 one-shot units: 单个 unit。 + + unit_id = data_type, 例 'index_const' + """ + return [(data_type, fetch_fn)] + + +# ======================== CLI / main ======================== + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="AKShare A 股静态数据全量下载 (16 类, 四种模式)", + ) + p.add_argument( + "--types", default=",".join(ALL_TYPES), + help=f"逗号分隔类型, 默认全部 ({','.join(ALL_TYPES)})", + ) + p.add_argument( + "--start", default=START_DATE_DEFAULT, + help=f"起始日期 YYYYMMDD 或 YYYY-MM-DD, 默认 {START_DATE_DEFAULT}", + ) + p.add_argument("--end", default="", help="结束日期, 默认今天") + p.add_argument( + "--codes", + help="指定代码逗号分隔 (6 位无前缀), 如 600519,000001 (仅 per-stock 类生效)", + ) + p.add_argument( + "--limit", type=int, default=0, + help="限制处理股票数 (per-stock 类生效), 测试用", + ) + p.add_argument("--force", action="store_true", help="强制重下, 忽略 marker") + return p.parse_args() + + +def resolve_types(s: str) -> List[str]: + """解析 --types 字符串 → 去重保序的类型列表。无效类型 SystemExit。""" + parts = [t.strip() for t in s.split(",") if t.strip()] + bad = [t for t in parts if t not in ALL_TYPES] + if bad: + raise SystemExit(f"未知 --types: {bad}, 可选 {list(ALL_TYPES)}") + if not parts: + return list(ALL_TYPES) + seen: set = set() + out: List[str] = [] + for t in parts: + if t not in seen: + out.append(t) + seen.add(t) + return out + + +def run_type_dispatch( + t: str, + all_codes: List[Tuple[str, str]], + args: argparse.Namespace, +) -> Tuple[dict, bool]: + """按类型路由到对应模式 + fetch 函数。返 (stats, circuit_triggered)。""" + if t in PER_STOCK_TYPES: + endpoint_map = { + "valuation": ("valuation", fetch_valuation), + "northbound": ("northbound", fetch_northbound), + "share_capital": ("share_capital", fetch_share_capital), + "balance": ("balance_sheet", fetch_balance_sheet), + "income": ("income_sheet", fetch_income_sheet), + "cashflow": ("cashflow_sheet", fetch_cashflow_sheet), + "financial_abstract": ("financial_abstract", fetch_financial_abstract), + } + endpoint, fetch_fn = endpoint_map[t] + units = build_per_stock_units(t, endpoint, fetch_fn, all_codes, args) + return run_one_type(t, units, args) + + if t == TOP_HOLDERS: + units = build_top_holders_units(all_codes, args) + return run_one_type(t, units, args) + + if t in PER_DATE_TYPES: + fetch_map = { + "dragon_tiger": fetch_dragon_tiger, + "block_trade": fetch_block_trade, + "margin_sse": fetch_margin_sse, + "restricted": fetch_restricted, + } + units = build_per_date_units(t, fetch_map[t], args) + return run_one_type(t, units, args) + + if t in PER_PERIOD_TYPES: + fetch_map = { + "forecast": fetch_forecast, + "express": fetch_express, + } + units = build_per_period_units(t, fetch_map[t]) + return run_one_type(t, units, args) + + if t in ONE_SHOT_TYPES: + fetch_map = { + "index_const": fetch_index_const, + "industry": fetch_industry, + } + units = build_one_shot_units(t, fetch_map[t]) + return run_one_type(t, units, args) + + logger.error("未知类型 (跳过): %s", t) + return {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "rows": 0}, False + + +def main() -> None: + args = parse_args() + start_date = normalize_date(args.start) + end_date = normalize_date(args.end) + types = resolve_types(args.types) + + logger.info("=" * 60) + logger.info("AKShare A 股静态数据全量下载 (16 类, 四种模式)") + logger.info(" 输出目录: %s", OUT_DIR) + logger.info(" 日志文件: %s", LOG_FILE) + logger.info(" 日期范围: %s ~ %s", start_date, end_date) + logger.info(" 报告期数: %d (%s..%s)", + len(REPORT_PERIODS), + REPORT_PERIODS[0] if REPORT_PERIODS else "-", + REPORT_PERIODS[-1] if REPORT_PERIODS else "-") + logger.info(" 类型: %s", types) + logger.info(" 限速: 单线程串行, AK_INTERVAL=%.1fs, AK_TIMEOUT=%.0fs", + AK_INTERVAL, AK_TIMEOUT) + logger.info(" 当前时间: %s", datetime.datetime.now().isoformat()) + logger.info("=" * 60) + + # 检测 akshare 版本 (日志) + try: + logger.info("akshare 版本: %s", ak.__version__) + except AttributeError: + logger.info("akshare 版本: 未知 (无 __version__ 属性)") + + any_circuit = False + # 是否需要股票列表 + need_codes = any( + t in PER_STOCK_TYPES or t == TOP_HOLDERS for t in types + ) + all_codes: List[Tuple[str, str]] = [] + + if need_codes: + try: + all_codes = fetch_all_stocks() + except Exception as e: + logger.error("[FATAL] 获取股票列表失败: %s", e) + sys.exit(1) + + for t in types: + logger.info("-" * 50) + logger.info(">>> 类型: %s", t) + try: + stats, circuit = run_type_dispatch(t, all_codes, args) + except Exception as e: + logger.exception("[%s] 类型执行异常: %s", t, e) + any_circuit = True + break + if circuit: + any_circuit = True + logger.error("[%s] 断路器触发, 跳过后续类型", t) + break + + logger.info("=" * 60) + logger.info("全部完成, 退出码 %d", 2 if any_circuit else 0) + logger.info("=" * 60) + sys.exit(2 if any_circuit else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/baostock_15min_fullmarket_download.py b/scripts/data_platform/baostock_15min_fullmarket_download.py new file mode 100644 index 0000000..264d90d --- /dev/null +++ b/scripts/data_platform/baostock_15min_fullmarket_download.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""BaoStock 15 分钟线全市场历史下载脚本 (Windows VPS, 只产 parquet) + +目标: + 在 Windows Server 2022 VPS (49.232.102.198) 上用 baostock 全量拉取 A 股 + (含退市) 15min K 线历史 (2020-01-01 起, 5.5 年深度, 远超 xtdata 12 个月), + 输出 import-ready parquet, 后续用 import_vnpy_minute_fast.py 灌 DB。 + +与已有 backfill_15min_baostock.py 区别: + - 跑在 Windows (VPS), 不是 NAS + - 只产 parquet, **绝不碰 DB** (下载与灌库解耦, 用户铁律) + - 文件名/列格式匹配 import_vnpy_minute_fast.py 期望, 直接复用 importer + - 不用 SIGALRM (Windows 没有), 改用 socket.setdefaulttimeout(30) + +硬约束 (踩过的坑): + 1. 单进程单登录, 严禁并发 (baostock 并发会拉黑封 IP 6-24h) + 2. 直连不走代理 (脚本开头 unset proxy) + 3. 每只 sleep 0.4s 限速 + 4. baostock login 受 socket.setdefaulttimeout(30) 保护 + connect 卡死会抛 socket.timeout, 捕获后走 _relogin 重试 + (Windows 无 SIGALRM, 这是跨平台等价方案) + +用法: + python baostock_15min_fullmarket_download.py # 全市场 2020-01-01~今天 + python baostock_15min_fullmarket_download.py --limit 10 # 测试: 前 10 只 + python baostock_15min_fullmarket_download.py --codes 600519,000001 # 指定 + python baostock_15min_fullmarket_download.py --force # 强制重下, 忽略 marker + python baostock_15min_fullmarket_download.py --shard 0 --total 4 # 分片 + +退出码: 0=完成, 1=致命错误 (登录失败/列表拉取失败), 2=断路器触发 (可重试) +""" + +import argparse +import datetime +import json +import logging +import os +import socket +import sys +import time +from pathlib import Path +from typing import List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +# 必须在 import baostock 之前清理 (baostock 底层 urllib 会读 proxy 环境变量) +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(_k, None) + +# SIGALRM 跨平台替代: 全局 socket 超时 30s。 +# baostock 的 socket 连接会继承这个超时, connect 卡死时抛 socket.timeout, +# 捕获后走 _relogin 重试。Windows 没有 SIGALRM, 这是等价方案。 +socket.setdefaulttimeout(30) + +# stdout 行缓冲 (SSH-detached / pipe 重定向时也能看到实时进度) +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass # 不是 TextIOWrapper 或已关闭 + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + + +# ======================== 配置 (环境变量可覆盖) ======================== + +DEFAULT_OUT_DIR = r"C:\sanguo_vnpy_v2\data\minute_kline\15min_baostock" +OUT_DIR = Path(os.environ.get("BS_15MIN_OUT_DIR", DEFAULT_OUT_DIR)) +DEFAULT_LOG_DIR = OUT_DIR / "logs" +_log_dir_env = os.environ.get("BS_15MIN_LOG_DIR") +LOG_DIR = Path(_log_dir_env) if _log_dir_env else DEFAULT_LOG_DIR + +START_DATE_DEFAULT = "2020-01-01" # baostock 15min 实证可回溯 2020-01-02 (5.5 年) + +BS_INTERVAL = 0.4 # 每只间隔秒 (防封 IP) +BS_MAX_RETRIES = 3 # 单只下载重试次数 (含强制重登) +PROGRESS_LOG_EVERY = 500 # 每 N 只打一次进度日志 +RELOGIN_EVERY = 400 # 每 N 只定期重登保持连接 +CIRCUIT_BREAKER = 30 # 连续失败 N 只 → 断路退出 + +# baostock 查询字段 (不要 code, date+time 用于合成 datetime) +BS_FIELDS = "date,time,open,high,low,close,volume,amount" +# parquet 输出列 (匹配 import_vnpy_minute_fast.py 期望); 空数据返回此 schema 的空 df +OUT_COLS = ["datetime", "open", "high", "low", "close", "volume", "turnover"] + + +# ======================== 日志 ======================== + +def setup_logging() -> Tuple[logging.Logger, Path]: + """配置 root logger: 同时写 stdout + 文件。返回 (logger, log_file)。""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = LOG_DIR / f"baostock_15min_fullmarket_{ts}.log" + + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + root = logging.getLogger() + root.setLevel(logging.INFO) + # 清掉已有 handler, 防 re-init 重复 + for h in list(root.handlers): + root.removeHandler(h) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + root.addHandler(sh) + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + root.addHandler(fh) + + return logging.getLogger(__name__), log_file + + +logger, LOG_FILE = setup_logging() + + +# ======================== 工具函数 ======================== + +def normalize_date(s: str) -> str: + """接受 YYYYMMDD 或 YYYY-MM-DD, 返回 YYYY-MM-DD; 空串 → 今天。""" + s = (s or "").strip() + if not s: + return datetime.date.today().strftime("%Y-%m-%d") + if "-" in s: + return s + if len(s) == 8 and s.isdigit(): + return f"{s[:4]}-{s[4:6]}-{s[6:8]}" + raise ValueError(f"无效日期格式: {s} (期望 YYYYMMDD 或 YYYY-MM-DD)") + + +def baostock_code_to_parts(bs_code: str) -> Tuple[str, str]: + """sh.600519 → (600519, 'SH'); sz.000001 → (000001, 'SZ')。 + 前缀直接从 baostock code 拿, 不猜。 + """ + if "." not in bs_code: + raise ValueError(f"无效 baostock code: {bs_code}") + prefix, num = bs_code.split(".", 1) + if prefix == "sh": + return num, "SH" + if prefix == "sz": + return num, "SZ" + raise ValueError(f"未知 baostock 前缀: {bs_code}") + + +def parts_to_baostock(code: str, exchange: str) -> str: + """(600519, 'SH') → 'sh.600519'。""" + pfx = "sh" if exchange == "SH" else "sz" + return f"{pfx}.{code}" + + +def guess_exchange_by_code(code: str) -> str: + """无前缀时按代码开头猜交易所 (与 backfill 一致)。 + 6/68/51 开头 → SH, 其他 → SZ。仅用于 --codes 指定无前缀代码。 + """ + if code.startswith(("6", "68", "51")): + return "SH" + return "SZ" + + +# ======================== 股票列表 (baostock query_stock_basic) ======================== + +def fetch_all_stocks() -> List[Tuple[str, str]]: + """从 baostock 拉全市场 A 股列表 (含退市, 避免生存偏差)。 + + 过滤: type=='1' (股票); **不过滤 status** (保留退市股, 避免生存偏差)。 + 返回: [(code, exchange), ...], 例 [('600519', 'SH'), ('000001', 'SZ')] + + baostock 0.9.3 实测字段顺序: code, code_name, ipoDate, outDate, type, status + (无 code_english_name; 与 baostock 文档不一致)。这里用 rs.fields 动态建索引, + 按字段名取值, 不硬编码位置, 避免字段顺序差异导致过滤错位。 + """ + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError( + f"query_stock_basic 失败: code={rs.error_code} msg={rs.error_msg}" + ) + + fields = list(rs.fields) + idx = {name: i for i, name in enumerate(fields)} + logger.info("query_stock_basic fields=%s", fields) + + out: List[Tuple[str, str]] = [] + n_stock = 0 + n_skip_type = 0 + n_skip_code = 0 + while rs.next(): + r = rs.get_row_data() + type_ = r[idx["type"]] if "type" in idx and len(r) > idx["type"] else "" + if type_ != "1": # 只要股票 (排除指数/债券/其他) + n_skip_type += 1 + continue + bs_code = r[idx["code"]] if "code" in idx and len(r) > idx["code"] else "" + # status 不过滤 (保留退市, 避免生存偏差) + try: + code, exchange = baostock_code_to_parts(bs_code) + except ValueError: + n_skip_code += 1 + continue + if len(code) != 6 or not code.isdigit(): + n_skip_code += 1 + continue + out.append((code, exchange)) + n_stock += 1 + logger.info( + "股票列表: stocks(type=1)=%d skip_type=%d skip_code=%d (含退市, 不过滤 status)", + n_stock, n_skip_type, n_skip_code) + return out + + +# ======================== baostock 登录 ======================== + +def _login_once() -> bool: + """bs.login() — 已被 socket.setdefaulttimeout(30) 保护。 + + connect 卡死会抛 socket.timeout, 这里捕获。 + 返回 True=登录成功。 + """ + try: + lg = bs.login() + if lg.error_code == "0": + return True + logger.error("bs.login() 失败: code=%s msg=%s", lg.error_code, lg.error_msg) + return False + except (socket.timeout, TimeoutError) as e: + logger.error("bs.login() 超时: %s (baostock 疑似冷却)", e) + return False + except Exception as e: + logger.error("bs.login() 异常: %s", e) + return False + + +def _relogin() -> bool: + """强制重登: logout + login。失败等 2s 再试 1 次。""" + try: + bs.logout() + except Exception: + pass + if _login_once(): + return True + time.sleep(2) + try: + bs.logout() + except Exception: + pass + return _login_once() + + +# ======================== 数据下载 ======================== + +def fetch_one_15min( + bs_code: str, start_date: str, end_date: str, +) -> Optional[pd.DataFrame]: + """从 baostock 下载 15min 不复权数据, 转 importer-ready 格式。 + + 返回 df 列 (匹配 import_vnpy_minute_fast.py 期望): + datetime (str YYYY-MM-DD HH:MM:SS), open, high, low, close, volume, turnover + volume 单位: **股** (baostock 原始值, 绝不 ×100, 与 xtdata 关键区别) + 失败/空数据返回 None。 + """ + rs = bs.query_history_k_data_plus( + bs_code, + BS_FIELDS, + start_date=start_date, + end_date=end_date, + frequency="15", + adjustflag="3", # 不复权 raw + ) + if rs.error_code != "0": + # 真查询错误 (baostock 端报错) — 抛异常让上层 retry/relogin。 + # 区别于"合法空数据": 空数据 error_code=="0" 但 0 行。 + raise RuntimeError( + f"baostock {bs_code} 查询错误: {rs.error_code} {rs.error_msg}" + ) + + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + if not rows: + # 合法空数据 (退市/区间外/新股未交易) — 返回空 df, 上层计 "empty" 中性, 不重试。 + return pd.DataFrame(columns=OUT_COLS) + + df = pd.DataFrame( + rows, + columns=["date", "time", "open", "high", "low", "close", "volume", "amount"], + ) + + # datetime 合成 (照抄样本脚本 baostock_15min_sample_to_db.py 已验证逻辑): + # baostock time = 17 位 YYYYMMDDHHMMSSSSS, 15min bar 秒恒 00 + df["datetime"] = ( + df["date"] + + " " + + df["time"].str[8:10] + + ":" + + df["time"].str[10:12] + + ":00" + ) + + # 数值转换 (baostock 返回字符串) + for col in ("open", "high", "low", "close", "volume", "amount"): + df[col] = pd.to_numeric(df[col], errors="coerce") + df = df.rename(columns={"amount": "turnover"}) + + # 数据自检 (照抄样本脚本): dropna OHLC + 过滤非法 OHLC + df = df.dropna(subset=["open", "high", "low", "close"]) + bad_ohlc = (df["high"] < df[["open", "close"]].max(axis=1)) | \ + (df["low"] > df[["open", "close"]].min(axis=1)) + if bad_ohlc.any(): + df = df[~bad_ohlc] + if df.empty: + # 自检后全过滤掉 (脏数据) — 也算合法空, 不重试。 + return pd.DataFrame(columns=OUT_COLS) + + # importer 期望列: datetime, open, high, low, close, volume (+ turnover 可选) + df = df[OUT_COLS] + return df.sort_values("datetime").reset_index(drop=True) + + +# ======================== marker / parquet 路径 ======================== + +def parquet_path_for(code: str, exchange: str) -> Path: + """600519 + SH → OUT_DIR / 600519.SH_15m.parquet (匹配 importer 正则)。""" + return OUT_DIR / f"{code}.{exchange}_15m.parquet" + + +def marker_path_for(parquet_path: Path) -> Path: + """parquet → .{stem}.baostock marker (同目录)。""" + return parquet_path.parent / f".{parquet_path.stem}.baostock" + + +def load_done_set() -> set: + """扫 OUT_DIR 所有 marker 构造已完成集合 (真相源)。 + + marker 文件名格式: .600519.SH_15m.baostock + 只有 marker 存在 = 该票成功完成; failed 票无 marker, 下次复跑会重试。 + JSON 进度文件不在这里读 (可选, 仅观察用)。 + """ + done = set() + if not OUT_DIR.exists(): + return done + suffix = ".baostock" + for marker in OUT_DIR.glob(f".*{suffix}"): + name = marker.name + if not name.startswith(".") or not name.endswith(suffix): + continue + stem = name[1:-len(suffix)] # e.g. 600519.SH_15m + # 解析: ._15m + try: + head, period = stem.rsplit("_", 1) + if period != "15m": + continue + code, exc = head.split(".", 1) + if len(code) == 6 and code.isdigit() and exc in ("SH", "SZ"): + done.add((code, exc)) + except ValueError: + continue + return done + + +# ======================== 单只下载 ======================== + +def download_one( + code: str, + exchange: str, + start_date: str, + end_date: str, + force: bool, +) -> Tuple[str, int]: + """下载单只 → 写 parquet + marker。 + + 返回 (status, rows): status ∈ {'ok', 'skipped', 'failed'}。 + """ + parquet_path = parquet_path_for(code, exchange) + marker_path = marker_path_for(parquet_path) + + if not force and marker_path.exists(): + return "skipped", 0 + + bs_code = parts_to_baostock(code, exchange) + + # retry 循环: fetch_one_15min 仅在真错误时抛异常 (走 retry/relogin); + # 空数据返回空 df (合法, 不重试), 成功 break。 + df: Optional[pd.DataFrame] = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_one_15min(bs_code, start_date, end_date) + break # fetch 成功 (df 可能空 = 合法无数据) + except (socket.timeout, TimeoutError, OSError) as e: + logger.warning( + "%s socket 异常重试 %d/%d: %s — 强制重登", + code, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s", code) + df = None + break + except Exception as e: + logger.warning( + "%s 下载异常重试 %d/%d: %s — 强制重登", + code, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s", code) + df = None + break + + if df is None: + return "failed", 0 # 多次重试仍报错 (真失败, 计断路器) + if df.empty: + return "empty", 0 # 合法无数据 (退市/区间外), 中性, 不计失败不重试 + + # 写 parquet + marker (只有成功写完且自检通过才写 marker) + try: + OUT_DIR.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path.write_text(datetime.datetime.now().isoformat()) + return "ok", len(df) + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return "failed", 0 + + +# ======================== 主流程 ======================== + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="BaoStock 15min 全市场历史下载 (Windows VPS, 只产 parquet)", + ) + p.add_argument( + "--start", default=START_DATE_DEFAULT, + help=f"起始日期 YYYYMMDD 或 YYYY-MM-DD, 默认 {START_DATE_DEFAULT}", + ) + p.add_argument("--end", default="", help="结束日期, 默认今天") + p.add_argument("--codes", help="指定代码逗号分隔, 如 600519,000001 (测试用)") + p.add_argument("--limit", type=int, default=0, help="限制处理数量 (测试用)") + p.add_argument("--force", action="store_true", help="强制重下, 忽略 marker") + p.add_argument("--shard", type=int, default=None, help="分片号 0-indexed (并行下载用)") + p.add_argument("--total", type=int, default=1, help="总分片数 (与 --shard 配合)") + return p.parse_args() + + +def resolve_codes(args: argparse.Namespace) -> List[Tuple[str, str]]: + """根据 args 解析待处理的 (code, exchange) 列表。""" + if args.codes: + # 指定代码: 无前缀, 按代码开头猜交易所 (与 backfill 风格一致) + return [ + (c.strip(), guess_exchange_by_code(c.strip())) + for c in args.codes.split(",") + if c.strip() + ] + return fetch_all_stocks() + + +def main() -> None: + args = parse_args() + start_date = normalize_date(args.start) + end_date = normalize_date(args.end) + + logger.info("=" * 60) + logger.info("BaoStock 15min 全市场历史下载 (Windows VPS)") + logger.info(" 输出目录: %s", OUT_DIR) + logger.info(" 日志文件: %s", LOG_FILE) + logger.info(" 日期范围: %s ~ %s", start_date, end_date) + logger.info(" frequency=15, adjustflag=3 (不复权 raw)") + logger.info(" socket.setdefaulttimeout(30) (SIGALRM 替代)") + logger.info(" 当前时间: %s", datetime.datetime.now().isoformat()) + if args.shard is not None: + logger.info(" 分片: shard=%s/%s", args.shard, args.total) + logger.info("=" * 60) + + # 登录 + if not _login_once(): + logger.error("[FATAL] baostock 登录失败 (疑似冷却), 退出") + sys.exit(1) + logger.info("baostock 登录成功") + + # 股票列表 + try: + all_codes = resolve_codes(args) + except Exception as e: + logger.error("[FATAL] 获取股票列表失败: %s", e) + try: + bs.logout() + except Exception: + pass + sys.exit(1) + logger.info("全市场 A 股 (含退市): %d 只", len(all_codes)) + + # limit + if args.limit > 0: + all_codes = all_codes[:args.limit] + logger.info("limit=%d 截断", args.limit) + + # 分片 + if args.shard is not None and args.total > 1: + all_codes = [ + s for i, s in enumerate(all_codes) + if i % args.total == args.shard + ] + logger.info("shard=%s/%s 过滤后: %d 只", args.shard, args.total, len(all_codes)) + + # 断点续传 (marker 为真相源) + done_set = load_done_set() + if args.force: + todo = all_codes + else: + todo = [(c, e) for c, e in all_codes if (c, e) not in done_set] + logger.info("待处理: %d (已完成 %d)", len(todo), len(done_set)) + + # 主循环 + stats = {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "rows": 0} + consec_fail = 0 + circuit_triggered = False + t_start = time.time() + + for i, (code, exc) in enumerate(todo): + # 定期重登保持连接 + if i > 0 and i % RELOGIN_EVERY == 0: + logger.info("定期重登 @ %d/%d", i, len(todo)) + if not _relogin(): + logger.warning("定期重登失败, 继续 (单次失败不致命)") + + try: + status, rows = download_one(code, exc, start_date, end_date, args.force) + except Exception as e: + status, rows = "failed", 0 + logger.debug("download_one %s.%s 异常: %s", code, exc, e) + + stats[status] = stats.get(status, 0) + 1 + if status == "ok": + stats["rows"] += rows + consec_fail = 0 + elif status == "failed": + consec_fail += 1 + # skipped / empty 中性: 不重置也不递增 (不证明 baostock 可用, 也不算失败) + + # 断路器: 连续 N 只全 failed → baostock 疑似不可达, 保存进度主动退出 + if consec_fail >= CIRCUIT_BREAKER: + logger.error( + "[FATAL] 断路器触发: 连续 %d 只失败, baostock 疑似不可达, " + "退出 (done_set 不含 failed 票, 复跑会重试)", + consec_fail, + ) + circuit_triggered = True + break + + # 进度日志 + if (i + 1) % PROGRESS_LOG_EVERY == 0: + elapsed = time.time() - t_start + logger.info( + "进度 %d/%d ok=%d skipped=%d empty=%d failed=%d rows=%d (%.0f秒)", + i + 1, len(todo), stats["ok"], stats["skipped"], + stats["empty"], stats["failed"], stats["rows"], elapsed, + ) + + # 限速 + if i < len(todo) - 1: + time.sleep(BS_INTERVAL) + + # 登出 + try: + bs.logout() + except Exception: + pass + + elapsed = time.time() - t_start + logger.info("=" * 60) + if circuit_triggered: + logger.info("[RESULT] 断路器触发中止, 耗时 %.1f 秒", elapsed) + else: + logger.info("[RESULT] 完成, 耗时 %.1f 秒", elapsed) + logger.info("统计: %s", json.dumps(stats, ensure_ascii=False)) + + sys.exit(2 if circuit_triggered else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/baostock_15min_sample_to_db.py b/scripts/data_platform/baostock_15min_sample_to_db.py new file mode 100644 index 0000000..399b200 --- /dev/null +++ b/scripts/data_platform/baostock_15min_sample_to_db.py @@ -0,0 +1,551 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""BaoStock 15min 样本验证脚本 (独立运行,不走现有 backfill/import 链路) + +目的: + 在上全市场 15min baostock 灌库前,先用 10 只样本股走通完整链路: + baostock 下载 → pandas 转换 → sqlite 灌库 → 读回验证 → 自检断言 + 通过则证明 baostock 数据形态、字段映射、DB schema 全部兼容,可放心上全市场。 + +硬约束(踩过的坑): + 1. 必须用 venv311 (已装 baostock 0.9.30 / pyarrow 25.0 / sqlite3) + 2. baostock 单进程单登录,严禁并发,每只 sleep 0.4s(并发会拉黑封IP) + 3. 直连不走代理(脚本开头 unset proxy 环境变量) + 4. bs.login() 套 SIGALRM 30s 超时,防永久挂起 + 5. 只灌 /tmp/quant_trading_sample.db 本地测试库,绝不碰 NAS/项目主库 + +surgical 原则: + 不修改 backfill_15min_baostock.py(全市场下载)/ import_vnpy_minute_fast.py(全市场灌库)。 + 本脚本只做样本验证,每次运行前重建测试库。 + +用法: + /Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/venv311/bin/python \ + scripts/data_platform/baostock_15min_sample_to_db.py +退出码: 0=成功 1=致命错误(baostock login 失/断路触发) +""" +import datetime +import os +import signal +import sqlite3 +import sys +import time +from typing import Optional, Tuple + +# ======================== 硬约束: unset proxy ======================== +# 必须在 import baostock 之前清理(baostock 底层 urllib 会读 proxy 环境变量) +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(_k, None) + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + + +# ======================== 配置 ======================== +VENV_PY = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/venv311/bin/python" +SAMPLE_DB = "/tmp/quant_trading_sample.db" +SAMPLE_CODES = [ + "600519", # 贵州茅台 + "600000", # 浦发银行 + "000001", # 平安银行 + "000858", # 五粮液 + "688981", # 中芯国际 + "300750", # 宁德时代 + "601318", # 中国平安 + "002594", # 比亚迪 + "600036", # 招商银行 + "000651", # 格力电器 +] +START_DATE = "2020-01-01" # 验证 baostock 5.5 年深度 +END_DATE = datetime.date.today().strftime("%Y-%m-%d") +FREQUENCY = "15" +ADJUST_FLAG = "3" # 3=不复权(raw 真实价) +BS_INTERVAL = 0.4 # 每只间隔秒(防封 IP) +BS_LOGIN_TIMEOUT = 30 # bs.login() SIGALRM 超时 +CIRCUIT_THRESHOLD = 3 # 连续失败 N 只 → 断路退出(样本验证保守值) + +# baostock 字段(注意 adjustflag 即使在 fields 里也返回,但我们不用它) +BS_FIELDS = "date,time,code,open,high,low,close,volume,amount,adjustflag" + + +# ======================== 建表 SQL (vnpy 标准 schema) ======================== +CREATE_TABLE_SQL = """ +CREATE TABLE IF NOT EXISTS dbbardata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol TEXT, exchange TEXT, datetime TEXT, interval TEXT, + volume REAL, turnover REAL DEFAULT 0, open_interest REAL DEFAULT 0, + open_price REAL, high_price REAL, low_price REAL, close_price REAL, + UNIQUE(symbol, exchange, datetime, interval) +) +""" +INDEX_SQL = ( + "CREATE INDEX IF NOT EXISTS idx_dbbardata_symbol_interval " + "ON dbbardata(symbol, interval)" +) +INSERT_SQL = ( + "INSERT OR REPLACE INTO dbbardata " + "(symbol,exchange,datetime,interval,volume,turnover,open_interest," + "open_price,high_price,low_price,close_price) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)" +) + + +# ======================== baostock 登录(带 SIGALRM 超时) ======================== +class _LoginTimeout(Exception): + """bs.login() SIGALRM 超时异常""" + + +def _login_timeout_handler(signum, frame): + raise _LoginTimeout("bs.login() 超时") + + +def login_with_timeout(timeout: int = BS_LOGIN_TIMEOUT) -> bool: + """bs.login() 套 SIGALRM 超时,防永久挂起。返回 True=登录成功。""" + old_handler = signal.signal(signal.SIGALRM, _login_timeout_handler) + signal.alarm(timeout) + try: + lg = bs.login() + ok = lg.error_code == "0" + if not ok: + print(f"[FATAL] bs.login() 失败: code={lg.error_code} msg={lg.error_msg}", + flush=True) + return ok + except _LoginTimeout: + print(f"[FATAL] bs.login() 超时 {timeout}s (baostock 疑似冷却中)", flush=True) + return False + except Exception as e: + print(f"[FATAL] bs.login() 异常: {e}", flush=True) + return False + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + +# ======================== 代码 → (baostock_code, exchange) ======================== +def code_to_baostock(code: str) -> Tuple[str, str]: + """6位代码 → (baostock格式 sh.XXXXXX/sz.XXXXXX, vnpy exchange SSE/SZSE)。 + + 规则: 6/68/51 开头 → sh / SSE, 其他 → sz / SZSE + """ + if code.startswith(("6", "68", "51")): + return f"sh.{code}", "SSE" + else: + return f"sz.{code}", "SZSE" + + +# ======================== 下载单只 ======================== +def fetch_one(code: str, start_date: str, end_date: str) -> Optional[pd.DataFrame]: + """从 baostock 下载 15min 不复权数据,做字段转换。 + + 返回处理后的 DataFrame,列: + symbol, exchange, datetime, open_price, high_price, low_price, + close_price, volume, turnover + 失败/空数据返回 None。 + """ + bs_code, exchange = code_to_baostock(code) + + rs = bs.query_history_k_data_plus( + bs_code, + BS_FIELDS, + start_date=start_date, + end_date=end_date, + frequency=FREQUENCY, + adjustflag=ADJUST_FLAG, + ) + if rs.error_code != "0": + print(f" [baostock] {code} ({bs_code}) 查询错误: " + f"code={rs.error_code} msg={rs.error_msg}", flush=True) + return None + + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + + if not rows: + print(f" [baostock] {code} ({bs_code}) 返回 0 行(新股/停牌?)", flush=True) + return None + + df = pd.DataFrame(rows, columns=[ + "date", "time", "code", "open", "high", "low", + "close", "volume", "amount", "adjustflag", + ]) + + # datetime 合成: baostock time 是 17 位 YYYYMMDDHHMMSSSSS + # date 是 YYYY-MM-DD, 取 time 的 HH:MM 部分, 秒恒 00 + df["datetime"] = ( + df["date"] + + " " + + df["time"].str[8:10] + + ":" + + df["time"].str[10:12] + + ":00" + ) + + # 数值转换(baostock 返回字符串) + for col in ("open", "high", "low", "close"): + df[col] = pd.to_numeric(df[col], errors="coerce") + df["volume"] = pd.to_numeric(df["volume"], errors="coerce") + df["amount"] = pd.to_numeric(df["amount"], errors="coerce") + + # 重命名 → vnpy schema + df = df.rename(columns={ + "open": "open_price", + "high": "high_price", + "low": "low_price", + "close": "close_price", + "amount": "turnover", + }) + df["symbol"] = code + df["exchange"] = exchange + + # 字段顺序对齐 + df = df[[ + "symbol", "exchange", "datetime", + "open_price", "high_price", "low_price", "close_price", + "volume", "turnover", + ]] + + # 删除 NaN OHLCV(baostock 偶有空值,无意义) + df = df.dropna( + subset=["open_price", "high_price", "low_price", "close_price", "volume"] + ) + + return df if not df.empty else None + + +# ======================== 自检断言 ======================== +def self_check(df: pd.DataFrame, code: str) -> Tuple[bool, str]: + """对下载并转换后的 df 做数据质量自检。 + + 断言: + A. high >= low + B. low <= close <= high + C. volume > 0 + D. datetime 格式 YYYY-MM-DD HH:MM:SS(19 字符) + 返回 (all_pass, 报告字符串)。 + """ + n = len(df) + if n == 0: + return False, "empty dataframe" + + # A. high >= low + bad_hl = df[df["high_price"] < df["low_price"]] + a_pass = len(bad_hl) == 0 + + # B. low <= close <= high + bad_close = df[(df["close_price"] < df["low_price"]) | + (df["close_price"] > df["high_price"])] + b_pass = len(bad_close) == 0 + + # C. volume > 0 + bad_vol = df[df["volume"] <= 0] + c_pass = len(bad_vol) == 0 + + # D. datetime 格式 (YYYY-MM-DD HH:MM:SS = 19 字符) + dt = df["datetime"].astype(str) + bad_fmt = dt[(dt.str.len() != 19) | (~dt.str.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$"))] + d_pass = len(bad_fmt) == 0 + + all_pass = a_pass and b_pass and c_pass and d_pass + report = ( + f"high>=low={'PASS' if a_pass else 'FAIL(%d)' % len(bad_hl)} | " + f"low<=close<=high={'PASS' if b_pass else 'FAIL(%d)' % len(bad_close)} | " + f"volume>0={'PASS' if c_pass else 'FAIL(%d)' % len(bad_vol)} | " + f"dt_format={'PASS' if d_pass else 'FAIL(%d)' % len(bad_fmt)}" + ) + return all_pass, report + + +# ======================== 灌库 ======================== +def init_db(db_path: str) -> sqlite3.Connection: + """(重)建测试库 + dbbardata 表。每次重跑前 drop 旧表。""" + conn = sqlite3.connect(db_path) + c = conn.cursor() + # 幂等: 若表存在先 drop(测试库不做增量,每次 fresh start) + c.execute("DROP TABLE IF EXISTS dbbardata") + c.execute(CREATE_TABLE_SQL) + c.execute(INDEX_SQL) + conn.commit() + return conn + + +def insert_df(conn: sqlite3.Connection, df: pd.DataFrame) -> int: + """批量 INSERT OR REPLACE 灌入 15min bars。返回灌入行数。""" + values = df[[ + "symbol", "exchange", "datetime", + "open_price", "high_price", "low_price", "close_price", + "volume", "turnover", + ]].values.tolist() + # 在对应位置插入 interval='15m' 和 open_interest=0.0 + # SQL 字段顺序: symbol,exchange,datetime,interval,volume,turnover,open_interest, + # open_price,high_price,low_price,close_price + rows = [ + (v[0], v[1], v[2], "15m", float(v[7]), float(v[8]), 0.0, + float(v[3]), float(v[4]), float(v[5]), float(v[6])) + for v in values + ] + conn.executemany(INSERT_SQL, rows) + conn.commit() + return len(rows) + + +def read_back_one(conn: sqlite3.Connection, symbol: str, exchange: str) -> pd.DataFrame: + """从 DB 读回单只, 验证落库完整性。""" + sql = ( + "SELECT datetime, open_price, high_price, low_price, close_price, " + "volume, turnover FROM dbbardata " + "WHERE symbol=? AND exchange=? AND interval='15m' " + "ORDER BY datetime" + ) + return pd.read_sql(sql, conn, params=(symbol, exchange)) + + +# ======================== 主流程 ======================== +def main(): + print("=" * 72, flush=True) + print("BaoStock 15min 样本验证脚本", flush=True) + print("=" * 72, flush=True) + print(f"Python: {sys.executable}", flush=True) + print(f"测试库: {SAMPLE_DB}", flush=True) + print(f"样本股 ({len(SAMPLE_CODES)}): {SAMPLE_CODES}", flush=True) + print(f"日期范围: {START_DATE} ~ {END_DATE}", flush=True) + print(f"frequency={FREQUENCY} adjustflag={ADJUST_FLAG} (不复权 raw)", flush=True) + print(f"baostock 间隔: {BS_INTERVAL}s/只 (单进程串行,防封IP)", flush=True) + print(f"proxy 环境: 已 unset", flush=True) + print(f"当前时间: {datetime.datetime.now().isoformat()}", flush=True) + print("-" * 72, flush=True) + + # 1. 登录 baostock (带超时) + print("[1/4] 登录 baostock ...", flush=True) + if not login_with_timeout(): + print("[FATAL] baostock 登录失败, 退出 (避免硬撑被拉黑)", flush=True) + sys.exit(1) + print(" ✅ baostock 登录成功", flush=True) + + # 2. 初始化测试库 + print(f"[2/4] 初始化测试库 {SAMPLE_DB} ...", flush=True) + if os.path.exists(SAMPLE_DB): + os.remove(SAMPLE_DB) + print(f" 已删除旧库", flush=True) + conn = init_db(SAMPLE_DB) + print(f" ✅ 新建 dbbardata 表", flush=True) + + # 3. 逐只下载 → 灌库 + print(f"[3/4] 开始逐只下载 (串行, 间隔 {BS_INTERVAL}s) ...", flush=True) + print("-" * 72, flush=True) + + per_stock_results = [] # list of dict + total_rows = 0 + success_count = 0 + consec_fail = 0 + circuit_triggered = False + t_start = time.time() + + for idx, code in enumerate(SAMPLE_CODES, 1): + bs_code, exchange = code_to_baostock(code) + print(f"[{idx}/{len(SAMPLE_CODES)}] {code} ({bs_code} / {exchange})", + flush=True) + + try: + df = fetch_one(code, START_DATE, END_DATE) + except Exception as e: + print(f" ❌ 下载异常: {e}", flush=True) + df = None + + if df is None or df.empty: + print(f" ❌ 无数据,跳过灌库", flush=True) + per_stock_results.append({ + "code": code, "bs_code": bs_code, "exchange": exchange, + "status": "FAILED", "rows": 0, + "dt_min": "-", "dt_max": "-", + "first_ohlcv": "-", "last_ohlcv": "-", + "self_check": "-", + }) + consec_fail += 1 + if consec_fail >= CIRCUIT_THRESHOLD: + print(f"\n[FATAL] 连续 {consec_fail} 只失败, 触发断路器, 停止后续 " + f"(baostock 疑似不可达, 避免硬撑被拉黑)", flush=True) + circuit_triggered = True + break + if idx < len(SAMPLE_CODES): + time.sleep(BS_INTERVAL) + continue + + # 自检 + ok, check_report = self_check(df, code) + status_flag = "PASS" if ok else "WARN" + + # 灌库 + try: + n_inserted = insert_df(conn, df) + except Exception as e: + print(f" ❌ 灌库异常: {e}", flush=True) + per_stock_results.append({ + "code": code, "bs_code": bs_code, "exchange": exchange, + "status": "DB_FAIL", "rows": 0, + "dt_min": "-", "dt_max": "-", + "first_ohlcv": "-", "last_ohlcv": "-", + "self_check": check_report, + }) + consec_fail += 1 + if consec_fail >= CIRCUIT_THRESHOLD: + print(f"\n[FATAL] 连续 {consec_fail} 只失败, 触发断路器", flush=True) + circuit_triggered = True + break + if idx < len(SAMPLE_CODES): + time.sleep(BS_INTERVAL) + continue + + consec_fail = 0 # 成功重置 + total_rows += n_inserted + success_count += 1 + + # 详细打印 + dt_min = df["datetime"].min() + dt_max = df["datetime"].max() + first = df.iloc[0] + last = df.iloc[-1] + first_str = ( + f"dt={first['datetime']} O={first['open_price']:.2f} " + f"H={first['high_price']:.2f} L={first['low_price']:.2f} " + f"C={first['close_price']:.2f} V={int(first['volume'])} " + f"amt={first['turnover']:.0f}" + ) + last_str = ( + f"dt={last['datetime']} O={last['open_price']:.2f} " + f"H={last['high_price']:.2f} L={last['low_price']:.2f} " + f"C={last['close_price']:.2f} V={int(last['volume'])} " + f"amt={last['turnover']:.0f}" + ) + + # 读回验证 + readback = read_back_one(conn, code, exchange) + readback_ok = len(readback) == n_inserted + + print(f" ✅ 下载 {len(df)} 行 | 时间 {dt_min} ~ {dt_max}", flush=True) + print(f" 首行: {first_str}", flush=True) + print(f" 尾行: {last_str}", flush=True) + print(f" 自检 [{status_flag}]: {check_report}", flush=True) + print(f" 灌库 {n_inserted} 行 | 读回 {len(readback)} 行 " + f"({'✅ 一致' if readback_ok else '❌ 不一致'})", flush=True) + + per_stock_results.append({ + "code": code, "bs_code": bs_code, "exchange": exchange, + "status": "OK", "rows": n_inserted, + "dt_min": dt_min, "dt_max": dt_max, + "first_ohlcv": first_str, "last_ohlcv": last_str, + "self_check": check_report, + "readback_rows": len(readback), + "readback_ok": readback_ok, + }) + + # 间隔(最后一只不用 sleep) + if idx < len(SAMPLE_CODES): + time.sleep(BS_INTERVAL) + + elapsed = time.time() - t_start + + # 4. 登出 baostock + try: + bs.logout() + print("\n[4/4] baostock 已登出", flush=True) + except Exception as e: + print(f"\n[4/4] baostock 登出异常(忽略): {e}", flush=True) + + conn.close() + + # ======================== 汇总报告 ======================== + print("\n" + "=" * 72, flush=True) + print("汇总报告", flush=True) + print("=" * 72, flush=True) + + if not per_stock_results: + print("[FATAL] 所有样本均失败, 无数据可汇总", flush=True) + sys.exit(1) + + # 每只结果表 + print(f"\n{'code':<8}{'status':<10}{'rows':>10} {'dt_min':<20}{'dt_max':<20}", + flush=True) + print("-" * 72, flush=True) + for r in per_stock_results: + print(f"{r['code']:<8}{r['status']:<10}{r['rows']:>10} " + f"{str(r['dt_min']):<20}{str(r['dt_max']):<20}", flush=True) + + # 汇总指标 + all_mins = [r["dt_min"] for r in per_stock_results if r["status"] == "OK"] + all_maxs = [r["dt_max"] for r in per_stock_results if r["status"] == "OK"] + earliest = min(all_mins) if all_mins else "-" + latest = max(all_maxs) if all_maxs else "-" + success_rate = success_count / len(SAMPLE_CODES) * 100 + + print(f"\n总行数 (灌入 dbbardata): {total_rows}", flush=True) + print(f"成功率: {success_count}/{len(SAMPLE_CODES)} = {success_rate:.1f}%", + flush=True) + print(f"最早日期: {earliest} (期望接近 {START_DATE} 验证 baostock 深度)", + flush=True) + print(f"最晚日期: {latest}", flush=True) + print(f"总耗时: {elapsed:.1f}s ({elapsed / 60:.1f}min)", flush=True) + + # 茅台 600519 抽样(任务要求) + moutai = next((r for r in per_stock_results if r["code"] == "600519"), None) + if moutai and moutai["status"] == "OK": + print(f"\n茅台 600519 抽样 OHLCV (原始 baostock 值):", flush=True) + print(f" 首行: {moutai['first_ohlcv']}", flush=True) + print(f" 尾行: {moutai['last_ohlcv']}", flush=True) + print(f" 自检: {moutai['self_check']}", flush=True) + + # 全局自检总结 + print(f"\n自检总结 (所有 OK 的样本):", flush=True) + all_check_pass = True + for r in per_stock_results: + if r["status"] != "OK": + continue + flag = "PASS" if "PASS" in r["self_check"] and "FAIL" not in r["self_check"] else "WARN" + if "FAIL" in r["self_check"]: + all_check_pass = False + print(f" {r['code']:<8} [{flag}] {r['self_check']}", flush=True) + print(f"\n全局自检: {'✅ ALL PASS' if all_check_pass else '❌ 有 FAIL 项, 请检查'}", + flush=True) + + # 深度验证 (是否到 2020-01-01) + print(f"\n深度验证 (最早日期是否接近 {START_DATE}):", flush=True) + if all_mins: + earliest_code = per_stock_results[ + next(i for i, r in enumerate(per_stock_results) if r["dt_min"] == earliest) + ]["code"] if earliest in all_mins else "?" + # 严格匹配最早 dt_min 对应的 code + candidates = [r for r in per_stock_results if r["status"] == "OK"] + if candidates: + earliest_row = min(candidates, key=lambda r: r["dt_min"]) + print(f" 最早样本: {earliest_row['code']} ({earliest_row['bs_code']}) " + f"起始 {earliest_row['dt_min']}", flush=True) + if earliest.startswith("2020-"): + print(f" ✅ 深度达到 2020 年 (baostock 5.5 年历史可用)", flush=True) + elif earliest.startswith("2021") or earliest.startswith("2022"): + print(f" ⚠️ 深度晚于 2020-01-01 (可能新股上市晚, 看具体样本)", + flush=True) + else: + print(f" ❌ 深度严重不足, 检查 baostock 配置", flush=True) + + # 读回一致性验证总结 + readback_all_ok = all( + r.get("readback_ok", False) for r in per_stock_results + if r["status"] == "OK" + ) + print(f"\n读回一致性: {'✅ 所有样本 灌入行数 == 读回行数' if readback_all_ok else '❌ 有样本读回不一致'}", + flush=True) + + print("\n" + "=" * 72, flush=True) + if circuit_triggered: + print(f"[RESULT] 断路器触发, 部分样本未处理 - 请检查 baostock 状态后重试", + flush=True) + sys.exit(1) + elif success_count == len(SAMPLE_CODES) and all_check_pass and readback_all_ok: + print(f"[RESULT] ✅✅✅ 全部 10 只样本链路验证通过, 可以上全市场", flush=True) + sys.exit(0) + else: + print(f"[RESULT] ⚠️ 部分异常, 请审阅上面的详细报告", flush=True) + sys.exit(0 if success_count == len(SAMPLE_CODES) else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/baostock_constituent_download.py b/scripts/data_platform/baostock_constituent_download.py new file mode 100755 index 0000000..f908953 --- /dev/null +++ b/scripts/data_platform/baostock_constituent_download.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""BaoStock 成分股历史下载脚本 (只产 parquet) + +目标: + 在 Mac 本机用 baostock 拉取 HS300/ZZ500/SZ50 历史成分股快照, + 按历史日期循环 (每周一一个快照, 从 2006-01 至今)。 + 输出 parquet 到 staging 目录。 + +硬约束 (踩过的坑): + 1. 单进程单登录, 严禁并发 (baostock 并发会拉黑封 IP 6-24h) + 2. 直连不走代理 (脚本开头 unset proxy) + 3. 每次 query 后 sleep 0.4s 限速 + 4. staging parquet, 绝不直接写主库 + +用法: + python baostock_constituent_download.py # 全指数 2006-01~至今 + python baostock_constituent_download.py --limit 5 # 测试: 前 5 个快照 + python baostock_constituent_download.py --indices hs300 # 只跑指定指数 + python baostock_constituent_download.py --start 2020-01-01 # 指定起始日期 + +退出码: 0=完成, 1=致命错误, 2=断路器触发 (可重试) +""" + +import argparse +import datetime +import json +import logging +import os +import socket +import sys +import time +from datetime import date, timedelta +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(_k, None) + +socket.setdefaulttimeout(30) + +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + + +# ======================== 配置 ======================== + +DEFAULT_OUT_DIR = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/data/constituent_baostock" +OUT_DIR = Path(os.environ.get("BS_CONSTITUENT_OUT_DIR", DEFAULT_OUT_DIR)) +DEFAULT_LOG_DIR = OUT_DIR / "logs" +_log_dir_env = os.environ.get("BS_CONSTITUENT_LOG_DIR") +LOG_DIR = Path(_log_dir_env) if _log_dir_env else DEFAULT_LOG_DIR + +START_DATE_DEFAULT = "2006-01-01" # 成分股历史起始 +END_DATE_DEFAULT = "" # 默认今天 + +BS_INTERVAL = 0.4 # 每次间隔秒 (防封 IP) +BS_MAX_RETRIES = 3 # 单次下载重试次数 +PROGRESS_LOG_EVERY = 50 # 每 N 个快照打一次进度日志 +RELOGIN_EVERY = 100 # 每 N 个快照定期重登 +CIRCUIT_BREAKER = 10 # 连续失败 N 次 → 断路退出 + +# 指数配置 (baostock code, 显示名) +INDICES_CONFIG = { + "hs300": {"code": "000300", "name": "HS300"}, + "zz500": {"code": "000905", "name": "ZZ500"}, + "sz50": {"code": "000016", "name": "SZ50"}, +} + + +# ======================== 日志 ======================== + +def setup_logging() -> Tuple[logging.Logger, Path]: + """配置 root logger: 同时写 stdout + 文件。""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = LOG_DIR / f"baostock_constituent_{ts}.log" + + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + root = logging.getLogger() + root.setLevel(logging.INFO) + for h in list(root.handlers): + root.removeHandler(h) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + root.addHandler(sh) + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + root.addHandler(fh) + + return logging.getLogger(__name__), log_file + + +logger, LOG_FILE = setup_logging() + + +# ======================== 工具函数 ======================== + +def normalize_date(s: str) -> str: + """接受 YYYYMMDD 或 YYYY-MM-DD, 返回 YYYY-MM-DD; 空串 → 今天。""" + s = (s or "").strip() + if not s: + return datetime.date.today().strftime("%Y-%m-%d") + if "-" in s: + return s + if len(s) == 8 and s.isdigit(): + return f"{s[:4]}-{s[4:6]}-{s[6:8]}" + raise ValueError(f"无效日期格式: {s}") + + +def get_monday_dates(start_date: str, end_date: str) -> List[str]: + """生成每周一日期列表 (包含 start_date 所在周, 含 end_date)。 + + 返回: YYYY-MM-DD 格式日期列表。 + """ + start = datetime.datetime.strptime(start_date, "%Y-%m-%d").date() + end = datetime.datetime.strptime(end_date, "%Y-%m-%d").date() + + # 找到 start_date 后的第一个周一 + d = start + while d.weekday() != 0: # Monday=0 + d += timedelta(days=1) + + dates = [] + while d <= end: + dates.append(d.strftime("%Y-%m-%d")) + d += timedelta(days=7) # 下一周 + + return dates + + +def baostock_code_to_parts(bs_code: str) -> Tuple[str, str]: + """sh.600519 → (600519, 'SH'); sz.000001 → (000001, 'SZ')。""" + if "." not in bs_code: + raise ValueError(f"无效 baostock code: {bs_code}") + prefix, num = bs_code.split(".", 1) + if prefix == "sh": + return num, "SH" + if prefix == "sz": + return num, "SZ" + raise ValueError(f"未知 baostock 前缀: {bs_code}") + + +# ======================== baostock 登录 ======================== + +def _login_once() -> bool: + """bs.login() — 已被 socket.setdefaulttimeout(30) 保护。""" + try: + lg = bs.login() + if lg.error_code == "0": + return True + logger.error("bs.login() 失败: code=%s msg=%s", lg.error_code, lg.error_msg) + return False + except (socket.timeout, TimeoutError) as e: + logger.error("bs.login() 超时: %s (baostock 疑似冷却)", e) + return False + except Exception as e: + logger.error("bs.login() 异常: %s", e) + return False + + +def _relogin() -> bool: + """强制重登: logout + login。失败等 2s 再试 1 次。""" + try: + bs.logout() + except Exception: + pass + if _login_once(): + return True + time.sleep(2) + try: + bs.logout() + except Exception: + pass + return _login_once() + + +# ======================== 数据下载 ======================== + +def fetch_constituent_stocks(index_name: str, date_str: str) -> pd.DataFrame: + """拉取指定指数在指定日期的成分股。 + + 返回 df 列: code (baostock格式, 如 sh.600519) + 空数据返回空 df。 + """ + index_info = INDICES_CONFIG.get(index_name) + if not index_info: + raise ValueError(f"未知指数: {index_name}") + + index_code = index_info["code"] + + if index_name == "hs300": + rs = bs.query_hs300_stocks(date=date_str) + elif index_name == "zz500": + rs = bs.query_zz500_stocks(date=date_str) + elif index_name == "sz50": + rs = bs.query_sz50_stocks(date=date_str) + else: + raise ValueError(f"未实现的指数: {index_name}") + + if rs.error_code != "0": + raise RuntimeError( + f"query_{index_name}_stocks({date_str}) 错误: {rs.error_code} {rs.error_msg}" + ) + + if rs.error_code != "0": + raise RuntimeError( + f"query_{index_name}_stocks({date_str}) 错误: {rs.error_code} {rs.error_msg}" + ) + + # 获取字段列表 + fields = list(rs.fields) + + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + + if not rows: + # 合法空数据 (指数不存在于该日期) + return pd.DataFrame(columns=fields) + + # 使用 baostock 实际返回的字段名 + return pd.DataFrame(rows, columns=fields) + + +# ======================== 路径 / marker ======================== + +def parquet_path_for(index_name: str, date_str: str) -> Path: + """hs300 + 2020-01-06 → OUT_DIR / hs300_2020-01-06.parquet""" + return OUT_DIR / f"{index_name}_{date_str}.parquet" + + +def marker_path_for(parquet_path: Path) -> Path: + """parquet → .{stem}.baostock marker""" + return parquet_path.parent / f".{parquet_path.stem}.baostock" + + +def load_done_set(indices: List[str]) -> Dict[str, set]: + """扫 OUT_DIR 所有 marker 构造已完成集合。 + + 返回: {index_name: set(dates)} 字典。 + """ + done: Dict[str, set] = {idx: set() for idx in indices} + if not OUT_DIR.exists(): + return done + + suffix = ".baostock" + for marker in OUT_DIR.glob(f".*{suffix}"): + name = marker.name + if not name.startswith(".") or not name.endswith(suffix): + continue + stem = name[1:-len(suffix)] # e.g. hs300_2020-01-06 + + # 解析: _ + try: + parts = stem.split("_", 1) + if len(parts) != 2: + continue + index_name, date_str = parts + if index_name in done: + done[index_name].add(date_str) + except ValueError: + continue + return done + + +# ======================== 单次下载 ======================== + +def download_one_snapshot( + index_name: str, + date_str: str, + force: bool, +) -> Tuple[str, int]: + """下载单个成分股快照 → 写 parquet + marker。 + + 返回 (status, rows): status ∈ {'ok', 'skipped', 'failed', 'empty'}。 + """ + parquet_path = parquet_path_for(index_name, date_str) + marker_path = marker_path_for(parquet_path) + + if not force and marker_path.exists(): + return "skipped", 0 + + # retry 循环 + df: Optional[pd.DataFrame] = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_constituent_stocks(index_name, date_str) + break + except (socket.timeout, TimeoutError, OSError) as e: + logger.warning( + "%s %s socket 异常重试 %d/%d: %s — 强制重登", + index_name, date_str, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s %s", index_name, date_str) + df = None + break + except Exception as e: + logger.warning( + "%s %s 下载异常重试 %d/%d: %s — 强制重登", + index_name, date_str, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s %s", index_name, date_str) + df = None + break + + if df is None: + return "failed", 0 + + if df.empty: + # 合法空数据 (指数不存在于该日期) — 也写 marker + try: + OUT_DIR.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path.write_text(datetime.datetime.now().isoformat()) + return "empty", 0 + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return "failed", 0 + + # 写 parquet + marker + try: + OUT_DIR.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path.write_text(datetime.datetime.now().isoformat()) + return "ok", len(df) + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return "failed", 0 + + +# ======================== 主流程 ======================== + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="BaoStock 成分股历史下载 (只产 parquet)", + ) + p.add_argument( + "--start", default=START_DATE_DEFAULT, + help=f"起始日期 YYYYMMDD 或 YYYY-MM-DD, 默认 {START_DATE_DEFAULT}", + ) + p.add_argument("--end", default=END_DATE_DEFAULT, help="结束日期, 默认今天") + p.add_argument( + "--indices", + default="hs300,zz500,sz50", + help="逗号分隔指数名称, 默认 hs300,zz500,sz50", + ) + p.add_argument("--limit", type=int, default=0, help="限制处理快照数 (测试用)") + p.add_argument("--force", action="store_true", help="强制重下, 忽略 marker") + return p.parse_args() + + +def resolve_indices(s: str) -> List[str]: + """解析 --indices 字符串 → 去重保序的指数列表。""" + parts = [p.strip() for p in s.split(",") if p.strip()] + valid = set(INDICES_CONFIG.keys()) + bad = [p for p in parts if p not in valid] + if bad: + raise SystemExit(f"未知 --indices: {bad}, 可选 {list(valid)}") + seen: set = set() + out: List[str] = [] + for p in parts: + if p not in seen: + out.append(p) + seen.add(p) + return out + + +def main() -> None: + args = parse_args() + start_date = normalize_date(args.start) + end_date = normalize_date(args.end) + indices = resolve_indices(args.indices) + + logger.info("=" * 60) + logger.info("BaoStock 成分股历史下载") + logger.info(" 输出目录: %s", OUT_DIR) + logger.info(" 日志文件: %s", LOG_FILE) + logger.info(" 日期范围: %s ~ %s", start_date, end_date) + logger.info(" 指数: %s", indices) + logger.info(" 频率: 每周一快照") + logger.info(" socket.setdefaulttimeout(30)") + logger.info(" 当前时间: %s", datetime.datetime.now().isoformat()) + logger.info("=" * 60) + + # 登录 + if not _login_once(): + logger.error("[FATAL] baostock 登录失败 (疑似冷却), 退出") + sys.exit(1) + logger.info("baostock 登录成功") + + # 生成日期列表 (每周一) + monday_dates = get_monday_dates(start_date, end_date) + logger.info("快照日期列表: %d 个周一 (%s ~ %s)", len(monday_dates), + monday_dates[0] if monday_dates else "N/A", + monday_dates[-1] if monday_dates else "N/A") + + # limit + if args.limit > 0: + monday_dates = monday_dates[:args.limit] + logger.info("limit=%d 截断", args.limit) + + # 总快照数 + total_snapshots = len(monday_dates) * len(indices) + logger.info("总快照数: %d (日期%d × 指数%d)", total_snapshots, + len(monday_dates), len(indices)) + + # 断点续传 + done_set = load_done_set(indices) + if args.force: + todo_snapshots = [(idx, d) for idx in indices for d in monday_dates] + else: + todo_snapshots = [ + (idx, d) for idx in indices for d in monday_dates + if d not in done_set.get(idx, set()) + ] + logger.info("待处理: %d (已完成 %d)", len(todo_snapshots), + total_snapshots - len(todo_snapshots)) + + # 主循环 + stats = {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "rows": 0} + consec_fail = 0 + circuit_triggered = False + t_start = time.time() + + for i, (index_name, date_str) in enumerate(todo_snapshots): + # 定期重登 + if i > 0 and i % RELOGIN_EVERY == 0: + logger.info("定期重登 @ %d/%d", i, len(todo_snapshots)) + if not _relogin(): + logger.warning("定期重登失败, 继续") + + try: + status, rows = download_one_snapshot(index_name, date_str, args.force) + except Exception as e: + status, rows = "failed", 0 + logger.debug("download_one_snapshot %s %s 异常: %s", index_name, date_str, e) + + stats[status] = stats.get(status, 0) + 1 + if status == "ok": + stats["rows"] += rows + consec_fail = 0 + elif status == "failed": + consec_fail += 1 + + # 断路器 + if consec_fail >= CIRCUIT_BREAKER: + logger.error( + "[FATAL] 断路器触发: 连续 %d 次失败, baostock 疑似不可达", + consec_fail, + ) + circuit_triggered = True + break + + # 进度日志 + if (i + 1) % PROGRESS_LOG_EVERY == 0: + elapsed = time.time() - t_start + logger.info( + "进度 %d/%d ok=%d skipped=%d empty=%d failed=%d rows=%d (%.0f秒)", + i + 1, len(todo_snapshots), stats["ok"], stats["skipped"], + stats["empty"], stats["failed"], stats["rows"], elapsed, + ) + + # 限速 + if i < len(todo_snapshots) - 1: + time.sleep(BS_INTERVAL) + + # 登出 + try: + bs.logout() + except Exception: + pass + + elapsed = time.time() - t_start + logger.info("=" * 60) + if circuit_triggered: + logger.info("[RESULT] 断路器触发中止, 耗时 %.1f 秒", elapsed) + else: + logger.info("[RESULT] 完成, 耗时 %.1f 秒", elapsed) + logger.info("统计: %s", json.dumps(stats, ensure_ascii=False)) + + sys.exit(2 if circuit_triggered else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/baostock_daily_fullmarket_download.py b/scripts/data_platform/baostock_daily_fullmarket_download.py new file mode 100755 index 0000000..9f424a4 --- /dev/null +++ b/scripts/data_platform/baostock_daily_fullmarket_download.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""BaoStock 日K线全市场全字段下载脚本 (只产 parquet) + +目标: + 在 Mac 本机用 baostock 全量拉取 A 股 (含退市) 日K线全字段历史, + 输出 parquet 到 staging 目录, 后续验证+合并主库 (不直接写 DB). + +与已有脚本复用: + - baostock_15min_fullmarket_download.py 模板: 单进程单登录/marker 断点续传/query 计数器 + - baostock_static_download.py: 复权因子+除权除息已覆盖, 不重复 + +核心价值字段 (补 DB 日线缺口): + turn(换手率), pctChg(涨跌幅), peTTM(市盈率TTM), pbMRQ(市净率), + psTTM(市销率TTM), pcfNcfTTM(市现率TTM), isST(ST标记) + +硬约束 (踩过的坑): + 1. 单进程单登录, 严禁并发 (baostock 并发会拉黑封 IP 6-24h) + 2. 直连不走代理 (脚本开头 unset proxy) + 3. 每只 sleep 0.4s 限速 + 4. baostock 每日 ≤ 48000 query (硬上限, 超额即拉黑) + 5. staging parquet, 绝不直接写主库 (quant_trading.db) + +用法: + python baostock_daily_fullmarket_download.py # 全市场 1990-12-19~今天 + python baostock_daily_fullmarket_download.py --limit 10 # 测试: 前 10 只 + python baostock_daily_fullmarket_download.py --codes 600519,000001 # 指定 + python baostock_daily_fullmarket_download.py --force # 强制重下, 忽略 marker + python baostock_daily_fullmarket_download.py --shard 0 --total 4 # 分片 + +退出码: 0=完成, 1=致命错误 (登录失败/列表拉取失败), 2=断路器触发 (可重试) +""" + +import argparse +import datetime +import json +import logging +import os +import socket +import sys +import time +from pathlib import Path +from typing import List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +# 必须在 import baostock 之前清理 (baostock 底层 urllib 会读 proxy 环境变量) +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(_k, None) + +# SIGALRM 跨平台替代: 全局 socket 超时 30s。 +# baostock 的 socket 连接会继承这个超时, connect 卡死时抛 socket.timeout, +# 捕获后走 _relogin 重试。 +socket.setdefaulttimeout(30) + +# stdout 行缓冲 (SSH-detached / pipe 重定向时也能看到实时进度) +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass # 不是 TextIOWrapper 或已关闭 + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + + +# ======================== 配置 (环境变量可覆盖) ======================== + +DEFAULT_OUT_DIR = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/data/daily_baostock" +OUT_DIR = Path(os.environ.get("BS_DAILY_OUT_DIR", DEFAULT_OUT_DIR)) +DEFAULT_LOG_DIR = OUT_DIR / "logs" +_log_dir_env = os.environ.get("BS_DAILY_LOG_DIR") +LOG_DIR = Path(_log_dir_env) if _log_dir_env else DEFAULT_LOG_DIR + +START_DATE_DEFAULT = "1990-12-19" # A 股最早交易日 + +BS_INTERVAL = 0.4 # 每只间隔秒 (防封 IP) +BS_MAX_RETRIES = 3 # 单只下载重试次数 (含强制重登) +PROGRESS_LOG_EVERY = 500 # 每 N 只打一次进度日志 +RELOGIN_EVERY = 400 # 每 N 只定期重登保持连接 +CIRCUIT_BREAKER = 30 # 连续失败 N 只 → 断路退出 + +# baostock 查询全字段 (日K线, 不复权 raw) +BS_FIELDS = "date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST" + +# parquet 输出列 (保持 baostock 原始字段名) +OUT_COLS = BS_FIELDS.split(",") + +# query 计数器 (防超 48000/天硬限) +QUERY_COUNT = 0 +DAILY_LIMIT = 40000 # 接近 40000 自动停 (留余量) + + +# ======================== 日志 ======================== + +def setup_logging() -> Tuple[logging.Logger, Path]: + """配置 root logger: 同时写 stdout + 文件。返回 (logger, log_file)。""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = LOG_DIR / f"baostock_daily_fullmarket_{ts}.log" + + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + root = logging.getLogger() + root.setLevel(logging.INFO) + # 清掉已有 handler, 防 re-init 重复 + for h in list(root.handlers): + root.removeHandler(h) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + root.addHandler(sh) + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + root.addHandler(fh) + + return logging.getLogger(__name__), log_file + + +logger, LOG_FILE = setup_logging() + + +# ======================== 工具函数 ======================== + +def normalize_date(s: str) -> str: + """接受 YYYYMMDD 或 YYYY-MM-DD, 返回 YYYY-MM-DD; 空串 → 今天。""" + s = (s or "").strip() + if not s: + return datetime.date.today().strftime("%Y-%m-%d") + if "-" in s: + return s + if len(s) == 8 and s.isdigit(): + return f"{s[:4]}-{s[4:6]}-{s[6:8]}" + raise ValueError(f"无效日期格式: {s} (期望 YYYYMMDD 或 YYYY-MM-DD)") + + +def baostock_code_to_parts(bs_code: str) -> Tuple[str, str]: + """sh.600519 → (600519, 'SH'); sz.000001 → (000001, 'SZ')。 + 前缀直接从 baostock code 拿, 不猜。 + """ + if "." not in bs_code: + raise ValueError(f"无效 baostock code: {bs_code}") + prefix, num = bs_code.split(".", 1) + if prefix == "sh": + return num, "SH" + if prefix == "sz": + return num, "SZ" + raise ValueError(f"未知 baostock 前缀: {bs_code}") + + +def parts_to_baostock(code: str, exchange: str) -> str: + """(600519, 'SH') → 'sh.600519'。""" + pfx = "sh" if exchange == "SH" else "sz" + return f"{pfx}.{code}" + + +def guess_exchange_by_code(code: str) -> str: + """无前缀时按代码开头猜交易所 (与 15min 模板一致)。 + 6/68/51 开头 → SH, 其他 → SZ。仅用于 --codes 指定无前缀代码。 + """ + if code.startswith(("6", "68", "51")): + return "SH" + return "SZ" + + +# ======================== 股票列表 (baostock query_stock_basic) ======================== + +def fetch_all_stocks() -> List[Tuple[str, str]]: + """从 baostock 拉全市场 A 股列表 (含退市, 避免生存偏差)。 + + 过滤: type=='1' (股票); **不过滤 status** (保留退市股, 避免生存偏差)。 + 返回: [(code, exchange), ...], 例 [('600519', 'SH'), ('000001', 'SZ')] + """ + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError( + f"query_stock_basic 失败: code={rs.error_code} msg={rs.error_msg}" + ) + + fields = list(rs.fields) + idx = {name: i for i, name in enumerate(fields)} + logger.info("query_stock_basic fields=%s", fields) + + out: List[Tuple[str, str]] = [] + n_stock = 0 + n_skip_type = 0 + n_skip_code = 0 + while rs.next(): + r = rs.get_row_data() + type_ = r[idx["type"]] if "type" in idx and len(r) > idx["type"] else "" + if type_ != "1": # 只要股票 (排除指数/债券/其他) + n_skip_type += 1 + continue + bs_code = r[idx["code"]] if "code" in idx and len(r) > idx["code"] else "" + # status 不过滤 (保留退市, 避免生存偏差) + try: + code, exchange = baostock_code_to_parts(bs_code) + except ValueError: + n_skip_code += 1 + continue + if len(code) != 6 or not code.isdigit(): + n_skip_code += 1 + continue + out.append((code, exchange)) + n_stock += 1 + logger.info( + "股票列表: stocks(type=1)=%d skip_type=%d skip_code=%d (含退市, 不过滤 status)", + n_stock, n_skip_type, n_skip_code) + return out + + +# ======================== baostock 登录 ======================== + +def _login_once() -> bool: + """bs.login() — 已被 socket.setdefaulttimeout(30) 保护。 + + connect 卡死会抛 socket.timeout, 这里捕获。 + 返回 True=登录成功。 + """ + try: + lg = bs.login() + if lg.error_code == "0": + return True + logger.error("bs.login() 失败: code=%s msg=%s", lg.error_code, lg.error_msg) + return False + except (socket.timeout, TimeoutError) as e: + logger.error("bs.login() 超时: %s (baostock 疑似冷却)", e) + return False + except Exception as e: + logger.error("bs.login() 异常: %s", e) + return False + + +def _relogin() -> bool: + """强制重登: logout + login。失败等 2s 再试 1 次。""" + try: + bs.logout() + except Exception: + pass + if _login_once(): + return True + time.sleep(2) + try: + bs.logout() + except Exception: + pass + return _login_once() + + +# ======================== 数据下载 ======================== + +def fetch_one_daily( + bs_code: str, start_date: str, end_date: str, +) -> Optional[pd.DataFrame]: + """从 baostock 下载日K线全字段不复权数据。 + + 返回 df 列 (保持 baostock 原始字段名): + date, code, open, high, low, close, preclose, volume, amount, + adjustflag, turn, tradestatus, pctChg, peTTM, psTTM, pcfNcfTTM, pbMRQ, isST + 失败/空数据返回 None。 + """ + global QUERY_COUNT + QUERY_COUNT += 1 + + rs = bs.query_history_k_data_plus( + bs_code, + BS_FIELDS, + start_date=start_date, + end_date=end_date, + frequency="d", # 日K线 + adjustflag="3", # 不复权 raw + ) + if rs.error_code != "0": + # 真查询错误 (baostock 端报错) — 抛异常让上层 retry/relogin。 + raise RuntimeError( + f"baostock {bs_code} 查询错误: {rs.error_code} {rs.error_msg}" + ) + + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + if not rows: + # 合法空数据 (退市/区间外/新股未交易) — 返回空 df, 上层计 "empty" 中性, 不重试。 + return pd.DataFrame(columns=OUT_COLS) + + df = pd.DataFrame(rows, columns=OUT_COLS) + + # 数值转换 (baostock 返回字符串) + numeric_cols = ["open", "high", "low", "close", "preclose", "volume", "amount", + "turn", "pctChg", "peTTM", "psTTM", "pcfNcfTTM", "pbMRQ"] + for col in numeric_cols: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + + # 数据自检: dropna OHLC + 过滤非法 OHLC + df = df.dropna(subset=["open", "high", "low", "close"]) + bad_ohlc = (df["high"] < df[["open", "close"]].max(axis=1)) | \ + (df["low"] > df[["open", "close"]].min(axis=1)) + if bad_ohlc.any(): + df = df[~bad_ohlc] + if df.empty: + # 自检后全过滤掉 (脏数据) — 也算合法空, 不重试。 + return pd.DataFrame(columns=OUT_COLS) + + return df.sort_values("date").reset_index(drop=True) + + +# ======================== marker / parquet 路径 ======================== + +def parquet_path_for(code: str, exchange: str) -> Path: + """600519 + SH → OUT_DIR / 600519.SH_daily.parquet。""" + return OUT_DIR / f"{code}.{exchange}_daily.parquet" + + +def marker_path_for(parquet_path: Path) -> Path: + """parquet → .{stem}.baostock marker (同目录)。""" + return parquet_path.parent / f".{parquet_path.stem}.baostock" + + +def load_done_set() -> set: + """扫 OUT_DIR 所有 marker 构造已完成集合 (真相源)。 + + marker 文件名格式: .600519.SH_daily.baostock + 只有 marker 存在 = 该票成功完成; failed 票无 marker, 下次复跑会重试。 + """ + done = set() + if not OUT_DIR.exists(): + return done + suffix = ".baostock" + for marker in OUT_DIR.glob(f".*{suffix}"): + name = marker.name + if not name.startswith(".") or not name.endswith(suffix): + continue + stem = name[1:-len(suffix)] # e.g. 600519.SH_daily + # 解析: ._daily + try: + head, period = stem.rsplit("_", 1) + if period != "daily": + continue + code, exc = head.split(".", 1) + if len(code) == 6 and code.isdigit() and exc in ("SH", "SZ"): + done.add((code, exc)) + except ValueError: + continue + return done + + +# ======================== 单只下载 ======================== + +def download_one( + code: str, + exchange: str, + start_date: str, + end_date: str, + force: bool, +) -> Tuple[str, int]: + """下载单只 → 写 parquet + marker。 + + 返回 (status, rows): status ∈ {'ok', 'skipped', 'failed', 'empty'}。 + """ + global QUERY_COUNT + + # query 计数器检查 (接近 40000 自动停) + if QUERY_COUNT >= DAILY_LIMIT: + logger.warning("query 计数器已达 %d, 接近 48000 上限, graceful stop", QUERY_COUNT) + return "limit_reached", 0 + + parquet_path = parquet_path_for(code, exchange) + marker_path = marker_path_for(parquet_path) + + if not force and marker_path.exists(): + return "skipped", 0 + + bs_code = parts_to_baostock(code, exchange) + + # retry 循环: fetch_one_daily 仅在真错误时抛异常 (走 retry/relogin); + # 空数据返回空 df (合法, 不重试), 成功 break。 + df: Optional[pd.DataFrame] = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_one_daily(bs_code, start_date, end_date) + break # fetch 成功 (df 可能空 = 合法无数据) + except (socket.timeout, TimeoutError, OSError) as e: + logger.warning( + "%s socket 异常重试 %d/%d: %s — 强制重登", + code, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s", code) + df = None + break + except Exception as e: + logger.warning( + "%s 下载异常重试 %d/%d: %s — 强制重登", + code, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s", code) + df = None + break + + if df is None: + return "failed", 0 # 多次重试仍报错 (真失败, 计断路器) + if df.empty: + return "empty", 0 # 合法无数据 (退市/区间外), 中性, 不计失败不重试 + + # 写 parquet + marker (只有成功写完且自检通过才写 marker) + try: + OUT_DIR.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path.write_text(datetime.datetime.now().isoformat()) + return "ok", len(df) + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return "failed", 0 + + +# ======================== 主流程 ======================== + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="BaoStock 日K线全市场全字段下载 (只产 parquet)", + ) + p.add_argument( + "--start", default=START_DATE_DEFAULT, + help=f"起始日期 YYYYMMDD 或 YYYY-MM-DD, 默认 {START_DATE_DEFAULT}", + ) + p.add_argument("--end", default="", help="结束日期, 默认今天") + p.add_argument("--codes", help="指定代码逗号分隔, 如 600519,000001 (测试用)") + p.add_argument("--limit", type=int, default=0, help="限制处理数量 (测试用)") + p.add_argument("--force", action="store_true", help="强制重下, 忽略 marker") + p.add_argument("--shard", type=int, default=None, help="分片号 0-indexed (并行下载用)") + p.add_argument("--total", type=int, default=1, help="总分片数 (与 --shard 配合)") + return p.parse_args() + + +def resolve_codes(args: argparse.Namespace) -> List[Tuple[str, str]]: + """根据 args 解析待处理的 (code, exchange) 列表。""" + if args.codes: + # 指定代码: 无前缀, 按代码开头猜交易所 (与 15min 风格一致) + return [ + (c.strip(), guess_exchange_by_code(c.strip())) + for c in args.codes.split(",") + if c.strip() + ] + return fetch_all_stocks() + + +def main() -> None: + global QUERY_COUNT + + args = parse_args() + start_date = normalize_date(args.start) + end_date = normalize_date(args.end) + + logger.info("=" * 60) + logger.info("BaoStock 日K线全市场全字段下载") + logger.info(" 输出目录: %s", OUT_DIR) + logger.info(" 日志文件: %s", LOG_FILE) + logger.info(" 日期范围: %s ~ %s", start_date, end_date) + logger.info(" frequency=d, adjustflag=3 (不复权 raw)") + logger.info(" 全字段: turn/pctChg/peTTM/pbMRQ/psTTM/pcfNcfTTM/isST") + logger.info(" socket.setdefaulttimeout(30) (SIGALRM 替代)") + logger.info(" query 计数器: %d/%d (接近上限自动停)", QUERY_COUNT, DAILY_LIMIT) + logger.info(" 当前时间: %s", datetime.datetime.now().isoformat()) + if args.shard is not None: + logger.info(" 分片: shard=%s/%s", args.shard, args.total) + logger.info("=" * 60) + + # 登录 + if not _login_once(): + logger.error("[FATAL] baostock 登录失败 (疑似冷却), 退出") + sys.exit(1) + logger.info("baostock 登录成功") + + # 股票列表 + try: + all_codes = resolve_codes(args) + except Exception as e: + logger.error("[FATAL] 获取股票列表失败: %s", e) + try: + bs.logout() + except Exception: + pass + sys.exit(1) + logger.info("全市场 A 股 (含退市): %d 只", len(all_codes)) + + # limit + if args.limit > 0: + all_codes = all_codes[:args.limit] + logger.info("limit=%d 截断", args.limit) + + # 分片 + if args.shard is not None and args.total > 1: + all_codes = [ + s for i, s in enumerate(all_codes) + if i % args.total == args.shard + ] + logger.info("shard=%s/%s 过滤后: %d 只", args.shard, args.total, len(all_codes)) + + # 断点续传 (marker 为真相源) + done_set = load_done_set() + if args.force: + todo = all_codes + else: + todo = [(c, e) for c, e in all_codes if (c, e) not in done_set] + logger.info("待处理: %d (已完成 %d)", len(todo), len(done_set)) + + # 主循环 + stats = {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "limit_reached": 0, "rows": 0} + consec_fail = 0 + circuit_triggered = False + limit_reached = False + t_start = time.time() + + for i, (code, exc) in enumerate(todo): + # 定期重登保持连接 + if i > 0 and i % RELOGIN_EVERY == 0: + logger.info("定期重登 @ %d/%d (query=%d)", i, len(todo), QUERY_COUNT) + if not _relogin(): + logger.warning("定期重登失败, 继续 (单次失败不致命)") + + try: + status, rows = download_one(code, exc, start_date, end_date, args.force) + except Exception as e: + status, rows = "failed", 0 + logger.debug("download_one %s.%s 异常: %s", code, exc, e) + + stats[status] = stats.get(status, 0) + 1 + if status == "ok": + stats["rows"] += rows + consec_fail = 0 + elif status == "failed": + consec_fail += 1 + elif status == "limit_reached": + limit_reached = True + break + # skipped / empty 中性: 不重置也不递增 (不证明 baostock 可用, 也不算失败) + + # 断路器: 连续 N 只全 failed → baostock 疑似不可达, 保存进度主动退出 + if consec_fail >= CIRCUIT_BREAKER: + logger.error( + "[FATAL] 断路器触发: 连续 %d 只失败, baostock 疑似不可达, " + "退出 (done_set 不含 failed 票, 复跑会重试)", + consec_fail, + ) + circuit_triggered = True + break + + # 进度日志 + if (i + 1) % PROGRESS_LOG_EVERY == 0: + elapsed = time.time() - t_start + logger.info( + "进度 %d/%d ok=%d skipped=%d empty=%d failed=%d rows=%d query=%d (%.0f秒)", + i + 1, len(todo), stats["ok"], stats["skipped"], + stats["empty"], stats["failed"], stats["rows"], QUERY_COUNT, elapsed, + ) + + # 限速 + if i < len(todo) - 1: + time.sleep(BS_INTERVAL) + + # 登出 + try: + bs.logout() + except Exception: + pass + + elapsed = time.time() - t_start + logger.info("=" * 60) + if circuit_triggered: + logger.info("[RESULT] 断路器触发中止, 耗时 %.1f 秒", elapsed) + elif limit_reached: + logger.info("[RESULT] query 计数器触达上限, graceful stop, 耗时 %.1f 秒", elapsed) + else: + logger.info("[RESULT] 完成, 耗时 %.1f 秒", elapsed) + logger.info("统计: %s", json.dumps(stats, ensure_ascii=False)) + logger.info("query 总计: %d", QUERY_COUNT) + + sys.exit(2 if circuit_triggered else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/baostock_day1_wrapper.ps1 b/scripts/data_platform/baostock_day1_wrapper.ps1 new file mode 100644 index 0000000..6d77c40 --- /dev/null +++ b/scripts/data_platform/baostock_day1_wrapper.ps1 @@ -0,0 +1,35 @@ +# baostock Day1 全量下载 wrapper（成份股 → 日K线 串行，单进程单登录） +# 环境变量覆盖脚本默认 Mac 路径 → Windows 输出路径 +$ErrorActionPreference = 'Continue' +$ProgressPreference = 'SilentlyContinue' +$py = 'C:\Python310\python.exe' +$dir = 'C:\sanguo_vnpy_v2\scripts\data_platform' +$env:BS_DAILY_OUT_DIR = 'C:\sanguo_vnpy_v2\data\daily_baostock' +$env:BS_CONSTITUENT_OUT_DIR = 'C:\sanguo_vnpy_v2\data\constituent_baostock' +$env:PYTHONIOENCODING = 'utf-8' +$wrapper_log = 'C:\sanguo_vnpy_v2\data\baostock_day1_wrapper.log' +$const_log = 'C:\sanguo_vnpy_v2\data\constituent_baostock_download.log' +$daily_log = 'C:\sanguo_vnpy_v2\data\daily_baostock_download.log' + +function Log-W($m) { + "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $m" | Out-File -FilePath $wrapper_log -Append -Encoding UTF8 +} + +Log-W "=== WRAPPER START ===" +Log-W "BS_DAILY_OUT_DIR=$($env:BS_DAILY_OUT_DIR)" +Log-W "DiskFree_GB_start=$((Get-PSDrive C).Free / 1GB)" + +# PHASE1 成份股历史（快，~20min，~1200 query）— 同时验证 VPS 环境(login/网络/parquet写) +Log-W "PHASE1 constituent START" +& $py -X utf8 "$dir\baostock_constituent_download.py" *>&1 | Tee-Object -FilePath $const_log +$const_exit = $LASTEXITCODE +Log-W "PHASE1 constituent DONE exit=$const_exit" + +# PHASE2 日K线全市场全字段（慢，~15h，~5537 query）— 主线，过夜跑 +Log-W "PHASE2 daily_kline START" +& $py -X utf8 "$dir\baostock_daily_fullmarket_download.py" *>&1 | Tee-Object -FilePath $daily_log +$daily_exit = $LASTEXITCODE +Log-W "PHASE2 daily_kline DONE exit=$daily_exit" + +Log-W "DiskFree_GB_end=$((Get-PSDrive C).Free / 1GB)" +Log-W "=== WRAPPER EXIT const_exit=$const_exit daily_exit=$daily_exit ===" diff --git a/scripts/data_platform/baostock_day2a_wrapper.ps1 b/scripts/data_platform/baostock_day2a_wrapper.ps1 new file mode 100644 index 0000000..51bd413 --- /dev/null +++ b/scripts/data_platform/baostock_day2a_wrapper.ps1 @@ -0,0 +1,24 @@ +# baostock Day2a: 复权因子 + 基本资料（5538 query, ~1.5h） +# dividend 单独 Day2b 明天跑（38759 query, 7年默认, 单独一天 fresh 预算） +$ErrorActionPreference = 'Continue' +$ProgressPreference = 'SilentlyContinue' +$py = 'C:\Python310\python.exe' +$dir = 'C:\sanguo_vnpy_v2\scripts\data_platform' +$env:BS_STATIC_OUT_DIR = 'C:\sanguo_vnpy_v2\data\static' +$env:PYTHONIOENCODING = 'utf-8' +$wrapper_log = 'C:\sanguo_vnpy_v2\data\baostock_day2a_wrapper.log' +$dl_log = 'C:\sanguo_vnpy_v2\data\baostock_day2a_download.log' + +function Log-W($m) { + "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $m" | Out-File -FilePath $wrapper_log -Append -Encoding UTF8 +} + +Log-W "=== DAY2a START (adjust_factor + basic) ===" +Log-W "BS_STATIC_OUT_DIR=$($env:BS_STATIC_OUT_DIR)" +Log-W "DiskFree_GB_start=$((Get-PSDrive C).Free / 1GB)" + +& $py -X utf8 "$dir\baostock_static_download.py" --types basic,adjust_factor *>&1 | Tee-Object -FilePath $dl_log +$exit_code = $LASTEXITCODE +Log-W "DAY2a DONE exit=$exit_code" +Log-W "DiskFree_GB_end=$((Get-PSDrive C).Free / 1GB)" +Log-W "=== DAY2a WRAPPER EXIT ===" diff --git a/scripts/data_platform/baostock_day2b_wrapper.ps1 b/scripts/data_platform/baostock_day2b_wrapper.ps1 new file mode 100644 index 0000000..69c1390 --- /dev/null +++ b/scripts/data_platform/baostock_day2b_wrapper.ps1 @@ -0,0 +1,23 @@ +# baostock Day2b: dividend 除权除息（DIVIDEND_YEAR_RANGE=2020-2026 默认7年, 38759 query） +# 定时 07-22 02:00 跑（fresh 48000 预算, 与 Day1/Day2a 隔天） +$ErrorActionPreference = 'Continue' +$ProgressPreference = 'SilentlyContinue' +$py = 'C:\Python310\python.exe' +$dir = 'C:\sanguo_vnpy_v2\scripts\data_platform' +$env:BS_STATIC_OUT_DIR = 'C:\sanguo_vnpy_v2\data\static' +$env:PYTHONIOENCODING = 'utf-8' +$wrapper_log = 'C:\sanguo_vnpy_v2\data\baostock_day2b_wrapper.log' +$dl_log = 'C:\sanguo_vnpy_v2\data\baostock_day2b_download.log' + +function Log-W($m) { + "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $m" | Out-File -FilePath $wrapper_log -Append -Encoding UTF8 +} + +Log-W "=== DAY2b START (dividend 2020-2026) ===" +Log-W "DiskFree_GB_start=$((Get-PSDrive C).Free / 1GB)" + +& $py -X utf8 "$dir\baostock_static_download.py" --types dividend *>&1 | Tee-Object -FilePath $dl_log +$exit_code = $LASTEXITCODE +Log-W "DAY2b DONE exit=$exit_code" +Log-W "DiskFree_GB_end=$((Get-PSDrive C).Free / 1GB)" +Log-W "=== DAY2b WRAPPER EXIT ===" diff --git a/scripts/data_platform/baostock_delisted_download.py b/scripts/data_platform/baostock_delisted_download.py new file mode 100644 index 0000000..394c617 --- /dev/null +++ b/scripts/data_platform/baostock_delisted_download.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""P0 Task3: 退市股 K 线采集 (反幸存者偏差核心) + +只在 Mac 跑 (VPS baostock IP 已黑名单 10001011)。Mac staging → rsync VPS → VPS 灌库。 + +Phase 1 (1 query): bs.query_stock_basic() 无参拉全市场 → 内存过滤 + type=='1' & status=='0' & outDate>='2021-01-01' +Phase 2 (per-stock 1 query): 逐只 query_history_k_data_plus 18 字段 → parquet staging + +硬约束 (探针实证): + 1. 单进程单登录, 严禁并发 (baostock 并发会拉黑封 IP 6-24h) + 2. 直连不走代理 (脚本开头 unset proxy) + 3. 每只 sleep 0.3s, DAILY_LIMIT=9000 (安全日预算 <10000, 非 48000) + 4. 退市最后一日 (outDate) 行的 preclose/volume/amount/turn/tradestatus/pctChg 是空字符串非 0 + → pd.to_numeric(errors='coerce') 自动转 NaN → parquet None, 不能 float() + 5. 持久目录 data_cache/staging/delisted_202607 (防重启丢, /tmp 会清) + 6. 断点续传 (marker 为真相源), 复跑只补未完成的 + +退出码: 0=完成, 1=致命错误 (登录失败/列表拉取失败), 2=断路器触发 +""" + +import datetime +import io +import json +import logging +import os +import socket +import sys +import time +from pathlib import Path +from typing import List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +for _k in ("http_proxy", "https_proxy", "all_proxy", + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): + os.environ.pop(_k, None) + +socket.setdefaulttimeout(30) + +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + +# ======================== 配置 ======================== +REPO_ROOT = Path(__file__).resolve().parents[2] +OUT_DIR = REPO_ROOT / "data_cache" / "staging" / "delisted_202607" +LOG_DIR = OUT_DIR / "logs" + +START_DATE_FLOOR = "1990-12-19" # A 股最早交易日 (plan 规定, 对齐 daily_fullmarket_download) +END_DATE = "2026-07-18" # baostock 探针当天可用末日 +DELIST_SINCE = "2021-01-01" # 近 5 年阈值 + +BS_INTERVAL = 0.3 +BS_MAX_RETRIES = 3 +PROGRESS_LOG_EVERY = 20 +RELOGIN_EVERY = 100 +CIRCUIT_BREAKER = 10 + +BS_FIELDS = ("date,code,open,high,low,close,preclose,volume,amount," + "adjustflag,turn,tradestatus,pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST") +OUT_COLS = BS_FIELDS.split(",") + +QUERY_COUNT = 0 +DAILY_LIMIT = 9000 # 安全预算 <10000, 不是 48000 + +# ======================== 日志 ======================== +def setup_logging() -> logging.Logger: + LOG_DIR.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = LOG_DIR / f"delisted_{ts}.log" + + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + root = logging.getLogger() + root.setLevel(logging.INFO) + for h in list(root.handlers): + root.removeHandler(h) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + root.addHandler(sh) + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + root.addHandler(fh) + return logging.getLogger(__name__) + + +logger = setup_logging() + + +# ======================== baostock 登录 ======================== +def _login_once() -> bool: + try: + lg = bs.login() + if lg.error_code == "0": + return True + logger.error("bs.login() 失败: code=%s msg=%s", lg.error_code, lg.error_msg) + return False + except (socket.timeout, TimeoutError) as e: + logger.error("bs.login() 超时: %s (疑似冷却/黑名单)", e) + return False + except Exception as e: + logger.error("bs.login() 异常: %s", e) + return False + + +def _relogin() -> bool: + try: + bs.logout() + except Exception: + pass + if _login_once(): + return True + time.sleep(2) + try: + bs.logout() + except Exception: + pass + return _login_once() + + +# ======================== Phase 1: 扫全市场找近 5 年退市股 ======================== +def scan_delisted() -> Tuple[List[dict], List[dict]]: + """query_stock_basic() 无参拉全市场, 内存过滤近 5 年退市股。 + + 返回: + delisted_list: 退市股 [{code, code_name, ipoDate, outDate, status, type}, ...] + all_basic_stats: 全市场统计 (debug) + """ + global QUERY_COUNT + QUERY_COUNT += 1 + + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError( + f"query_stock_basic 失败: code={rs.error_code} msg={rs.error_msg}" + ) + + fields = list(rs.fields) + idx = {name: i for i, name in enumerate(fields)} + logger.info("query_stock_basic fields=%s", fields) + + all_rows: List[dict] = [] + while rs.next(): + r = rs.get_row_data() + all_rows.append({ + "code": r[idx["code"]], + "code_name": r[idx["code_name"]] if "code_name" in idx else "", + "ipoDate": r[idx["ipoDate"]] if "ipoDate" in idx else "", + "outDate": r[idx["outDate"]] if "outDate" in idx else "", + "type": r[idx["type"]] if "type" in idx else "", + "status": r[idx["status"]] if "status" in idx else "", + }) + + logger.info("query_stock_basic 返回总行数: %d", len(all_rows)) + + # 统计 + n_type1 = sum(1 for r in all_rows if r["type"] == "1") + n_type1_status0 = sum(1 for r in all_rows if r["type"] == "1" and r["status"] == "0") + n_type1_status0_recent = sum( + 1 for r in all_rows + if r["type"] == "1" + and r["status"] == "0" + and r["outDate"] + and r["outDate"] >= DELIST_SINCE + ) + all_basic_stats = { + "total_rows": len(all_rows), + "type_1_count": n_type1, + "type_1_status_0": n_type1_status0, + "type_1_status_0_recent5y": n_type1_status0_recent, + } + logger.info("全市场统计: %s", json.dumps(all_basic_stats, ensure_ascii=False)) + + # 过滤: type==1 (股票) & status==0 (退市) & outDate >= DELIST_SINCE + delisted = [ + r for r in all_rows + if r["type"] == "1" + and r["status"] == "0" + and r["outDate"] + and r["outDate"] >= DELIST_SINCE + ] + delisted.sort(key=lambda r: r["outDate"]) + logger.info("近 5 年退市股: %d 只 (阈值 %s)", len(delisted), DELIST_SINCE) + + return delisted, all_basic_stats + + +# ======================== Phase 2: 逐只 K 线下载 ======================== +def fetch_one_kline(bs_code: str, start_date: str, end_date: str) -> Optional[pd.DataFrame]: + """单只 query_history_k_data_plus。空数据返回空 df (合法, 不重试)。""" + global QUERY_COUNT + QUERY_COUNT += 1 + + rs = bs.query_history_k_data_plus( + bs_code, BS_FIELDS, + start_date=start_date, end_date=end_date, + frequency="d", adjustflag="3", + ) + if rs.error_code != "0": + raise RuntimeError( + f"baostock {bs_code} 查询错误: {rs.error_code} {rs.error_msg}" + ) + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + if not rows: + return pd.DataFrame(columns=OUT_COLS) + df = pd.DataFrame(rows, columns=OUT_COLS) + # 退市当日空字符串 → NaN (关键! 探针实证 sh.600432 末日 volume/amount/turn 为空) + numeric_cols = ["open", "high", "low", "close", "preclose", "volume", "amount", + "turn", "tradestatus", "pctChg", "peTTM", "psTTM", "pcfNcfTTM", "pbMRQ"] + for col in numeric_cols: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + # isST 整数 + if "isST" in df.columns: + df["isST"] = pd.to_numeric(df["isST"], errors="coerce").astype("Int64") + # adjustflag 保留字符串 + # 数据自检: dropna OHLC + df = df.dropna(subset=["open", "high", "low", "close"]) + bad_ohlc = (df["high"] < df[["open", "close"]].max(axis=1)) | \ + (df["low"] > df[["open", "close"]].min(axis=1)) + if bad_ohlc.any(): + df = df[~bad_ohlc] + if df.empty: + return pd.DataFrame(columns=OUT_COLS) + return df.sort_values("date").reset_index(drop=True) + + +def parse_bs_code(bs_code: str) -> Tuple[str, str]: + """sh.600074 → ('600074', 'SH')""" + prefix, num = bs_code.split(".", 1) + exc = "SH" if prefix == "sh" else "SZ" + return num, exc + + +def parquet_path(bs_code: str) -> Path: + num, exc = parse_bs_code(bs_code) + return OUT_DIR / f"{num}.{exc}_delisted.parquet" + + +def marker_path(parq: Path) -> Path: + return parq.parent / f".{parq.stem}.done" + + +def load_done_set() -> set: + done = set() + if not OUT_DIR.exists(): + return done + for marker in OUT_DIR.glob(".*_delisted.done"): + stem = marker.name[1:-len(".done")] # 600074.SH_delisted + try: + head, period = stem.rsplit("_", 1) + if period != "delisted": + continue + code, exc = head.split(".", 1) + if len(code) == 6 and code.isdigit() and exc in ("SH", "SZ"): + done.add(f"{code}.{exc}") + except ValueError: + continue + return done + + +def download_one(rec: dict) -> Tuple[str, int]: + """下载单只退市股 → staging parquet + marker。返回 (status, rows)。""" + global QUERY_COUNT + if QUERY_COUNT >= DAILY_LIMIT: + logger.warning("query 计数器 %d ≥ DAILY_LIMIT %d, graceful stop", + QUERY_COUNT, DAILY_LIMIT) + return "limit_reached", 0 + + bs_code = rec["code"] + parq = parquet_path(bs_code) + mk = marker_path(parq) + if mk.exists(): + return "skipped", 0 + + # start_date = max(ipoDate, '2000-01-01') + ipo = rec.get("ipoDate", "") or "" + start = max(ipo, START_DATE_FLOOR) if ipo else START_DATE_FLOOR + + df: Optional[pd.DataFrame] = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_one_kline(bs_code, start, END_DATE) + break + except (socket.timeout, TimeoutError, OSError) as e: + logger.warning("%s socket 异常重试 %d/%d: %s", bs_code, attempt + 1, BS_MAX_RETRIES, e) + if not _relogin(): + logger.error("重登失败, 放弃 %s", bs_code) + df = None + break + except Exception as e: + logger.warning("%s 下载异常重试 %d/%d: %s", bs_code, attempt + 1, BS_MAX_RETRIES, e) + if not _relogin(): + logger.error("重登失败, 放弃 %s", bs_code) + df = None + break + + if df is None: + return "failed", 0 + if df.empty: + # 合法空 (baostock 无该股该区间数据), 写空 marker 防复跑, 但不计入 ok + try: + mk.write_text(datetime.datetime.now().isoformat() + "\tempty") + except Exception: + pass + return "empty", 0 + + try: + OUT_DIR.mkdir(parents=True, exist_ok=True) + df.to_parquet(parq, index=False) + mk.write_text(datetime.datetime.now().isoformat()) + return "ok", len(df) + except Exception as e: + logger.error("写入 %s 失败: %s", parq, e) + return "failed", 0 + + +# ======================== 主流程 ======================== +def main() -> None: + t0 = time.time() + + logger.info("=" * 60) + logger.info("P0 Task3 退市股 K 线采集 (Mac baostock, 反幸存者偏差)") + logger.info(" 输出: %s", OUT_DIR) + logger.info(" end_date=%s delist_since=%s", END_DATE, DELIST_SINCE) + logger.info(" socket.setdefaulttimeout(30), BS_INTERVAL=%ss", BS_INTERVAL) + logger.info(" DAILY_LIMIT=%d (安全预算 <10000)", DAILY_LIMIT) + logger.info("=" * 60) + + if not _login_once(): + logger.error("[FATAL] baostock 登录失败 (疑似 Mac IP 黑名单), 退出") + sys.exit(1) + logger.info("baostock 登录成功 (Mac IP 未被封)") + + # Phase 1: 扫描 + try: + delisted, stats = scan_delisted() + except Exception as e: + logger.error("[FATAL] Phase 1 扫描失败: %s", e) + try: + bs.logout() + except Exception: + pass + sys.exit(1) + + list_file = OUT_DIR / "delisted_list.json" + OUT_DIR.mkdir(parents=True, exist_ok=True) + with open(list_file, "w", encoding="utf-8") as f: + json.dump({ + "generated_at": datetime.datetime.now().isoformat(), + "filter": {"type": "1", "status": "0", "outDate_gte": DELIST_SINCE}, + "stats": stats, + "count": len(delisted), + "items": delisted, + }, f, ensure_ascii=False, indent=2) + logger.info("退市清单写入 %s", list_file) + + if not delisted: + logger.info("无近 5 年退市股, 直接退出") + try: + bs.logout() + except Exception: + pass + sys.exit(0) + + # Phase 2: 逐只下载 + done_set = load_done_set() + todo = [r for r in delisted if parse_bs_code(r["code"])[0] + "." + parse_bs_code(r["code"])[1] not in done_set] + logger.info("Phase 2 待下载: %d (已完成 %d)", len(todo), len(done_set)) + + stats_dl = {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "limit_reached": 0, "rows": 0} + consec_fail = 0 + circuit = False + limit_reached = False + + for i, rec in enumerate(todo): + if i > 0 and i % RELOGIN_EVERY == 0: + logger.info("定期重登 @ %d/%d (query=%d)", i, len(todo), QUERY_COUNT) + if not _relogin(): + logger.warning("定期重登失败, 继续") + + try: + status, rows = download_one(rec) + except Exception as e: + status, rows = "failed", 0 + logger.debug("download_one %s 异常: %s", rec["code"], e) + + stats_dl[status] = stats_dl.get(status, 0) + 1 + if status == "ok": + stats_dl["rows"] += rows + consec_fail = 0 + elif status == "failed": + consec_fail += 1 + elif status == "limit_reached": + limit_reached = True + break + + if consec_fail >= CIRCUIT_BREAKER: + logger.error("[FATAL] 断路器触发: 连续 %d 只失败, 主动退出", consec_fail) + circuit = True + break + + if (i + 1) % PROGRESS_LOG_EVERY == 0: + elapsed = time.time() - t0 + logger.info( + "进度 %d/%d ok=%d empty=%d failed=%d skipped=%d rows=%d query=%d (%.0fs)", + i + 1, len(todo), stats_dl["ok"], stats_dl["empty"], + stats_dl["failed"], stats_dl["skipped"], stats_dl["rows"], QUERY_COUNT, elapsed, + ) + + if i < len(todo) - 1: + time.sleep(BS_INTERVAL) + + try: + bs.logout() + except Exception: + pass + + elapsed = time.time() - t0 + logger.info("=" * 60) + if circuit: + logger.info("[RESULT] 断路器触发中止, 耗时 %.1fs", elapsed) + elif limit_reached: + logger.info("[RESULT] query 上限 graceful stop, 耗时 %.1fs", elapsed) + else: + logger.info("[RESULT] 完成, 耗时 %.1fs", elapsed) + logger.info("下载统计: %s", json.dumps(stats_dl, ensure_ascii=False)) + logger.info("query 总计: %d / DAILY_LIMIT %d", QUERY_COUNT, DAILY_LIMIT) + logger.info("=" * 60) + + sys.exit(2 if circuit else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/baostock_static_download.py b/scripts/data_platform/baostock_static_download.py new file mode 100644 index 0000000..2624599 --- /dev/null +++ b/scripts/data_platform/baostock_static_download.py @@ -0,0 +1,795 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""BaoStock 静态数据下载 (basic / adjust_factor / dividend 3 类, 只产 parquet) + +目标: + 在 Windows Server 2022 VPS (49.232.102.198) 上用 baostock 全量拉取 A 股 + (含退市) 3 类静态数据: + 1. basic - 基础信息 (全市场单文件, 不按只) + 2. adjust_factor - 复权因子 (per-stock) + 3. dividend - 分红送转 (per-stock, 循环年份 2020..2026) + + 输出 import-ready parquet, 后续数据平台直接读. 与 15min 下载脚本共用硬化机制. + +与 baostock_15min_fullmarket_download.py 共用的硬约束 (踩过的坑): + 1. 单进程单登录, 严禁并发 (baostock 并发会拉黑封 IP 6-24h) + 2. 直连不走代理 (脚本开头 unset proxy) + 3. 每次 baostock query 后 sleep BS_INTERVAL=0.4s 限速 + 4. baostock login 受 socket.setdefaulttimeout(30) 保护 + (Windows 无 SIGALRM, 这是跨平台等价方案) + 5. empty vs failed 区分: + - 空数据 (退市/无除权事件) → fetch 返空 df → status='empty' 中性, 不重试 + - 真错误 → fetch raise → retry → status='failed' 计断路器 + +与 15min 脚本差异: + - 3 类 fetch 各自实现 (basic 单调用 / adjust_factor per-stock / dividend per-year) + - per-stock empty **也写 marker** (静态空 = 确定性"无除权事件", + 重跑只是浪费 baostock 配额, 与 K 线 empty 不写 marker 的语义不同) + - 输出 3 个子目录: basic / adjust_factor / dividend + - 单次进程可顺序跑多类 (--types basic,adjust_factor,dividend) + - basic 单文件全量刷新, 不受 --limit / --codes 影响 + +⚠️ 构建期禁止任何 live baostock 调用 (VPS 15min 下载并行会封 IP). + 本脚本只 py_compile + 代码 review, smoke 测试延后. + +用法: + # 全跑 (basic + adjust_factor + dividend) + python baostock_static_download.py + # 只跑指定类型 (逗号分隔) + python baostock_static_download.py --types basic,adjust_factor + python baostock_static_download.py --types dividend + # 测试 (前 10 只, basic 不受影响) + python baostock_static_download.py --types adjust_factor --limit 10 + # 指定股票 (6 位 code, 不带前缀) + python baostock_static_download.py --types dividend --codes 600519,000001 + # 强制重下, 忽略 marker + python baostock_static_download.py --types basic --force + +输出目录结构: + {OUT_DIR}/ + ├── basic/ + │ ├── stock_basic.parquet + │ └── .stock_basic.baostock (marker) + ├── adjust_factor/ + │ ├── 600519.SH_factor.parquet + │ ├── .600519.SH_factor.baostock (marker) + │ └── ... + ├── dividend/ + │ ├── 600519.SH_dividend.parquet + │ ├── .600519.SH_dividend.baostock (marker) + │ └── ... + └── logs/baostock_static_YYYYMMDD_HHMMSS.log + +退出码: 0=完成, 1=致命错误 (登录/列表拉取失败), 2=断路器触发 (可重试) +""" + +import argparse +import datetime +import json +import logging +import os +import socket +import sys +import time +from functools import partial +from pathlib import Path +from typing import Callable, List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +# 必须在 import baostock 之前清理 (baostock 底层 urllib 会读 proxy 环境变量) +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(_k, None) + +# SIGALRM 跨平台替代: 全局 socket 超时 30s (Windows 无 SIGALRM)。 +# baostock 的 socket 连接会继承这个超时, connect 卡死时抛 socket.timeout。 +socket.setdefaulttimeout(30) + +# stdout 行缓冲 (SSH-detached / pipe 重定向时也能看到实时进度) +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + + +# ======================== 配置 (环境变量可覆盖) ======================== + +DEFAULT_OUT_DIR = r"C:\sanguo_vnpy_v2\data\static" +OUT_DIR = Path(os.environ.get("BS_STATIC_OUT_DIR", DEFAULT_OUT_DIR)) +DEFAULT_LOG_DIR = OUT_DIR / "logs" +_log_dir_env = os.environ.get("BS_STATIC_LOG_DIR") +LOG_DIR = Path(_log_dir_env) if _log_dir_env else DEFAULT_LOG_DIR + +START_DATE_DEFAULT = "2020-01-01" +# 分红查询年份范围 (含两端). 2020..2026 = 7 年, 与 START_DATE_DEFAULT 对齐。 +DIVIDEND_YEAR_RANGE = (2020, 2026) + +BS_INTERVAL = 0.4 # 每次 baostock query 后间隔秒 (防封 IP) +BS_MAX_RETRIES = 3 # 单次下载重试次数 (含强制重登) +PROGRESS_LOG_EVERY = 500 # 每 N 只打一次进度日志 +RELOGIN_EVERY = 400 # 每 N 只定期重登保持连接 +CIRCUIT_BREAKER = 30 # 连续失败 N 只 → 断路退出 + +# 子目录名 (同时也是 --types 合法值) +DIR_BASIC = "basic" +DIR_ADJUST = "adjust_factor" +DIR_DIVIDEND = "dividend" + +# 文件名后缀 (parquet / marker stem 解析用) +SUFFIX_BASIC = "stock_basic" # basic 单文件 stem (无 per-stock) +SUFFIX_ADJUST = "_factor" # 600519.SH_factor +SUFFIX_DIVIDEND = "_dividend" # 600519.SH_dividend + +VALID_TYPES = (DIR_BASIC, DIR_ADJUST, DIR_DIVIDEND) + + +# ======================== 日志 ======================== + +def setup_logging() -> Tuple[logging.Logger, Path]: + """配置 root logger: 同时写 stdout + 文件。返回 (logger, log_file)。""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = LOG_DIR / f"baostock_static_{ts}.log" + + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + root = logging.getLogger() + root.setLevel(logging.INFO) + for h in list(root.handlers): + root.removeHandler(h) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + root.addHandler(sh) + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + root.addHandler(fh) + + return logging.getLogger(__name__), log_file + + +logger, LOG_FILE = setup_logging() + + +# ======================== 工具函数 ======================== + +def normalize_date(s: str) -> str: + """接受 YYYYMMDD 或 YYYY-MM-DD, 返回 YYYY-MM-DD; 空串 → 今天。""" + s = (s or "").strip() + if not s: + return datetime.date.today().strftime("%Y-%m-%d") + if "-" in s: + return s + if len(s) == 8 and s.isdigit(): + return f"{s[:4]}-{s[4:6]}-{s[6:8]}" + raise ValueError(f"无效日期格式: {s} (期望 YYYYMMDD 或 YYYY-MM-DD)") + + +def baostock_code_to_parts(bs_code: str) -> Tuple[str, str]: + """sh.600519 → ('600519', 'SH'); sz.000001 → ('000001', 'SZ')。 + 前缀直接从 baostock code 拿, 不猜。 + """ + if "." not in bs_code: + raise ValueError(f"无效 baostock code: {bs_code}") + prefix, num = bs_code.split(".", 1) + if prefix == "sh": + return num, "SH" + if prefix == "sz": + return num, "SZ" + raise ValueError(f"未知 baostock 前缀: {bs_code}") + + +def parts_to_baostock(code: str, exchange: str) -> str: + """('600519', 'SH') → 'sh.600519'。""" + pfx = "sh" if exchange == "SH" else "sz" + return f"{pfx}.{code}" + + +def guess_exchange_by_code(code: str) -> str: + """6/68/51 开头 → SH, 其他 → SZ (无前缀代码用, 与 15min 模板一致)。""" + if code.startswith(("6", "68", "51")): + return "SH" + return "SZ" + + +def _rs_to_rows(rs) -> Tuple[List[str], List[List[str]]]: + """读取 ResultData, 返回 (fields, rows)。 + 用 rs.fields 动态拿字段名, 不硬编码位置 (踩过坑: baostock 字段顺序与文档不一致)。 + """ + fields = list(rs.fields) + rows: List[List[str]] = [] + while rs.next(): + rows.append(rs.get_row_data()) + return fields, rows + + +# ======================== 股票列表 (baostock query_stock_basic) ======================== + +def fetch_all_stocks() -> List[Tuple[str, str]]: + """baostock query_stock_basic 拉全市场 A 股列表 (含退市, type=='1')。 + + 返回: [(code, exchange), ...], 例 [('600519', 'SH'), ('000001', 'SZ')] + 过滤: type=='1' (股票); **不过滤 status** (保留退市, 避免生存偏差)。 + + baostock 0.9.3 实测字段顺序与文档不一致, 用 rs.fields 动态建索引按名取, + 避免字段位置差异导致过滤错位 (照抄 15min 模板已验证逻辑)。 + """ + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError( + f"query_stock_basic 失败: code={rs.error_code} msg={rs.error_msg}" + ) + + fields = list(rs.fields) + idx = {name: i for i, name in enumerate(fields)} + logger.info("query_stock_basic fields=%s", fields) + + out: List[Tuple[str, str]] = [] + n_stock = 0 + n_skip_type = 0 + n_skip_code = 0 + while rs.next(): + r = rs.get_row_data() + type_ = r[idx["type"]] if "type" in idx and len(r) > idx["type"] else "" + if type_ != "1": # 只要股票 (排除指数/债券/其他) + n_skip_type += 1 + continue + bs_code = r[idx["code"]] if "code" in idx and len(r) > idx["code"] else "" + try: + code, exchange = baostock_code_to_parts(bs_code) + except ValueError: + n_skip_code += 1 + continue + if len(code) != 6 or not code.isdigit(): + n_skip_code += 1 + continue + out.append((code, exchange)) + n_stock += 1 + logger.info( + "股票列表: stocks(type=1)=%d skip_type=%d skip_code=%d (含退市, 不过滤 status)", + n_stock, n_skip_type, n_skip_code) + return out + + +# ======================== baostock 登录 ======================== + +def _login_once() -> bool: + """bs.login() — 已被 socket.setdefaulttimeout(30) 保护。 + connect 卡死会抛 socket.timeout, 这里捕获。 + """ + try: + lg = bs.login() + if lg.error_code == "0": + return True + logger.error("bs.login() 失败: code=%s msg=%s", lg.error_code, lg.error_msg) + return False + except (socket.timeout, TimeoutError) as e: + logger.error("bs.login() 超时: %s (baostock 疑似冷却)", e) + return False + except Exception as e: + logger.error("bs.login() 异常: %s", e) + return False + + +def _relogin() -> bool: + """强制重登: logout + login。失败等 2s 再试 1 次。""" + try: + bs.logout() + except Exception: + pass + if _login_once(): + return True + time.sleep(2) + try: + bs.logout() + except Exception: + pass + return _login_once() + + +# ======================== 数据 fetch (3 类各自实现) ======================== + +def fetch_basic() -> pd.DataFrame: + """拉全市场基础信息, 单次调用返 df。 + + 返回列: baostock 原始字段 (code, code_name, ipoDate, outDate, type, status) + + 派生列 code (6位纯数字) / exchange (SH/SZ) / baostock_code (sh.XXXXXX) + 原始 code 列被覆盖为 6 位纯数字, baostock_code 保留 sh.XXXXXX 形式。 + + baostock 字段顺序与文档不一致时, 用 rs.fields 动态拿列名, 不硬编码位置。 + """ + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError( + f"query_stock_basic 失败: code={rs.error_code} msg={rs.error_msg}" + ) + + fields, rows = _rs_to_rows(rs) + extra_cols = ["code_6digit", "exchange", "baostock_code"] + if not rows: + return pd.DataFrame(columns=fields + extra_cols) + + df = pd.DataFrame(rows, columns=fields) + # 派生列: 从原始 baostock code 拆出 6 位 + 交易所 + df["baostock_code"] = df["code"] + parts = df["code"].apply(baostock_code_to_parts) + df["code_6digit"] = parts.str[0] + df["exchange"] = parts.str[1] + # 列重排: 把派生 6 位 code 放前面 (与原始 baostock code 区分, 避免歧义) + return df.reset_index(drop=True) + + +def fetch_adjust_factor(bs_code: str, start_date: str, end_date: str) -> pd.DataFrame: + """拉单只复权因子, 返回 df (可能空)。 + + 返回字段 (baostock 文档): code, dividOperateDate, foreAdjustFactor, + backAdjustFactor, adjustFactor + 数值列 to_numeric; 空数据 (无除权事件的票) → empty df, 上层计 empty 中性。 + + 字段顺序按 rs.fields 动态拿, 不硬编码位置 (容错 baostock 字段顺序差异)。 + """ + rs = bs.query_adjust_factor( + code=bs_code, start_date=start_date, end_date=end_date, + ) + if rs.error_code != "0": + raise RuntimeError( + f"query_adjust_factor {bs_code} 错误: {rs.error_code} {rs.error_msg}" + ) + + fields, rows = _rs_to_rows(rs) + if not rows: + return pd.DataFrame(columns=fields) + + df = pd.DataFrame(rows, columns=fields) + # 数值列 to_numeric (baostock 返字符串) + for col in ("foreAdjustFactor", "backAdjustFactor", "adjustFactor"): + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + return df + + +def fetch_dividend(bs_code: str, year_range: Tuple[int, int]) -> pd.DataFrame: + """拉单只分红送转, 循环年份 concat。返回 df (可能空)。 + + 每年调用 bs.query_dividend_data(code, year, yearType="report"), + 返回该票当年分红记录 (多行)。concat 所有年份。 + **保留 baostock 返回的全部列** (用 rs.fields 动态拿, 不硬编码字段名)。 + + 限速: 每次年份 query 后 sleep BS_INTERVAL (除最后一次, 由主循环负责)。 + yearType="report" = 预案公告年份 (与用户指定一致)。 + """ + years = list(range(year_range[0], year_range[1] + 1)) + frames: List[pd.DataFrame] = [] + fields_ref: Optional[List[str]] = None + + for i, year in enumerate(years): + rs = bs.query_dividend_data( + code=bs_code, year=year, yearType="report", + ) + if rs.error_code != "0": + raise RuntimeError( + f"query_dividend_data {bs_code} year={year} 错误: " + f"{rs.error_code} {rs.error_msg}" + ) + fields, rows = _rs_to_rows(rs) + if fields_ref is None: + fields_ref = fields # 锁定首次响应的字段列表 + if rows: + # 各年字段应一致; 以当前响应字段为准构造 df + frames.append(pd.DataFrame(rows, columns=fields)) + # 限速: 非最后一年时 sleep (最后一次由主循环负责 stock 间隔) + if i < len(years) - 1: + time.sleep(BS_INTERVAL) + + if not frames: + cols = fields_ref if fields_ref is not None else [] + return pd.DataFrame(columns=cols) + return pd.concat(frames, ignore_index=True) + + +# ======================== 路径 / marker ======================== + +def subdir_for(data_type: str) -> Path: + """data_type → OUT_DIR / 。""" + return OUT_DIR / data_type + + +def parquet_path_basic() -> Path: + """basic 单文件 parquet 路径。""" + return subdir_for(DIR_BASIC) / f"{SUFFIX_BASIC}.parquet" + + +def marker_path_basic() -> Path: + """basic 单文件 marker。""" + return subdir_for(DIR_BASIC) / f".{SUFFIX_BASIC}.baostock" + + +def parquet_path_per_stock( + code: str, exchange: str, data_type: str, suffix: str, +) -> Path: + """('600519', 'SH', 'adjust_factor', '_factor') + → OUT_DIR/adjust_factor/600519.SH_factor.parquet + """ + return subdir_for(data_type) / f"{code}.{exchange}{suffix}.parquet" + + +def marker_path_for(parquet_path: Path) -> Path: + """parquet → 同目录 .{stem}.baostock marker。""" + return parquet_path.parent / f".{parquet_path.stem}.baostock" + + +def load_done_set_per_stock(data_type: str, suffix: str) -> set: + """扫子目录 marker 构造已完成 (code, exchange) 集合 (真相源)。 + + marker 文件名格式: .600519.SH_factor.baostock + 解析 stem = '600519.SH_factor', 按 suffix='_factor' 剥出 head='600519.SH', + 再拆 code='600519' / exc='SH', 校验 6 位数字 + 交易所。 + """ + done: set = set() + d = subdir_for(data_type) + if not d.exists(): + return done + ext = ".baostock" + for marker in d.glob(f".*{ext}"): + name = marker.name + if not name.startswith(".") or not name.endswith(ext): + continue + stem = name[1:-len(ext)] # e.g. '600519.SH_factor' + if not suffix or not stem.endswith(suffix): + continue + head = stem[:-len(suffix)] if suffix else stem + # head: '600519.SH' + try: + code, exc = head.split(".", 1) + if len(code) == 6 and code.isdigit() and exc in ("SH", "SZ"): + done.add((code, exc)) + except ValueError: + continue + return done + + +# ======================== 单只下载 ======================== + +def download_basic(force: bool) -> Tuple[str, int]: + """basic 单文件全量刷新。返 (status, rows), status ∈ {'ok','skipped','failed'}。 + + basic 全市场不应为空 (5537 票级别), 空数据 = 失败 (与 per-stock 语义不同)。 + """ + parquet_path = parquet_path_basic() + marker_path = marker_path_basic() + + if not force and marker_path.exists(): + return "skipped", 0 + + df: Optional[pd.DataFrame] = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_basic() + break # 成功 (df 空仍 break, 下方判 failed) + except (socket.timeout, TimeoutError, OSError) as e: + logger.warning("basic socket 异常重试 %d/%d: %s — 强制重登", + attempt + 1, BS_MAX_RETRIES, e) + if not _relogin(): + logger.error("重登失败, 放弃 basic") + df = None + break + except Exception as e: + logger.warning("basic 异常重试 %d/%d: %s — 强制重登", + attempt + 1, BS_MAX_RETRIES, e) + if not _relogin(): + logger.error("重登失败, 放弃 basic") + df = None + break + + if df is None: + return "failed", 0 + if df.empty: + # 全市场不应为空 — 视为失败 (重试或人工排查) + logger.error("basic 返回空 (异常, 全市场 type=1 不应为空)") + return "failed", 0 + + try: + d = subdir_for(DIR_BASIC) + d.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path.write_text(datetime.datetime.now().isoformat()) + return "ok", len(df) + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return "failed", 0 + + +def download_one_per_stock( + code: str, + exchange: str, + data_type: str, + suffix: str, + fetch_fn: Callable[[str], pd.DataFrame], + force: bool, +) -> Tuple[str, int]: + """通用 per-stock 下载 → 写 parquet + marker。 + + fetch_fn(bs_code) → df (可能空) 或 raise。返 (status, rows), status ∈ + {'ok', 'skipped', 'empty', 'failed'}。 + + 与 15min 模板差异: per-stock empty **也写 marker** (静态空 = "查过了确实无除权事件", + 重跑只是浪费 baostock 配额; 与 K 线 empty "可能是延迟" 语义不同)。 + """ + parquet_path = parquet_path_per_stock(code, exchange, data_type, suffix) + marker_path = marker_path_for(parquet_path) + + if not force and marker_path.exists(): + return "skipped", 0 + + bs_code = parts_to_baostock(code, exchange) + + # retry 循环: fetch_fn 仅在真错误时抛 (走 retry/relogin); + # 空数据返空 df (合法, 不重试), 成功 break。 + df: Optional[pd.DataFrame] = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_fn(bs_code) + break # fetch 成功 (df 可能空 = 合法无数据) + except (socket.timeout, TimeoutError, OSError) as e: + logger.warning( + "%s %s socket 异常重试 %d/%d: %s — 强制重登", + code, data_type, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s %s", code, data_type) + df = None + break + except Exception as e: + logger.warning( + "%s %s 下载异常重试 %d/%d: %s — 强制重登", + code, data_type, attempt + 1, BS_MAX_RETRIES, e, + ) + if not _relogin(): + logger.error("重登失败, 放弃 %s %s", code, data_type) + df = None + break + + if df is None: + return "failed", 0 # 多次重试仍报错 (真失败, 计断路器) + + # df 可能空 (合法) 或非空 — 两种都写 marker (静态数据语义, 见 docstring) + try: + d = subdir_for(data_type) + d.mkdir(parents=True, exist_ok=True) + df.to_parquet(parquet_path, index=False) + marker_path.write_text(datetime.datetime.now().isoformat()) + return ("ok" if not df.empty else "empty"), len(df) + except Exception as e: + logger.error("写入 %s 失败: %s", parquet_path, e) + return "failed", 0 + + +# ======================== per-stock 主循环 (通用) ======================== + +def run_per_stock_type( + data_type: str, + suffix: str, + fetch_fn: Callable[[str], pd.DataFrame], + all_codes: List[Tuple[str, str]], + args: argparse.Namespace, + skip_loop_sleep: bool, +) -> Tuple[dict, bool]: + """运行 per-stock 类型 (adjust_factor / dividend)。 + + fetch_fn: 已绑定参数的 callable, 接受 bs_code, 返回 df (可能空) 或 raise。 + skip_loop_sleep: True 时主循环不再 sleep (fetch_fn 内部已自限速, 如 dividend)。 + 返回 (stats, circuit_triggered)。 + """ + # --codes 过滤 (无前缀, 按代码开头猜交易所, 与 15min 模板一致) + if args.codes: + todo_codes = [ + (c.strip(), guess_exchange_by_code(c.strip())) + for c in args.codes.split(",") if c.strip() + ] + else: + todo_codes = list(all_codes) + + # --limit 截断 + if args.limit > 0: + todo_codes = todo_codes[:args.limit] + logger.info("[%s] limit=%d 截断", data_type, args.limit) + + # marker 断点续传 + done_set = load_done_set_per_stock(data_type, suffix) + if args.force: + todo = todo_codes + else: + todo = [(c, e) for c, e in todo_codes if (c, e) not in done_set] + logger.info("[%s] 待处理: %d (已完成 %d)", data_type, len(todo), len(done_set)) + + stats = {"ok": 0, "skipped": 0, "empty": 0, "failed": 0, "rows": 0} + consec_fail = 0 + circuit_triggered = False + t_start = time.time() + + for i, (code, exc) in enumerate(todo): + # 定期重登保持连接 + if i > 0 and i % RELOGIN_EVERY == 0: + logger.info("[%s] 定期重登 @ %d/%d", data_type, i, len(todo)) + if not _relogin(): + logger.warning("[%s] 定期重登失败, 继续 (单次失败不致命)", data_type) + + try: + status, rows = download_one_per_stock( + code, exc, data_type, suffix, fetch_fn, args.force, + ) + except Exception as e: + status, rows = "failed", 0 + logger.debug("[%s] %s.%s 异常: %s", data_type, code, exc, e) + + stats[status] = stats.get(status, 0) + 1 + if status == "ok": + stats["rows"] += rows + consec_fail = 0 + elif status == "failed": + consec_fail += 1 + # skipped / empty 中性: 不重置也不递增 + + # 断路器: 连续 N 只全 failed → baostock 疑似不可达, 保存进度主动退出 + if consec_fail >= CIRCUIT_BREAKER: + logger.error( + "[%s] [FATAL] 断路器触发: 连续 %d 只失败, baostock 疑似不可达, " + "退出 (done_set 不含 failed 票, 复跑会重试)", + data_type, consec_fail, + ) + circuit_triggered = True + break + + # 进度日志 + if (i + 1) % PROGRESS_LOG_EVERY == 0: + elapsed = time.time() - t_start + logger.info( + "[%s] 进度 %d/%d ok=%d skipped=%d empty=%d failed=%d rows=%d (%.0f秒)", + data_type, i + 1, len(todo), stats["ok"], stats["skipped"], + stats["empty"], stats["failed"], stats["rows"], elapsed, + ) + + # 限速 (dividend 内部已 sleep, 跳过) + if not skip_loop_sleep and i < len(todo) - 1: + time.sleep(BS_INTERVAL) + + elapsed = time.time() - t_start + logger.info( + "[%s] 完成, 耗时 %.1f 秒, 统计: %s", + data_type, elapsed, json.dumps(stats, ensure_ascii=False), + ) + return stats, circuit_triggered + + +# ======================== CLI / main ======================== + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="BaoStock 静态数据下载 (basic / adjust_factor / dividend)", + ) + p.add_argument( + "--types", default=",".join(VALID_TYPES), + help=f"逗号分隔类型, 默认全部 ({','.join(VALID_TYPES)})", + ) + p.add_argument( + "--start", default=START_DATE_DEFAULT, + help=f"起始日期 YYYYMMDD 或 YYYY-MM-DD, 默认 {START_DATE_DEFAULT}", + ) + p.add_argument("--end", default="", help="结束日期, 默认今天") + p.add_argument( + "--codes", + help="指定代码逗号分隔 (6 位无前缀), 如 600519,000001 (basic 不受影响)", + ) + p.add_argument( + "--limit", type=int, default=0, + help="限制处理股票数, 测试用 (basic 不受影响)", + ) + p.add_argument("--force", action="store_true", help="强制重下, 忽略 marker") + return p.parse_args() + + +def resolve_types(s: str) -> List[str]: + """解析 --types 字符串 → 去重保序的类型列表。无效类型 SystemExit。""" + parts = [t.strip() for t in s.split(",") if t.strip()] + bad = [t for t in parts if t not in VALID_TYPES] + if bad: + raise SystemExit(f"未知 --types: {bad}, 可选 {list(VALID_TYPES)}") + if not parts: + return list(VALID_TYPES) + seen: set = set() + out: List[str] = [] + for t in parts: + if t not in seen: + out.append(t) + seen.add(t) + return out + + +def main() -> None: + args = parse_args() + start_date = normalize_date(args.start) + end_date = normalize_date(args.end) + types = resolve_types(args.types) + + logger.info("=" * 60) + logger.info("BaoStock 静态数据下载 (basic / adjust_factor / dividend)") + logger.info(" 输出目录: %s", OUT_DIR) + logger.info(" 日志文件: %s", LOG_FILE) + logger.info(" 日期范围: %s ~ %s", start_date, end_date) + logger.info(" 分红年份: %d..%d", *DIVIDEND_YEAR_RANGE) + logger.info(" 类型: %s", types) + logger.info(" socket.setdefaulttimeout(30) (SIGALRM 替代)") + logger.info(" 当前时间: %s", datetime.datetime.now().isoformat()) + logger.info("=" * 60) + + # 登录 + if not _login_once(): + logger.error("[FATAL] baostock 登录失败 (疑似冷却), 退出") + sys.exit(1) + logger.info("baostock 登录成功") + + all_codes: Optional[List[Tuple[str, str]]] = None + any_circuit = False + + try: + for t in types: + logger.info("-" * 50) + logger.info(">>> 类型: %s", t) + + if t == DIR_BASIC: + status, rows = download_basic(args.force) + logger.info("[basic] status=%s rows=%d", status, rows) + if status == "failed": + any_circuit = True + logger.error("[basic] 失败, 跳过后续类型") + break + continue + + # per-stock 类型: 需股票列表 (同进程内缓存, 不重复拉) + if all_codes is None: + try: + all_codes = fetch_all_stocks() + except Exception as e: + logger.error("[FATAL] 获取股票列表失败: %s", e) + any_circuit = True + break + logger.info("全市场 A 股 (含退市): %d 只", len(all_codes)) + + if t == DIR_ADJUST: + fetch_fn = partial(fetch_adjust_factor, + start_date=start_date, end_date=end_date) + _, circuit = run_per_stock_type( + DIR_ADJUST, SUFFIX_ADJUST, fetch_fn, + all_codes, args, skip_loop_sleep=False, + ) + elif t == DIR_DIVIDEND: + # fetch_dividend 内部已对每年 query sleep, 主循环跳过 sleep + fetch_fn = partial(fetch_dividend, + year_range=DIVIDEND_YEAR_RANGE) + _, circuit = run_per_stock_type( + DIR_DIVIDEND, SUFFIX_DIVIDEND, fetch_fn, + all_codes, args, skip_loop_sleep=True, + ) + else: + logger.error("未知类型 (跳过): %s", t) + continue + + if circuit: + any_circuit = True + logger.error("[%s] 断路器触发, 跳过后续类型", t) + break + finally: + try: + bs.logout() + except Exception: + pass + + if any_circuit: + sys.exit(2) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/build_daily_from_xtdata.py b/scripts/data_platform/build_daily_from_xtdata.py index d2cbc8c..5e2065c 100644 --- a/scripts/data_platform/build_daily_from_xtdata.py +++ b/scripts/data_platform/build_daily_from_xtdata.py @@ -37,7 +37,11 @@ def prefix_of(sym): log("start") -u = xd.get_stock_list_in_sector("沪深A股") or [] +u = list(set( + (xd.get_stock_list_in_sector("沪深A股") or []) + + (xd.get_stock_list_in_sector("沪深ETF") or []) + + (xd.get_stock_list_in_sector("沪深基金") or []) +)) log(f"universe={len(u)} sample={u[:3]}") # 1. 下载一次（raw 日线），分批 paced（别猛打券商后端） diff --git a/scripts/data_platform/daily_update_static.py b/scripts/data_platform/daily_update_static.py new file mode 100644 index 0000000..a72e78d --- /dev/null +++ b/scripts/data_platform/daily_update_static.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""baostock 日增量 → daily_baostock_full (VPS 直跑, schtask 17:00) + +设计 (用户决策 A: VPS 直跑增量, 守 48000/天): + - 跑前 login 探针: 黑名单(10001011)则 graceful skip (exit 2), 不硬跑 + - LOOKBACK_DAYS=7 覆盖周末/节假日 (baostock 日终更新, 17:00 跑当日 bar 已就绪) + - query_stock_basic() 无参 1 拉全 A (含退市, 与 Day1 同口径) → 每股 1 query + - 每股 query_history_k_data_plus(LOOKBACK 窗口) → staging parquet (审计) → executemany INSERT OR REPLACE daily_baostock_full + - 幂等: PK(symbol,exchange,date) + INSERT OR REPLACE, 重复跑安全 + - QUERY_COUNT 计数器 + DAILY_LIMIT 防线 (实际 ~5538/天 = 11.5%, 远低于 48000) + - 单进程单登录, sleep 0.3s 限速, socket timeout 30s + +预算 (实测): 1(list) + 5537(每股) = ~5538 query/天, 占 48000 的 11.5%, 安全。 +耗时: 5537 × (sleep 0.3 + query ~0.3) ≈ 55min, 17:00 跑 18:00 前完。 + +复用模式: + - baostock_daily_fullmarket_download.py: login/relogin/socket timeout/QUERY_COUNT/fetch_all_stocks + - import_delisted_to_db.py / import_baostock_to_db.py: parse_baostock_code/executemany/WAL + +用法 (VPS): + C:\\Python310\\python.exe -X utf8 daily_update_static.py # 正式跑 + python daily_update_static.py --limit 10 # 小样本测 + python daily_update_static.py --dry-run # 只下不灌 +环境变量: LOOKBACK_DAYS (默认7), BS_INCREMENT_OUT_DIR (staging, 默认 data/daily_baostock_increment/{YYYYMMDD}) +退出码: 0=完成; 1=致命错误; 2=黑名单 graceful skip; 3=query 超限 graceful stop +""" +import argparse +import datetime +import json +import logging +import os +import socket +import sys +import time +from pathlib import Path +from typing import List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +# 必须在 import baostock 之前清理 (baostock 底层 urllib 读 proxy env) +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"): + os.environ.pop(_k, None) +socket.setdefaulttimeout(30) +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass + +import baostock as bs # noqa: E402 +import pandas as pd # noqa: E402 + +# ======================== 配置 (VPS 路径) ======================== +BASE = Path(r"C:\sanguo_vnpy_v2") +DB_PATH = BASE / "data" / "quant_trading.db" + +LOOKBACK_DAYS = int(os.environ.get("LOOKBACK_DAYS", "7")) +_date_tag = datetime.date.today().strftime("%Y%m%d") +STAGING = Path(os.environ.get( + "BS_INCREMENT_OUT_DIR", + str(BASE / "data" / "daily_baostock_increment" / _date_tag), +)) + +# baostock 18 字段 (与 daily_baostock_full 同口径, adjustflag=3 raw) +BS_FIELDS = ("date,code,open,high,low,close,preclose,volume,amount,adjustflag," + "turn,tradestatus,pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST") +OUT_COLS = BS_FIELDS.split(",") +NUMERIC_COLS = ["open", "high", "low", "close", "preclose", "volume", "amount", + "turn", "pctChg", "peTTM", "psTTM", "pcfNcfTTM", "pbMRQ"] + +BS_INTERVAL = 0.3 +BS_MAX_RETRIES = 3 +DAILY_LIMIT = 40000 # 防线 (实际 ~5538); 接近则 graceful stop +QUERY_COUNT = 0 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) +logger = logging.getLogger(__name__) + + +# ======================== 登录 ======================== +def login_once() -> bool: + """login 探针。黑名单/超时返回 False (调用方 graceful skip)。""" + try: + lg = bs.login() + if lg.error_code == "0": + return True + logger.error("bs.login() 失败: code=%s msg=%s", lg.error_code, lg.error_msg) + return False + except (socket.timeout, TimeoutError) as e: + logger.error("bs.login() 超时: %s", e) + return False + except Exception as e: + logger.error("bs.login() 异常: %s", e) + return False + + +def relogin() -> bool: + try: + bs.logout() + except Exception: + pass + if login_once(): + return True + time.sleep(2) + try: + bs.logout() + except Exception: + pass + return login_once() + + +# ======================== 股票列表 (query_stock_basic 无参, 含退市) ======================== +def fetch_all_stocks() -> List[Tuple[str, str]]: + """query_stock_basic() 无参 → 全 A (type=1, 含退市 status=0), 与 Day1 同口径。 + 返回 [(code, exchange), ...], 例 [('600519', 'SH'), ('000001', 'SZ')]。""" + global QUERY_COUNT + QUERY_COUNT += 1 + rs = bs.query_stock_basic() + if rs.error_code != "0": + raise RuntimeError(f"query_stock_basic 失败: {rs.error_code} {rs.error_msg}") + fields = list(rs.fields) + idx = {n: i for i, n in enumerate(fields)} + out = [] + while rs.next(): + r = rs.get_row_data() + type_ = r[idx["type"]] if "type" in idx else "" + if type_ != "1": # 只要股票 + continue + bs_code = r[idx["code"]] if "code" in idx else "" + if "." not in bs_code: + continue + prefix, num = bs_code.split(".", 1) + if prefix not in ("sh", "sz") or len(num) != 6 or not num.isdigit(): + continue + out.append((num, "SH" if prefix == "sh" else "SZ")) + return out + + +# ======================== 单股增量下载 ======================== +def fetch_one_increment(bs_code: str, start: str, end: str) -> Optional[pd.DataFrame]: + """query_history_k_data_plus LOOKBACK 窗口。失败抛异常 (上层 retry); 空返回空 df。""" + global QUERY_COUNT + QUERY_COUNT += 1 + rs = bs.query_history_k_data_plus( + bs_code, BS_FIELDS, start_date=start, end_date=end, + frequency="d", adjustflag="3", + ) + if rs.error_code != "0": + raise RuntimeError(f"{bs_code} 查询错误: {rs.error_code} {rs.error_msg}") + rows = [] + while rs.next(): + rows.append(rs.get_row_data()) + if not rows: + return pd.DataFrame(columns=OUT_COLS) + df = pd.DataFrame(rows, columns=OUT_COLS) + for col in NUMERIC_COLS: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") # 空字符串→NaN→None + df = df.dropna(subset=["open", "high", "low", "close"]) + return df if not df.empty else pd.DataFrame(columns=OUT_COLS) + + +def parse_baostock_code(code: str) -> Tuple[str, str]: + """sh.600519 → ('600519', 'SH')。""" + prefix, symbol = code.split(".", 1) + return symbol, prefix.upper() + + +# ======================== 灌库 ======================== +def upsert_to_db(conn, df: pd.DataFrame) -> int: + """df → INSERT OR REPLACE daily_baostock_full (19 列)。返回写入行数。""" + if df.empty: + return 0 + work = df.copy() + work[["symbol", "exchange"]] = work["code"].apply( + lambda x: pd.Series(parse_baostock_code(x)) + ) + cols = ["symbol", "exchange", "date", "open", "high", "low", "close", + "preclose", "volume", "amount", "adjustflag", "turn", "tradestatus", + "pctChg", "peTTM", "psTTM", "pcfNcfTTM", "pbMRQ", "isST"] + work = work[cols] + work["adjustflag"] = work["adjustflag"].astype(str).replace({"nan": None, "None": None}) + # NaN → None (sqlite 写 NULL) + work = work.where(pd.notnull(work), None) + conn.executemany( + "INSERT OR REPLACE INTO daily_baostock_full VALUES " + "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + work.itertuples(index=False, name=None), + ) + return len(work) + + +# ======================== 主流程 ======================== +def parse_args(): + p = argparse.ArgumentParser(description="baostock 日增量 → daily_baostock_full (VPS 直跑)") + p.add_argument("--limit", type=int, default=0, help="限制处理股数 (测试)") + p.add_argument("--dry-run", action="store_true", help="只下不灌库") + return p.parse_args() + + +def main(): + global QUERY_COUNT + args = parse_args() + today = datetime.date.today() + end_date = today.strftime("%Y-%m-%d") + start_date = (today - datetime.timedelta(days=LOOKBACK_DAYS)).strftime("%Y-%m-%d") + + logger.info("=" * 60) + logger.info("baostock 日增量 → daily_baostock_full (VPS 直跑)") + logger.info(" DB: %s", DB_PATH) + logger.info(" 窗口: %s ~ %s (LOOKBACK=%d)", start_date, end_date, LOOKBACK_DAYS) + logger.info(" staging: %s", STAGING) + logger.info(" dry_run=%s limit=%s", args.dry_run, args.limit or "无") + logger.info("=" * 60) + + # 1. login 探针 — 黑名单 graceful skip + if not login_once(): + logger.error("[SKIP] baostock 登录失败 (疑似黑名单/冷却), graceful exit 2, 不硬跑") + sys.exit(2) + logger.info("baostock 登录成功 (IP 未被封)") + + # 2. 全 A 列表 + try: + all_codes = fetch_all_stocks() + except Exception as e: + logger.error("[FATAL] 取股票列表失败: %s", e) + try: + bs.logout() + except Exception: + pass + sys.exit(1) + logger.info("全 A (含退市): %d 只", len(all_codes)) + if args.limit > 0: + all_codes = all_codes[: args.limit] + logger.info("limit=%d 截断", args.limit) + + # 3. 连 DB (WAL + busy_timeout) + import sqlite3 + conn = sqlite3.connect(DB_PATH, timeout=60) + conn.execute("PRAGMA busy_timeout = 60000") + conn.execute("PRAGMA journal_mode = WAL") + rows_before = conn.execute("SELECT COUNT(*) FROM daily_baostock_full").fetchone()[0] + logger.info("灌库前 baseline rows=%d", rows_before) + + # 4. 主循环 + STAGING.mkdir(parents=True, exist_ok=True) + stats = {"ok": 0, "empty": 0, "failed": 0, "rows_db": 0, "rows_staged": 0} + limit_reached = False + sample_latest = [] + t0 = time.time() + + if not args.dry_run: + conn.execute("BEGIN") + try: + for i, (code, exc) in enumerate(all_codes): + if QUERY_COUNT >= DAILY_LIMIT: + logger.warning("query 计数 %d 达防线 %d, graceful stop", QUERY_COUNT, DAILY_LIMIT) + limit_reached = True + break + bs_code = f"{'sh' if exc == 'SH' else 'sz'}.{code}" + df = None + for attempt in range(BS_MAX_RETRIES): + try: + df = fetch_one_increment(bs_code, start_date, end_date) + break + except (socket.timeout, TimeoutError, OSError, Exception) as e: + logger.warning("%s 重试 %d/%d: %s", code, attempt + 1, BS_MAX_RETRIES, e) + if not relogin(): + df = None + break + if df is None: + stats["failed"] += 1 + elif df.empty: + stats["empty"] += 1 # 退市/停牌/区间外, 合法空 + else: + # staging (审计) + try: + df.to_parquet(STAGING / f"{code}.{exc}_incr.parquet", index=False) + stats["rows_staged"] += len(df) + except Exception as e: + logger.debug("staging 写失败 %s: %s", code, e) + # 灌库 + if not args.dry_run: + n = upsert_to_db(conn, df) + stats["rows_db"] += n + stats["ok"] += 1 + if len(sample_latest) < 5: + sample_latest.append((code, exc, len(df), df["date"].max())) + if (i + 1) % 500 == 0: + logger.info("进度 %d/%d ok=%d empty=%d failed=%d query=%d (%.0fs)", + i + 1, len(all_codes), stats["ok"], stats["empty"], + stats["failed"], QUERY_COUNT, time.time() - t0) + if i < len(all_codes) - 1: + time.sleep(BS_INTERVAL) + if args.dry_run: + logger.info("[DRY-RUN] 不灌库") + else: + conn.execute("COMMIT") + except Exception as e: + if not args.dry_run: + conn.execute("ROLLBACK") + logger.error("[FATAL] 事务回滚: %s", e) + try: + bs.logout() + except Exception: + pass + sys.exit(1) + + # 5. 统计 + rows_after = conn.execute("SELECT COUNT(*) FROM daily_baostock_full").fetchone()[0] + conn.close() + try: + bs.logout() + except Exception: + pass + + logger.info("=" * 60) + logger.info("[RESULT] dry_run=%s 耗时 %.0f 秒", args.dry_run, time.time() - t0) + logger.info(" ok=%d empty=%d failed=%d limit_reached=%s", + stats["ok"], stats["empty"], stats["failed"], limit_reached) + logger.info(" staging 行=%d 灌库行=%d query 总计=%d", + stats["rows_staged"], stats["rows_db"], QUERY_COUNT) + logger.info(" daily_baostock_full: %d → %d (+%d)", rows_before, rows_after, rows_after - rows_before) + logger.info(" 样本 latest: %s", sample_latest) + logger.info("=" * 60) + sys.exit(3 if limit_reached else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/daily_update_static_wrapper.ps1 b/scripts/data_platform/daily_update_static_wrapper.ps1 new file mode 100644 index 0000000..aa8d6c4 --- /dev/null +++ b/scripts/data_platform/daily_update_static_wrapper.ps1 @@ -0,0 +1,12 @@ +# daily_update_static_wrapper.ps1 — baostock 日增量 schtask wrapper (VPS SYSTEM) +# schtask: sanguo-bs-daily-increment /sc daily /st 17:00 /ru SYSTEM /sd 2026/07/23 +$env:http_proxy = '' +$env:https_proxy = '' +$env:all_proxy = '' +Set-Location C:\sanguo_vnpy_v2 +$ts = Get-Date -Format 'yyyyMMdd_HHmmss' +$logDir = 'C:\sanguo_vnpy_v2\data\daily_baostock_increment' +if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null } +$log = Join-Path $logDir "log_$ts.txt" +C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\daily_update_static.py *>> $log +exit $LASTEXITCODE diff --git a/scripts/data_platform/daily_update_xtdata.py b/scripts/data_platform/daily_update_xtdata.py index ac1f274..bbc7e3d 100644 --- a/scripts/data_platform/daily_update_xtdata.py +++ b/scripts/data_platform/daily_update_xtdata.py @@ -111,7 +111,11 @@ def merge_write(code, dividend_type, kind): def main(): end, start = today_str(), start_str() log(f"start LOOKBACK={LOOKBACK} window={start}~{end}") - u = xd.get_stock_list_in_sector("沪深A股") or [] + u = list(set( + (xd.get_stock_list_in_sector("沪深A股") or []) + + (xd.get_stock_list_in_sector("沪深ETF") or []) + + (xd.get_stock_list_in_sector("沪深基金") or []) + )) if not u: log("FATAL: empty universe（miniQMT 未连？）") os._exit(2) diff --git a/scripts/data_platform/import_baostock_to_db.py b/scripts/data_platform/import_baostock_to_db.py new file mode 100644 index 0000000..4a3a22d --- /dev/null +++ b/scripts/data_platform/import_baostock_to_db.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +导入baostock数据到quant_trading.db新表 + +保留dbbardata表不变，创建3个新表： +1. daily_baostock_full - 日K线数据 +2. bs_index_constituent - 指数成份股快照 +3. bs_adjust_factor - 复权因子 + +VPS路径： +- DB: C:\sanguo_vnpy_v2\data\quant_trading.db +- 日K: data\daily_baostock\*.parquet (5537 files) +- 成分: data\constituent_baostock\*.parquet (3219 files) +- 复权: data\static\adjust_factor\*.parquet (5537 files) +""" + +import sqlite3 +import pandas as pd +from pathlib import Path +import logging +from datetime import datetime +import sys + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler('import_baostock.log', encoding='utf-8') + ] +) +logger = logging.getLogger(__name__) + +# VPS路径配置 +BASE_DIR = Path(r'C:\sanguo_vnpy_v2') +DB_PATH = BASE_DIR / 'data' / 'quant_trading.db' +DAILY_DIR = BASE_DIR / 'data' / 'daily_baostock' +CONSTITUENT_DIR = BASE_DIR / 'data' / 'constituent_baostock' +ADJUST_DIR = BASE_DIR / 'data' / 'static' / 'adjust_factor' + +def parse_baostock_code(code: str) -> tuple: + """ + 解析baostock代码格式: sh.600000 / sz.000001 + 返回: (symbol, exchange) + 例: ('600000', 'SH') + """ + if '.' not in code: + return code, 'SZ' # 默认深圳 + prefix, symbol = code.split('.') + exchange = prefix.upper() + return symbol, exchange + +def create_tables(conn): + """创建3个新表""" + cursor = conn.cursor() + + # 1. daily_baostock_full + cursor.execute(''' + CREATE TABLE IF NOT EXISTS daily_baostock_full ( + symbol TEXT NOT NULL, + exchange TEXT NOT NULL, + date TEXT NOT NULL, + open REAL, + high REAL, + low REAL, + close REAL, + preclose REAL, + volume REAL, + amount REAL, + adjustflag TEXT, + turn REAL, + tradestatus REAL, + pctChg REAL, + peTTM REAL, + psTTM REAL, + pcfNcfTTM REAL, + pbMRQ REAL, + isST REAL, + PRIMARY KEY (symbol, exchange, date) + ) + ''') + + # 2. bs_index_constituent + cursor.execute(''' + CREATE TABLE IF NOT EXISTS bs_index_constituent ( + updateDate TEXT NOT NULL, + index_code TEXT NOT NULL, + code TEXT NOT NULL, + code_name TEXT, + PRIMARY KEY (updateDate, index_code, code) + ) + ''') + + # 3. bs_adjust_factor + cursor.execute(''' + CREATE TABLE IF NOT EXISTS bs_adjust_factor ( + code TEXT NOT NULL, + dividOperateDate TEXT NOT NULL, + foreAdjustFactor REAL, + backAdjustFactor REAL, + adjustFactor REAL, + PRIMARY KEY (code, dividOperateDate) + ) + ''') + + # 创建索引加速查询 + cursor.execute('CREATE INDEX IF NOT EXISTS idx_daily_symbol_date ON daily_baostock_full(symbol, exchange, date)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_constituent_index_date ON bs_index_constituent(index_code, updateDate)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_adjust_code ON bs_adjust_factor(code)') + + conn.commit() + logger.info("✅ 表创建完成") + +def import_daily_data(conn, chunk_size=100): + """导入日K线数据""" + logger.info("📊 开始导入日K线数据...") + cursor = conn.cursor() + + # 获取所有parquet文件 + parquet_files = list(DAILY_DIR.glob('*.parquet')) + total_files = len(parquet_files) + logger.info(f"找到 {total_files} 个日K线文件") + + processed = 0 + total_rows = 0 + + for file_path in parquet_files: + try: + # 读取parquet + df = pd.read_parquet(file_path) + + # 转换代码格式 + df[['symbol', 'exchange']] = df['code'].apply( + lambda x: pd.Series(parse_baostock_code(x)) + ) + + # 重命名列匹配数据库 + df = df.rename(columns={'pctChg': 'pctChg', 'isST': 'isST', 'peTTM': 'peTTM', + 'psTTM': 'psTTM', 'pcfNcfTTM': 'pcfNcfTTM', 'pbMRQ': 'pbMRQ'}) + + # 选择需要的列（确保顺序与表结构一致） + columns = ['symbol', 'exchange', 'date', 'open', 'high', 'low', 'close', + 'preclose', 'volume', 'amount', 'adjustflag', 'turn', 'tradestatus', + 'pctChg', 'peTTM', 'psTTM', 'pcfNcfTTM', 'pbMRQ', 'isST'] + df = df[columns] + + # 批量 executemany（替代逐行 iterrows；占位符严格 19 个 = 表 19 列，修原 20 占位符 bug） + df['adjustflag'] = df['adjustflag'].astype(str) + cursor.executemany( + 'INSERT OR REPLACE INTO daily_baostock_full VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', + df.itertuples(index=False, name=None), + ) + + total_rows += len(df) + processed += 1 + + # 每100个文件提交一次 + if processed % chunk_size == 0: + conn.commit() + logger.info(f"进度: {processed}/{total_files} ({processed/total_files*100:.1f}%) - {total_rows} rows") + + except Exception as e: + logger.error(f"处理文件 {file_path.name} 失败: {e}") + continue + + conn.commit() + logger.info(f"✅ 日K线数据导入完成: {total_rows} rows from {processed} files") + return total_rows + +def import_constituent_data(conn, chunk_size=100): + """导入指数成分股数据""" + logger.info("📊 开始导入指数成分股数据...") + cursor = conn.cursor() + + # 获取所有parquet文件 + parquet_files = list(CONSTITUENT_DIR.glob('*.parquet')) + total_files = len(parquet_files) + logger.info(f"找到 {total_files} 个成分股文件") + + processed = 0 + total_rows = 0 + + for file_path in parquet_files: + try: + # 从文件名提取index_code + # 例: hs300_2006-01-02.parquet -> hs300 + index_code = file_path.stem.split('_')[0] + + # 读取parquet + df = pd.read_parquet(file_path) + df['index_code'] = index_code + + # 选择需要的列 + df = df[['updateDate', 'index_code', 'code', 'code_name']] + + # 插入数据库 + for _, row in df.iterrows(): + cursor.execute(''' + INSERT OR REPLACE INTO bs_index_constituent VALUES (?, ?, ?, ?) + ''', tuple(row)) + + total_rows += len(df) + processed += 1 + + # 每100个文件提交一次 + if processed % chunk_size == 0: + conn.commit() + logger.info(f"进度: {processed}/{total_files} ({processed/total_files*100:.1f}%) - {total_rows} rows") + + except Exception as e: + logger.error(f"处理文件 {file_path.name} 失败: {e}") + continue + + conn.commit() + logger.info(f"✅ 成分股数据导入完成: {total_rows} rows from {processed} files") + return total_rows + +def import_adjust_factor_data(conn, chunk_size=100): + """导入复权因子数据""" + logger.info("📊 开始导入复权因子数据...") + cursor = conn.cursor() + + # 获取所有parquet文件 + parquet_files = list(ADJUST_DIR.glob('*.parquet')) + total_files = len(parquet_files) + logger.info(f"找到 {total_files} 个复权因子文件") + + processed = 0 + total_rows = 0 + + for file_path in parquet_files: + try: + # 读取parquet + df = pd.read_parquet(file_path) + + # 选择需要的列 + df = df[['code', 'dividOperateDate', 'foreAdjustFactor', 'backAdjustFactor', 'adjustFactor']] + + # 插入数据库 + for _, row in df.iterrows(): + cursor.execute(''' + INSERT OR REPLACE INTO bs_adjust_factor VALUES (?, ?, ?, ?, ?) + ''', tuple(row)) + + total_rows += len(df) + processed += 1 + + # 每100个文件提交一次 + if processed % chunk_size == 0: + conn.commit() + logger.info(f"进度: {processed}/{total_files} ({processed/total_files*100:.1f}%) - {total_rows} rows") + + except Exception as e: + logger.error(f"处理文件 {file_path.name} 失败: {e}") + continue + + conn.commit() + logger.info(f"✅ 复权因子数据导入完成: {total_rows} rows from {processed} files") + return total_rows + +def verify_import(conn): + """验证导入结果""" + logger.info("🔍 开始验证导入结果...") + cursor = conn.cursor() + + # 统计各表行数 + cursor.execute('SELECT COUNT(*) FROM daily_baostock_full') + daily_count = cursor.fetchone()[0] + + cursor.execute('SELECT COUNT(*) FROM bs_index_constituent') + constituent_count = cursor.fetchone()[0] + + cursor.execute('SELECT COUNT(*) FROM bs_adjust_factor') + adjust_count = cursor.fetchone()[0] + + # 抽样检查日K线数据 + cursor.execute(''' + SELECT symbol, date, close, turn, peTTM + FROM daily_baostock_full + WHERE symbol='000001' AND exchange='SZ' + ORDER BY date DESC LIMIT 5 + ''') + sample_daily = cursor.fetchall() + + # 检查日期范围 + cursor.execute('SELECT MIN(date), MAX(date) FROM daily_baostock_full') + date_range = cursor.fetchone() + + # 统计唯一symbol数量 + cursor.execute('SELECT COUNT(DISTINCT symbol) FROM daily_baostock_full') + symbol_count = cursor.fetchone()[0] + + logger.info(f"📊 验证结果:") + logger.info(f" daily_baostock_full: {daily_count:,} rows, {symbol_count} symbols") + logger.info(f" 日期范围: {date_range[0]} ~ {date_range[1]}") + logger.info(f" bs_index_constituent: {constituent_count:,} rows") + logger.info(f" bs_adjust_factor: {adjust_count:,} rows") + + if sample_daily: + logger.info(f" 抽样 000001.SZ 最新5行:") + for row in sample_daily: + logger.info(f" {row[0]} {row[1]}: close={row[2]:.2f}, turn={row[3]:.2f}, peTTM={row[4]:.2f}") + + return { + 'daily_count': daily_count, + 'constituent_count': constituent_count, + 'adjust_count': adjust_count, + 'symbol_count': symbol_count, + 'date_range': date_range, + 'sample_daily': sample_daily + } + +def main(): + """主函数""" + logger.info("=" * 50) + logger.info("开始导入baostock数据到quant_trading.db") + logger.info(f"DB路径: {DB_PATH}") + logger.info(f"日K目录: {DAILY_DIR}") + logger.info(f"成分目录: {CONSTITUENT_DIR}") + logger.info(f"复权目录: {ADJUST_DIR}") + logger.info("=" * 50) + + # 检查目录是否存在 + if not DB_PATH.exists(): + logger.error(f"❌ DB不存在: {DB_PATH}") + return + + for dir_path in [DAILY_DIR, CONSTITUENT_DIR, ADJUST_DIR]: + if not dir_path.exists(): + logger.error(f"❌ 目录不存在: {dir_path}") + return + + # 连接数据库（WAL + busy_timeout防写竞争） + conn = sqlite3.connect(DB_PATH, timeout=60) + conn.execute('PRAGMA busy_timeout = 60000') # 60秒 + conn.execute('PRAGMA journal_mode = WAL') # WAL模式 + + try: + # 记录开始时间 + start_time = datetime.now() + + # 1. 创建表 + logger.info("🔨 创建表结构...") + create_tables(conn) + + # 2. 导入日K线数据 + daily_count = import_daily_data(conn) + + # 3. 导入成分股数据 + constituent_count = import_constituent_data(conn) + + # 4. 导入复权因子数据 + adjust_count = import_adjust_factor_data(conn) + + # 5. 验证结果 + results = verify_import(conn) + + # 计算耗时 + elapsed = (datetime.now() - start_time).total_seconds() + + logger.info("=" * 50) + logger.info(f"✅ 导入完成! 总耗时: {elapsed/60:.1f} 分钟") + logger.info("=" * 50) + + # 输出最终报告 + print("\n" + "=" * 50) + print("📊 导入完成报告") + print("=" * 50) + print(f"daily_baostock_full: {results['daily_count']:,} rows / {results['symbol_count']} symbols / 日期范围 {results['date_range'][0]} ~ {results['date_range'][1]}") + print(f"bs_index_constituent: {results['constituent_count']:,} rows") + print(f"bs_adjust_factor: {results['adjust_count']:,} rows") + print("=" * 50) + + return results + + except Exception as e: + logger.error(f"❌ 导入失败: {e}") + import traceback + logger.error(traceback.format_exc()) + return None + finally: + conn.close() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/data_platform/import_bs_wrapper.ps1 b/scripts/data_platform/import_bs_wrapper.ps1 new file mode 100644 index 0000000..47d8e07 --- /dev/null +++ b/scripts/data_platform/import_bs_wrapper.ps1 @@ -0,0 +1,14 @@ +# baostock 灌库一次性 wrapper（schtask /ru SYSTEM 跑，独立 ssh 会话） +# 修 import_baostock_to_db.py 后重跑（幂等 INSERT OR REPLACE，空表重灌） +$ErrorActionPreference = 'Continue' +$ProgressPreference = 'SilentlyContinue' +Set-Location C:\sanguo_vnpy_v2 +$env:PYTHONIOENCODING = 'utf-8' +$env:VNPY_DB_PATH = 'C:\sanguo_vnpy_v2\data\quant_trading.db' +$log = 'C:\sanguo_vnpy_v2\data\import_bs_wrapper.log' +"=== IMPORT_BS START $(Get-Date -Format o) ===" | Out-File -FilePath $log -Encoding utf8 +"DiskFree_GB_start=$((Get-PSDrive C).Free / 1GB)" | Out-File -FilePath $log -Encoding utf8 -Append +& C:\Python310\python.exe -X utf8 C:\sanguo_vnpy_v2\scripts\data_platform\import_baostock_to_db.py *>&1 | Tee-Object -FilePath $log -Append +$exit_code = $LASTEXITCODE +"=== IMPORT_BS DONE $(Get-Date -Format o) exit=$exit_code ===" | Out-File -FilePath $log -Encoding utf8 -Append +"DiskFree_GB_end=$((Get-PSDrive C).Free / 1GB)" | Out-File -FilePath $log -Encoding utf8 -Append diff --git a/scripts/data_platform/import_delisted_to_db.py b/scripts/data_platform/import_delisted_to_db.py new file mode 100644 index 0000000..25aade5 --- /dev/null +++ b/scripts/data_platform/import_delisted_to_db.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""导入退市股 K 线到 VPS quant_trading.db 的 daily_baostock_full 表 (INSERT OR REPLACE) + +在 VPS 上跑 (ssh VPS). 不用 baostock. + +目标: + C:\\sanguo_vnpy_v2\\data\\delisted_kline\\*.parquet (rsync Mac→VPS 上来的) + → daily_baostock_full (19 列, PK(symbol, exchange, date)) + INSERT OR REPLACE, 不碰 dbbardata 表. + +空字符串字段已在 Mac 下载阶段 pd.to_numeric(errors='coerce') 转 NaN, parquet 存 None; +这里 pd.read_parquet → sqlite executemany 会写 NULL, 不会触发类型错误. + +用法: + python import_delisted_to_db.py # 全量灌 + python import_delisted_to_db.py --dry-run # 只校验不入库 +""" + +import argparse +import logging +import sqlite3 +import sys +from datetime import datetime +from pathlib import Path + +import pandas as pd + +# Windows VPS 路径 +BASE_DIR = Path(r"C:\sanguo_vnpy_v2") +DB_PATH = BASE_DIR / "data" / "quant_trading.db" +DELISTED_DIR = BASE_DIR / "data" / "delisted_kline" + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) +logger = logging.getLogger(__name__) + + +def parse_baostock_code(code: str) -> tuple: + """sh.600074 → ('600074', 'SH')""" + if "." not in code: + return code, "SZ" + prefix, symbol = code.split(".", 1) + return symbol, prefix.upper() + + +def table_exists(conn: sqlite3.Connection, name: str) -> bool: + cur = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (name,) + ) + return cur.fetchone() is not None + + +def get_row_count(conn: sqlite3.Connection, table: str, where: str = "") -> int: + sql = f"SELECT COUNT(*) FROM {table}" + if where: + sql += f" WHERE {where}" + return conn.execute(sql).fetchone()[0] + + +def import_one_parquet(conn: sqlite3.Connection, parq: Path) -> tuple: + """返回 (rows_inserted, symbol, exchange, max_date)""" + df = pd.read_parquet(parq) + if df.empty: + return 0, "", "", "" + + # 拆 baostock code → (symbol, exchange) + df[["symbol", "exchange"]] = df["code"].apply( + lambda x: pd.Series(parse_baostock_code(x)) + ) + + columns = ["symbol", "exchange", "date", "open", "high", "low", "close", + "preclose", "volume", "amount", "adjustflag", "turn", "tradestatus", + "pctChg", "peTTM", "psTTM", "pcfNcfTTM", "pbMRQ", "isST"] + df = df[columns] + # adjustflag 保留字符串, NaN→None 由 pandas → sqlite 自动转 + df["adjustflag"] = df["adjustflag"].astype(str).replace({"nan": None, "None": None}) + + conn.executemany( + "INSERT OR REPLACE INTO daily_baostock_full VALUES " + "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + df.itertuples(index=False, name=None), + ) + return len(df), df["symbol"].iloc[0], df["exchange"].iloc[0], df["date"].max() + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--dry-run", action="store_true", help="只校验不入库") + p.add_argument("--dir", default=str(DELISTED_DIR), help="退市 K 线 parquet 目录") + args = p.parse_args() + + src_dir = Path(args.dir) + logger.info("=" * 60) + logger.info("退市股 K 线灌库 → daily_baostock_full (INSERT OR REPLACE)") + logger.info(" DB: %s", DB_PATH) + logger.info(" 目录: %s", src_dir) + logger.info(" dry_run=%s", args.dry_run) + logger.info("=" * 60) + + if not DB_PATH.exists(): + logger.error("[FATAL] DB 不存在: %s", DB_PATH) + sys.exit(1) + if not src_dir.exists(): + logger.error("[FATAL] 目录不存在: %s", src_dir) + sys.exit(1) + + parquets = sorted(src_dir.glob("*_delisted.parquet")) + logger.info("找到 %d 个 parquet 文件", len(parquets)) + if not parquets: + logger.error("[FATAL] 无 parquet, 检查 rsync 是否完成") + sys.exit(1) + + conn = sqlite3.connect(DB_PATH, timeout=60) + conn.execute("PRAGMA busy_timeout = 60000") + conn.execute("PRAGMA journal_mode = WAL") + + if not table_exists(conn, "daily_baostock_full"): + logger.error("[FATAL] daily_baostock_full 表不存在 (先跑 import_baostock_to_db.py)") + sys.exit(1) + + # 灌前 baseline + rows_before = get_row_count(conn, "daily_baostock_full") + symbols_before = get_row_count( + conn, "daily_baostock_full", "1 GROUP BY symbol, exchange" + ) + logger.info("灌库前 baseline: rows=%d, distinct(symbol,exchange)=%d", + rows_before, symbols_before) + + total_rows = 0 + processed = 0 + failed = 0 + sample_inserted = [] + + conn.execute("BEGIN") + try: + for i, parq in enumerate(parquets): + try: + rows, symbol, exc, maxd = import_one_parquet(conn, parq) + total_rows += rows + processed += 1 + if i < 5: + sample_inserted.append((symbol, exc, rows, maxd)) + if (i + 1) % 20 == 0: + logger.info("进度 %d/%d rows=%d (latest: %s.%s max_date=%s)", + i + 1, len(parquets), total_rows, symbol, exc, maxd) + except Exception as e: + failed += 1 + logger.error("处理 %s 失败: %s", parq.name, e) + continue + if args.dry_run: + conn.execute("ROLLBACK") + logger.info("[DRY-RUN] 回滚, 不入库") + else: + conn.execute("COMMIT") + except Exception as e: + conn.execute("ROLLBACK") + logger.error("[FATAL] 事务回滚: %s", e) + sys.exit(2) + finally: + conn.close() + + # 灌后统计 (重新连) + conn2 = sqlite3.connect(DB_PATH, timeout=60) + rows_after = get_row_count(conn2, "daily_baostock_full") + symbols_after = get_row_count( + conn2, "daily_baostock_full", "1 GROUP BY symbol, exchange" + ) + conn2.close() + + logger.info("=" * 60) + logger.info("[RESULT] 灌库完成 dry_run=%s", args.dry_run) + logger.info(" 处理 parquet: %d (failed=%d)", processed, failed) + logger.info(" 插入/替换行: %d", total_rows) + logger.info(" daily_baostock_full: %d → %d (+%d rows)", + rows_before, rows_after, rows_after - rows_before) + logger.info(" distinct(symbol,exchange): %d → %d (+%d)", + symbols_before, symbols_after, symbols_after - symbols_before) + logger.info(" 样本 (前 5): %s", sample_inserted) + logger.info("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/import_vnpy_minute_fast.py b/scripts/data_platform/import_vnpy_minute_fast.py index cffe5a2..93d852d 100644 --- a/scripts/data_platform/import_vnpy_minute_fast.py +++ b/scripts/data_platform/import_vnpy_minute_fast.py @@ -208,7 +208,8 @@ def main(): print(f'FATAL: DB not found: {DB_PATH}', flush=True) sys.exit(1) - conn = sqlite3.connect(DB_PATH) + conn = sqlite3.connect(DB_PATH, timeout=60) + conn.execute("PRAGMA busy_timeout=60000") # wait vs concurrent web writer (WAL single-writer) t_start = time.time() grand_rows = 0 diff --git a/scripts/data_platform/index_const_hist_download.py b/scripts/data_platform/index_const_hist_download.py new file mode 100644 index 0000000..74280e8 --- /dev/null +++ b/scripts/data_platform/index_const_hist_download.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""历史成份股采集 (治幸存者偏差) - P0 Task1 + +数据源 (探针 2026-07-21 实证): + A. 深证/国证 adjust : ak.index_detail_hist_adjust_cni(sym) + 399001/399006/399330/399005 + 返回 10 期半年度调样 (2022-06~2026-12) + 列: 开始日期/结束日期/样本代码/样本简称/所属行业/调整类型(OLD/+/-/备选) + 陷阱: 必须 _adjust_cni (hist_cni 只返回最新一期, 陷阱) + B. 中证1000/2000快照: ak.index_stock_cons_csindex(sym) + 000852/932000 当前快照 (历史不可补 = 已知 gap) + C. 新浪交叉校验(可选): 000300/000016/000905 table[3] (非[0]陷阱) + baostock 已有 300/500/50 主源, 新浪仅交叉校验 + +输出 (OUT_DIR/{code}_*.parquet): + {code}_adjust.parquet - 原始调整记录 (统一 schema) + {code}_union.parquet - 曾经入选集 (OLD ∪ + ∪ -, 治幸存者偏差) + {code}_current.parquet - 最新一期成份股 (OLD ∪ +) + {code}_snapshot.parquet - csindex 当前快照 (仅 000852/932000) + {code}_sina.parquet - 新浪交叉校验 (可选) + +硬约束: + 1. 单进程串行, sleep 1.0~1.5s (akshare/新浪防封) + 2. 直连不走代理 (unset proxy) + 3. 不碰 baostock (Task3 退市探针在用, 并发会封 IP) + 4. 环境变量 INDEX_HIST_OUT_DIR 覆盖默认 Mac 路径 + +用法: + python index_const_hist_download.py # 全量 A+B+C + python index_const_hist_download.py --parts A # 只 A 深证/国证 + python index_const_hist_download.py --parts A B # A + B + python index_const_hist_download.py --skip-sina # 跳过 C 新浪 + +退出码: 0=完成, 1=致命错误 +""" + +import argparse +import logging +import os +import socket +import sys +import time +from pathlib import Path +from typing import List, Optional, Tuple + +# ======================== 硬约束: unset proxy + socket timeout ======================== +for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"): + os.environ.pop(_k, None) + +socket.setdefaulttimeout(30) + +try: + sys.stdout.reconfigure(line_buffering=True) +except (AttributeError, ValueError): + pass + +import pandas as pd # noqa: E402 + +# ======================== 配置 ======================== + +DEFAULT_OUT_DIR = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/data/index_const_hist" +OUT_DIR = Path(os.environ.get("INDEX_HIST_OUT_DIR", DEFAULT_OUT_DIR)) +LOG_DIR = OUT_DIR / "logs" + +# A. 深证/国证 adjust 指数 (cni 来源, 探针实证可返回 10 期半年度调样) +CNI_ADJUST_INDICES = [ + ("399001", "深证成指"), + ("399006", "创业板指"), + ("399330", "深证100"), + ("399005", "中小板指"), +] + +# B. 中证1000/2000 csindex 当前快照 (历史不可补 = 已知 gap) +CSINDEX_SNAPSHOT_INDICES = [ + ("000852", "中证1000"), + ("932000", "中证2000"), +] + +# C. 新浪交叉校验 (baostock 主源已有 300/500/50, 此处仅校验) +SINA_CROSS_INDICES = [ + ("000300", "沪深300"), + ("000905", "中证500"), + ("000016", "上证50"), +] + +# 限速 (秒) +SLEEP_AKSHARE = 1.2 # akshare 每次调用间隔 +SLEEP_SINA_PAGE = 1.5 # 新浪翻页间隔 +SLEEP_BETWEEN_PARTS = 2.0 # 阶段间额外间隔 + +# 新浪翻页上限 (探针: 7-9 页) +SINA_MAX_PAGES = 12 + + +# ======================== 日志 ======================== + +def setup_logging() -> Tuple[logging.Logger, Path]: + """配置 root logger: 同时写 stdout + 文件。""" + OUT_DIR.mkdir(parents=True, exist_ok=True) + LOG_DIR.mkdir(parents=True, exist_ok=True) + log_file = LOG_DIR / f"index_const_hist_{time.strftime('%Y%m%d_%H%M%S')}.log" + + logger = logging.getLogger("index_const_hist") + logger.setLevel(logging.INFO) + logger.handlers.clear() + + fmt = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S") + + fh = logging.FileHandler(log_file, encoding="utf-8") + fh.setFormatter(fmt) + logger.addHandler(fh) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(fmt) + logger.addHandler(sh) + + logger.info(f"OUT_DIR = {OUT_DIR}") + logger.info(f"log_file = {log_file}") + return logger, log_file + + +# ======================== A. 深证/国证 adjust ======================== + +def _normalize_akshare_code(raw: str) -> str: + """akshare 返回的样本代码可能是 '000001' 或 'sz000001' / 'sh600000' 形式, 归一化成 6 位代码.""" + if not isinstance(raw, str): + return "" + s = raw.strip() + # 去 sh/sz/bj 前缀 + for prefix in ("sh", "sz", "bj", "SH", "SZ", "BJ"): + if s.startswith(prefix): + s = s[len(prefix):] + break + return s.strip() + + +def download_cni_adjust(index_code: str, index_name: str, logger: logging.Logger) -> Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame], Optional[pd.DataFrame]]: + """A. ak.index_detail_hist_adjust_cni 拉取 + 重建. + + Returns: + (adjust_df, union_df, current_df) + adjust_df - 原始调整记录统一 schema + union_df - 曾经入选集 (OLD ∪ + ∪ -), 治幸存者偏差 + current_df - 最新一期成份股 (OLD ∪ +) + """ + import akshare as ak + + logger.info(f"[A] {index_code} {index_name} ak.index_detail_hist_adjust_cni ...") + try: + raw = ak.index_detail_hist_adjust_cni(symbol=index_code) + except Exception as e: + logger.error(f"[A] {index_code} ak.Index_detail_hist_adjust_cni failed: {e}") + return None, None, None + + if raw is None or len(raw) == 0: + logger.warning(f"[A] {index_code} empty result") + return None, None, None + + logger.info(f"[A] {index_code} raw rows={len(raw)}, cols={list(raw.columns)}") + + # 统一 schema (探针实证列名: 开始日期/结束日期/样本代码/样本简称/所属行业/调整类型) + col_map = { + "开始日期": "updateDate", + "结束日期": "endDate", + "样本代码": "code", + "样本简称": "code_name", + "所属行业": "industry", + "调整类型": "adjust_type", + } + df = raw.rename(columns=col_map).copy() + + # 归一化代码 + df["code"] = df["code"].apply(_normalize_akshare_code) + df["index_code"] = index_code + df["index_name"] = index_name + + # 幸存者偏差治理: 重建 + # 当期成份股 = OLD ∪ + (old 留存 + 新调入) + # 曾经入选 = OLD ∪ + ∪ - (含被踢) + def _codes_for_types(types: set) -> set: + mask = df["adjust_type"].isin(types) + return set(df.loc[mask, "code"].tolist()) + + old_codes = _codes_for_types({"OLD"}) + plus_codes = _codes_for_types({"+", "新增", "调入"}) + minus_codes = _codes_for_types({"-", "删除", "调出"}) + + current_codes = old_codes | plus_codes + ever_codes = old_codes | plus_codes | minus_codes + + logger.info( + f"[A] {index_code} adjust breakdown: OLD={len(old_codes)} " + f"+={len(plus_codes)} -={len(minus_codes)}" + ) + logger.info( + f"[A] {index_code} UNION (ever in)={len(ever_codes)}, " + f"CURRENT (OLD ∪ +)={len(current_codes)}" + ) + + # union df: 去重成份股 + 标注是否在当前 + union_records = [] + all_codes_with_name = ( + df[df["adjust_type"].isin({"OLD", "+", "-", "新增", "调入", "删除", "调出"})] + .drop_duplicates(subset=["code"]) + [["code", "code_name", "industry"]] + ) + for _, row in all_codes_with_name.iterrows(): + code = row["code"] + union_records.append({ + "code": code, + "code_name": row.get("code_name", ""), + "industry": row.get("industry", ""), + "index_code": index_code, + "index_name": index_name, + "in_current": code in current_codes, + "was_removed": code in minus_codes and code not in current_codes, + }) + union_df = pd.DataFrame(union_records) + + # current df: 最新一期 (endDate 最大那期的 OLD ∪ +) + current_df = df[df["adjust_type"].isin({"OLD", "+", "新增", "调入"})][ + ["updateDate", "endDate", "code", "code_name", "industry", "index_code"] + ].copy() + + return df[["updateDate", "endDate", "code", "code_name", "industry", "adjust_type", "index_code", "index_name"]], union_df, current_df + + +# ======================== B. 中证1000/2000 快照 ======================== + +def _fetch_csindex_snapshot_with_timeout(index_code: str, timeout: float = 15.0): + """用线程 + future.result(timeout) 包装 csindex 接口 (接口常超时挂死). + + 关键坑: ThreadPoolExecutor 的 with 块退出时 shutdown(wait=True) 会卡住等 worker, + timeout 就失效了. 这里手动 shutdown(wait=False) 让 worker 在后台自生自灭. + """ + import akshare as ak + from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout + + ex = ThreadPoolExecutor(max_workers=1) + fut = ex.submit(ak.index_stock_cons_csindex, symbol=index_code) + try: + result = fut.result(timeout=timeout) + ex.shutdown(wait=False) + return result, None + except FutTimeout: + ex.shutdown(wait=False) # 不等 worker, 主线程立即返回 + return None, "timeout" + except Exception as e: + ex.shutdown(wait=False) + return None, str(e) + + +def download_csindex_snapshot(index_code: str, index_name: str, logger: logging.Logger) -> Optional[pd.DataFrame]: + """B. ak.index_stock_cons_csindex 当前快照 (带 15s 超时保护, 3 次重试, 全失败标 gap).""" + raw, err = None, None + for attempt in range(1, 4): # 最多 3 次重试, csindex 接口时好时坏 + logger.info(f"[B] {index_code} {index_name} ak.index_stock_cons_csindex (attempt {attempt}/3, timeout=15s) ...") + raw, err = _fetch_csindex_snapshot_with_timeout(index_code, timeout=15.0) + if not err and raw is not None and len(raw) > 0: + break + logger.warning(f"[B] {index_code} attempt {attempt} fail: {err}") + if attempt < 3: + time.sleep(5.0) + if err: + logger.error(f"[B] {index_code} ak.index_stock_cons_csindex FAIL after 3 attempts: {err} → 标 placeholder gap") + # 写 placeholder parquet 标 historical=false + unavailable=true + return pd.DataFrame([{ + "updateDate": time.strftime("%Y-%m-%d"), + "code": "", + "code_name": "", + "index_code": index_code, + "index_name": index_name, + "historical": False, + "unavailable": True, + "note": f"csindex snapshot fetch {err} after 3 attempts, known gap", + }]) + + if raw is None or len(raw) == 0: + logger.warning(f"[B] {index_code} empty snapshot → placeholder gap") + return pd.DataFrame([{ + "updateDate": time.strftime("%Y-%m-%d"), + "code": "", + "code_name": "", + "index_code": index_code, + "index_name": index_name, + "historical": False, + "unavailable": True, + "note": "csindex snapshot empty, known gap", + }]) + + logger.info(f"[B] {index_code} raw rows={len(raw)}, cols={list(raw.columns)}") + + # 统一 schema: 尽量保留原始列 + 加 index_code/historical=False + df = raw.copy() + # 归一代码列 (csindex 实际列名: "成分券代码" - 注意是"分"不是"份"!) + code_col = None + for c in ("成分券代码", "成份券代码", "成分券代码 ", "代码", "code", "symbol"): + if c in df.columns: + code_col = c + break + if code_col: + df["code"] = df[code_col].apply(_normalize_akshare_code) + else: + logger.warning(f"[B] {index_code} no code column matched in {list(raw.columns)}") + name_col = None + for c in ("成分券名称", "成份券名称", "成分券简称", "成份券简称", "名称", "code_name", "name"): + if c in df.columns: + name_col = c + break + if name_col: + df["code_name"] = df[name_col] + + df["index_code"] = index_code + df["index_name"] = index_name + if "historical" not in df.columns: + df["historical"] = False # 标注: 仅当前快照, 无历史 + df["updateDate"] = time.strftime("%Y-%m-%d") # 采集日期 + + keep_cols = ["updateDate", "code", "code_name", "index_code", "index_name", "historical"] + if "unavailable" in df.columns: + keep_cols.append("unavailable") + if "note" in df.columns: + keep_cols.append("note") + keep_cols = [c for c in keep_cols if c in df.columns] + return df[keep_cols].reset_index(drop=True) + + +# ======================== C. 新浪交叉校验 ======================== + +def _fetch_sina_component_table(index_code: str, logger: logging.Logger) -> Optional[pd.DataFrame]: + """新浪 vII_HistoryComponent 翻页拉取, 返回合并后的成份股历史. + + 新浪真表是 table[3] (非[0] 陷阱). 翻页 ?pageNum=N. + """ + import requests + + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Referer": "https://vip.stock.finance.sina.com.cn/", + } + + all_rows = [] + prev_codes_signature = None # 假分页检测: 翻页前后内容签名 + for page in range(1, SINA_MAX_PAGES + 1): + # 新浪 URL: 历史成份股, 参数 sym (如 sh000300) + pageNum + prefix = "sh" if index_code.startswith(("000", "9")) else "sz" + url = ( + f"https://vip.stock.finance.sina.com.cn/corp/go.php/vII_HistoryComponent/" + f"{prefix}{index_code}.phtml" + ) + params = {"pageNum": page} + try: + resp = requests.get(url, headers=headers, params=params, timeout=15) + resp.encoding = "gb2312" + except Exception as e: + logger.warning(f"[C] {index_code} sina page {page} fetch fail: {e}") + break + + if resp.status_code != 200: + logger.warning(f"[C] {index_code} sina page {page} status={resp.status_code}") + break + + try: + from io import StringIO + tables = pd.read_html(StringIO(resp.text)) + except Exception as e: + logger.warning(f"[C] {index_code} sina page {page} read_html fail: {e}") + break + + # 陷阱: 真表是 table[3] 非 [0] + if len(tables) <= 3: + logger.info(f"[C] {index_code} sina page {page} tables={len(tables)}, stop") + break + tbl = tables[3] + if tbl is None or len(tbl) == 0: + logger.info(f"[C] {index_code} sina page {page} empty table[3], stop") + break + + logger.info(f"[C] {index_code} sina page {page} rows={len(tbl)}, cols={list(tbl.columns)}") + + # 假分页检测: 新浪 ?pageNum 是假分页, 每页同内容. 用前 4 列签名对比 + sig = tuple(tbl.iloc[:5, :4].astype(str).values.flatten()) + if prev_codes_signature is not None and sig == prev_codes_signature: + logger.info(f"[C] {index_code} sina page {page} same as prev (假分页 detected), stop") + break + prev_codes_signature = sig + + all_rows.append(tbl) + + time.sleep(SLEEP_SINA_PAGE) + + # 少于 30 行视为末页 + if len(tbl) < 30: + break + + if not all_rows: + return None + + big = pd.concat(all_rows, ignore_index=True) + return big + + +def download_sina_cross(index_code: str, index_name: str, logger: logging.Logger) -> Optional[pd.DataFrame]: + """C. 新浪历史成份股 (交叉校验 baostock 300/500/50). + + 新浪 schema (table[3]): 多级 header, 第一行数据是 真正列名: + 品种代码 / 品种名称 / 纳入日期 / 剔除日期 + 后续每只股票一行 (含被剔除的, 治幸存者偏差). + """ + logger.info(f"[C] {index_code} {index_name} sina vII_HistoryComponent ...") + raw = _fetch_sina_component_table(index_code, logger) + if raw is None or len(raw) == 0: + logger.warning(f"[C] {index_code} sina empty") + return None + + # 陷阱: table[3] 第一行是 "品种代码/品种名称/纳入日期/剔除日期" 字符串 (非数据) + # 列名是 "历史成分/历史成分.1/..." 需要重命名 + df = raw.copy() + # 找到包含 "品种代码" 的行作为 header 行, 丢弃它, 用前 4 列做数据 + header_mask = df.apply(lambda r: r.astype(str).str.contains("品种代码").any(), axis=1) + if header_mask.any(): + df = df[~header_mask].reset_index(drop=True) + + # 取前 4 列 (品种代码/品种名称/纳入日期/剔除日期), 重命名 + if df.shape[1] >= 4: + df = df.iloc[:, :4] + df.columns = ["code", "code_name", "in_date", "out_date"] + + # 清洗 (剔除空行/非字符串) + df = df[df["code"].notna() & (df["code"].astype(str).str.strip() != "")].copy() + df["code"] = df["code"].apply(_normalize_akshare_code) + # 保险 dedup (即使假分页检测漏网, 这里兜底) + before_dedup = len(df) + df = df.drop_duplicates(subset=["code", "in_date", "out_date"]).reset_index(drop=True) + if len(df) < before_dedup: + logger.info(f"[C] {index_code} dedup {before_dedup} -> {len(df)}") + df["index_code"] = index_code + df["index_name"] = index_name + df["source"] = "sina" + + # 治幸存者偏差证据: 有 out_date (非空) = 被剔除股票 + df["was_removed"] = df["out_date"].notna() & (df["out_date"].astype(str).str.strip() != "") + n_total = len(df) + n_removed = int(df["was_removed"].sum()) + n_current = n_total - n_removed + logger.info( + f"[C] {index_code} sina total={n_total} current(no out_date)={n_current} removed(out_date 非空)={n_removed}" + ) + return df.reset_index(drop=True) + + +# ======================== 保存 ======================== + +def save_parquet(df: pd.DataFrame, path: Path, logger: logging.Logger) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(path, index=False) + logger.info(f"saved {path} rows={len(df)} size={path.stat().st_size}B") + + +# ======================== 主流程 ======================== + +def run(orders: List[str], skip_sina: bool, logger: logging.Logger) -> int: + """串行跑 A/B/C.""" + summary_rows = [] + + # ----- A. 深证/国证 adjust ----- + if "A" in orders: + logger.info("=" * 60) + logger.info("PART A: CNI adjust (深证/国证) START") + logger.info("=" * 60) + for idx_code, idx_name in CNI_ADJUST_INDICES: + try: + adj, union, current = download_cni_adjust(idx_code, idx_name, logger) + except Exception as e: + logger.exception(f"[A] {idx_code} unexpected error: {e}") + adj, union, current = None, None, None + + if adj is not None: + save_parquet(adj, OUT_DIR / f"{idx_code}_adjust.parquet", logger) + if union is not None: + save_parquet(union, OUT_DIR / f"{idx_code}_union.parquet", logger) + if current is not None: + save_parquet(current, OUT_DIR / f"{idx_code}_current.parquet", logger) + + # 统计 (幸存者偏差证据) + n_union = len(union) if union is not None else 0 + n_current = len(current.drop_duplicates(subset=["code"])) if current is not None else 0 + n_removed = int((union["was_removed"].sum()) if (union is not None and "was_removed" in union.columns) else 0) + summary_rows.append({ + "part": "A", + "index_code": idx_code, + "index_name": idx_name, + "rows_adjust": len(adj) if adj is not None else 0, + "distinct_code_ever": n_union, + "current_codes": n_current, + "removed_codes": n_removed, + "note": "historical=true (adjust 10 期)", + }) + + time.sleep(SLEEP_AKSHARE) + time.sleep(SLEEP_BETWEEN_PARTS) + + # ----- B. 中证1000/2000 快照 ----- + if "B" in orders: + logger.info("=" * 60) + logger.info("PART B: CSINDEX snapshot (中证1000/2000) START") + logger.info("=" * 60) + for idx_code, idx_name in CSINDEX_SNAPSHOT_INDICES: + try: + snap = download_csindex_snapshot(idx_code, idx_name, logger) + except Exception as e: + logger.exception(f"[B] {idx_code} unexpected error: {e}") + snap = None + + if snap is not None: + save_parquet(snap, OUT_DIR / f"{idx_code}_snapshot.parquet", logger) + + # B 部分 summary: 如有 code 列算 distinct, 否则用总行数 + if snap is not None and "code" in snap.columns and (snap["code"].astype(str).str.strip() != "").any(): + n_codes = int(snap["code"].replace("", pd.NA).dropna().nunique()) + unavail = False + elif snap is not None and "unavailable" in snap.columns: + n_codes = 0 + unavail = bool(snap["unavailable"].iloc[0]) + else: + n_codes = len(snap) if snap is not None else 0 + unavail = False + note_b = "historical=false (snapshot only, 历史不可补 = 已知 gap)" + if unavail: + note_b = "UNAVAILABLE: csindex timeout after 3 attempts, placeholder gap" + + summary_rows.append({ + "part": "B", + "index_code": idx_code, + "index_name": idx_name, + "rows_adjust": len(snap) if snap is not None else 0, + "distinct_code_ever": n_codes, + "current_codes": n_codes, + "removed_codes": 0, + "note": note_b, + }) + + time.sleep(SLEEP_AKSHARE) + time.sleep(SLEEP_BETWEEN_PARTS) + + # ----- C. 新浪交叉校验 ----- + if "C" in orders and not skip_sina: + logger.info("=" * 60) + logger.info("PART C: SINA cross-check (300/500/50) START") + logger.info("=" * 60) + for idx_code, idx_name in SINA_CROSS_INDICES: + try: + sina_df = download_sina_cross(idx_code, idx_name, logger) + except Exception as e: + logger.exception(f"[C] {idx_code} unexpected error: {e}") + sina_df = None + + if sina_df is not None: + save_parquet(sina_df, OUT_DIR / f"{idx_code}_sina.parquet", logger) + + if sina_df is not None and "was_removed" in sina_df.columns: + n_total = len(sina_df) + n_removed = int(sina_df["was_removed"].sum()) + n_current = n_total - n_removed + note = "cross-check vs baostock (主源)" + else: + n_total = len(sina_df) if sina_df is not None else 0 + n_removed = 0 + n_current = 0 + note = "sina empty or fetch fail" + + summary_rows.append({ + "part": "C", + "index_code": idx_code, + "index_name": idx_name, + "rows_adjust": n_total, + "distinct_code_ever": n_total, + "current_codes": n_current, + "removed_codes": n_removed, + "note": note, + }) + + time.sleep(SLEEP_AKSHARE) + + # ----- summary ----- + if summary_rows: + summary = pd.DataFrame(summary_rows) + save_parquet(summary, OUT_DIR / "_summary.parquet", logger) + logger.info("=" * 60) + logger.info("SUMMARY:") + logger.info("\n" + summary.to_string(index=False)) + logger.info("=" * 60) + + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="历史成份股采集 (治幸存者偏差)") + parser.add_argument( + "--parts", + nargs="+", + default=["A", "B", "C"], + choices=["A", "B", "C"], + help="执行哪些部分 (A=深证/国证 adjust, B=csindex 快照, C=新浪交叉)", + ) + parser.add_argument("--skip-sina", action="store_true", help="跳过 C 新浪 (baostock 已有主源)") + args = parser.parse_args() + + logger, _ = setup_logging() + logger.info(f"args: parts={args.parts}, skip_sina={args.skip_sina}") + logger.info(f"CNI_ADJUST_INDICES={[c[0] for c in CNI_ADJUST_INDICES]}") + logger.info(f"CSINDEX_SNAPSHOT_INDICES={[c[0] for c in CSINDEX_SNAPSHOT_INDICES]}") + logger.info(f"SINA_CROSS_INDICES={[c[0] for c in SINA_CROSS_INDICES]}") + + try: + return run(args.parts, args.skip_sina, logger) + except KeyboardInterrupt: + logger.warning("interrupted by user") + return 1 + except Exception as e: + logger.exception(f"FATAL: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/data_platform/index_const_hist_wrapper.ps1 b/scripts/data_platform/index_const_hist_wrapper.ps1 new file mode 100644 index 0000000..eab2d4c --- /dev/null +++ b/scripts/data_platform/index_const_hist_wrapper.ps1 @@ -0,0 +1,31 @@ +# 历史成份股采集 wrapper (治幸存者偏差, P0 Task1) +# 环境变量覆盖脚本默认 Mac 路径 → Windows VPS 输出路径 +$ErrorActionPreference = 'Continue' +$ProgressPreference = 'SilentlyContinue' +$py = 'C:\Python310\python.exe' +$dir = 'C:\sanguo_vnpy_v2\scripts\data_platform' +$env:INDEX_HIST_OUT_DIR = 'C:\sanguo_vnpy_v2\data\index_const_hist' +$env:PYTHONIOENCODING = 'utf-8' +# unset proxy (直连, 避免数据源封 IP) +$env:http_proxy = $null +$env:https_proxy = $null +$env:all_proxy = $null +$wrapper_log = 'C:\sanguo_vnpy_v2\data\index_const_hist_wrapper.log' +$script_log = 'C:\sanguo_vnpy_v2\data\index_const_hist_download.log' + +function Log-W($m) { + "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $m" | Out-File -FilePath $wrapper_log -Append -Encoding UTF8 +} + +Log-W "=== WRAPPER START ===" +Log-W "INDEX_HIST_OUT_DIR=$($env:INDEX_HIST_OUT_DIR)" +Log-W "DiskFree_GB_start=$((Get-PSDrive C).Free / 1GB)" + +# 单进程串行跑 A (深证/国证 adjust) + B (中证1000/2000 快照) + C (新浪交叉校验) +Log-W "SCRIPT START" +& $py -X utf8 "$dir\index_const_hist_download.py" --parts A B C *>&1 | Tee-Object -FilePath $script_log +$script_exit = $LASTEXITCODE +Log-W "SCRIPT DONE exit=$script_exit" + +Log-W "DiskFree_GB_end=$((Get-PSDrive C).Free / 1GB)" +Log-W "=== WRAPPER EXIT script_exit=$script_exit ===" diff --git a/scripts/data_platform/merge_staging_to_data.py b/scripts/data_platform/merge_staging_to_data.py new file mode 100644 index 0000000..7755f7a --- /dev/null +++ b/scripts/data_platform/merge_staging_to_data.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""merge staging → data, 只复制 data 里不存在的文件(非破坏)。 +A 股 data 里已有 → 跳过保留(不被 staging 7/15 覆盖);ETF/基金 data 没有 → 复制全历史。 + +用法: + DRY-RUN(默认): set MERGE_DRY_RUN=1 && python merge_staging_to_data.py + ACTUAL: set MERGE_DRY_RUN=0 && python merge_staging_to_data.py +""" +import os +import shutil +import sys + +STAGING = r"C:\sanguo_vnpy_v2\data\_staging_xtdata" +DATA = r"C:\sanguo_vnpy_v2\data" + + +def main(): + dry = os.environ.get("MERGE_DRY_RUN", "1") == "1" + total = skip = copy = fail = 0 + new_by_kind = {"raw": 0, "qfq": 0} + for kind in ("raw", "qfq"): + s_root = os.path.join(STAGING, kind) + d_root = os.path.join(DATA, kind) + if not os.path.isdir(s_root): + continue + for year in os.listdir(s_root): + s_yr = os.path.join(s_root, year) + if not os.path.isdir(s_yr): + continue + d_yr = os.path.join(d_root, year) + for fname in os.listdir(s_yr): + if not fname.endswith("_daily.parquet"): + continue + total += 1 + s_file = os.path.join(s_yr, fname) + d_file = os.path.join(d_yr, fname) + if os.path.exists(d_file): + skip += 1 + continue + if dry: + copy += 1 + new_by_kind[kind] += 1 + continue + try: + os.makedirs(d_yr, exist_ok=True) + shutil.copy2(s_file, d_file) + copy += 1 + new_by_kind[kind] += 1 + except Exception as e: # noqa: BLE001 + fail += 1 + if fail <= 5: + print(f"copy err {s_file}: {e}", flush=True) + mode = "DRY-RUN" if dry else "ACTUAL" + print(f"[{mode}] total_staging={total} to_copy={copy} already_in_data(skip)={skip} fail={fail}", flush=True) + print(f"[{mode}] new raw={new_by_kind['raw']} new qfq={new_by_kind['qfq']}", flush=True) + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/probe_delisted_baostock.py b/scripts/data_platform/probe_delisted_baostock.py new file mode 100644 index 0000000..9b8bab2 --- /dev/null +++ b/scripts/data_platform/probe_delisted_baostock.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +""" +P0 Task3 退市股 K 线 baostock 接口探针(只读) +硬约束: +- 直连不走代理(baostock 服务端在境内) +- 单进程单登录串行(防黑名单) +- 只读探针:不灌库不下全量 +""" +import sys +import os +import io + +# Windows 控制台 utf-8 + 强制 flush +if sys.platform == "win32": + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace", write_through=True) + sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace", write_through=True) + +# 清代理(baostock 直连境内,走代理必挂) +for k in ("http_proxy", "https_proxy", "all_proxy", "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): + os.environ.pop(k, None) + +import baostock as bs +import pandas as pd +import time + +def section(title): + print("\n" + "=" * 60) + print(f"### {title}") + print("=" * 60, flush=True) + +def dump_rs(rs, label, max_rows=5): + """把 ResultData 读完并打印字段 + 前 N 行""" + rows = [] + while (rs.error_code == '0') & rs.next(): + rows.append(rs.get_row_data()) + df = pd.DataFrame(rows, columns=rs.fields) if rows else pd.DataFrame(columns=rs.fields) + print(f"[{label}] error={rs.error_code} {rs.error_msg} | fields={rs.fields} | rows={len(df)}") + if len(df) > 0: + print(f"[{label}] head({max_rows}):") + print(df.head(max_rows).to_string()) + return df + +# ============ 1. 登录 ============ +section("LOGIN") +t0 = time.time() +lg = bs.login() +print(f"login: error_code={lg.error_code} error_msg={lg.error_msg} elapsed={time.time()-t0:.2f}s", flush=True) +if lg.error_code != '0': + print("LOGIN_FAILED_ABORT") + sys.exit(1) + +# ============ 2. query_all_stock(某日全市场列表) ============ +section("query_all_stock day=2026-07-18") +rs = bs.query_all_stock(day="2026-07-18") +df_all = dump_rs(rs, "query_all_stock(2026-07-18)", max_rows=5) +# 看字段含不含 status / 退市日 +if len(df_all) > 0: + print("\n字段分析:") + print(f" columns = {list(df_all.columns)}") + print(f" has_status = {'status' in df_all.columns}") + print(f" has_outDate = {'outDate' in df_all.columns}") + # code 前缀分布 + if 'code' in df_all.columns: + df_all['prefix'] = df_all['code'].str.split('.').str[0] + print(f" code 前缀分布:\n{df_all['prefix'].value_counts().to_string()}") + # tradeStatus 分布(若有) + if 'tradeStatus' in df_all.columns: + print(f" tradeStatus 分布:\n{df_all['tradeStatus'].value_counts().to_string()}") + +# ============ 3. query_all_stock 不同日期对比(取旧日,看是否还能查到已退市股) ============ +section("query_all_stock day=2021-01-04(旧日,看是否含现已退市)") +rs_old = bs.query_all_stock(day="2021-01-04") +df_old = dump_rs(rs_old, "query_all_stock(2021-01-04)", max_rows=3) +# 对比两日 code 差集(2021 在但 2026 不在 = 期间退市的候选) +if len(df_all) > 0 and len(df_old) > 0 and 'code' in df_all.columns: + s_new = set(df_all['code']) + s_old = set(df_old['code']) + only_old = sorted(s_old - s_new) + only_new = sorted(s_new - s_old) + print(f"\n2021 有 / 2026 无(候选退市): {len(only_old)} 只 | 前 10 示例: {only_old[:10]}") + print(f"2026 有 / 2021 无(期间上市): {len(only_new)} 只 | 前 10 示例: {only_new[:10]}") + # 存下来供后面 K 线验证用 + candidate_delisted = only_old[:10] # 前 10 只候选退市股 +else: + candidate_delisted = [] + +# ============ 4. query_stock_basic(单只基本信息 —— 找 status + outDate 字段) ============ +section("query_stock_basic 字段探查(活跃股 + 已知退市股)") +# 先打活跃股(确认字段集) +active_codes = ["sh.600000", "sz.000001", "sz.000002"] +# 已知退市股候选(内存里的 + 常见历史退市) +# sh.600074 退市保千 / sh.600432 退市吉恩 / sz.002450 *ST康得新 / sz.000033 新都退 / sh.600632 *ST 神城 +# 另外从 only_old 候选里挑几只(2021 有 2026 无 = 确认退市) +delisted_candidates = ["sh.600074", "sh.600432", "sz.002450", "sz.000033", "sh.600632"] +# 如果上面候选 only_old 里有,优先用(那些是 baostock 自己承认 2021 存在过的) +probe_codes = active_codes + delisted_candidates +if candidate_delisted: + probe_codes = probe_codes + candidate_delisted[:5] + +found_delisted_samples = [] +for code in probe_codes: + rs2 = bs.query_stock_basic(code=code) + b = [] + while (rs2.error_code == '0') & rs2.next(): + b.append(rs2.get_row_data()) + df2 = pd.DataFrame(b, columns=rs2.fields) if b else pd.DataFrame(columns=rs2.fields) + print(f"\n--- query_stock_basic({code}) error={rs2.error_code} {rs2.error_msg}") + print(f" fields = {rs2.fields}") + if len(df2) > 0: + print(f" data = {df2.to_dict('records')}") + # 记录退市样本(outDate 非空 或 status=0) + rec = df2.iloc[0].to_dict() + status_val = str(rec.get('status', '')) + outdate_val = str(rec.get('outDate', '')) + if status_val == '0' or (outdate_val and outdate_val not in ('', 'nan', 'None')): + found_delisted_samples.append((code, rec)) + else: + print(" (empty)") + time.sleep(0.3) # 温柔一点 + +print(f"\n找到的退市样本: {len(found_delisted_samples)}") +for code, rec in found_delisted_samples: + print(f" {code}: status={rec.get('status')} outDate={rec.get('outDate')} type={rec.get('type')}") + +# ============ 5. 退市股 K 线验证 ============ +section("query_history_k_data_plus 退市股 K 线") +fields = "date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,peTTM,pbMRQ,psTTM,pcfNcfTTM,isST" + +# 优先用真实退市股(status=0),否则用候选 +kline_targets = [] +for code, rec in found_delisted_samples: + kline_targets.append((code, rec.get('outDate', ''))) +# 如果没找到 status=0 的退市股,直接用候选列表 +if not kline_targets: + for code in delisted_candidates + candidate_delisted[:3]: + kline_targets.append((code, '')) + +# 去重 +seen = set() +kline_targets_uniq = [] +for code, outdate in kline_targets: + if code not in seen: + seen.add(code) + kline_targets_uniq.append((code, outdate)) + +print(f"K 线验证标的({len(kline_targets_uniq)}): {kline_targets_uniq}") + +for code, outdate in kline_targets_uniq[:6]: + print(f"\n--- K 线 {code} (outDate={outdate}) ---") + rs4 = bs.query_history_k_data_plus( + code, fields, + start_date='2020-01-01', end_date='2026-07-18', + frequency="d", adjustflag="3" + ) + k = [] + while (rs4.error_code == '0') & rs4.next(): + k.append(rs4.get_row_data()) + df4 = pd.DataFrame(k, columns=rs4.fields) if k else pd.DataFrame(columns=rs4.fields) + print(f" error={rs4.error_code} {rs4.error_msg} | rows={len(df4)}") + if len(df4) > 0: + # 字符串转日期比大小 + try: + df4['date_dt'] = pd.to_datetime(df4['date'], errors='coerce') + maxd = df4['date_dt'].max() + mind = df4['date_dt'].min() + print(f" date range: {mind.date()} ~ {maxd.date()}") + print(f" tail(3):") + print(df4.drop(columns=['date_dt']).tail(3).to_string()) + except Exception as e: + print(f" date parse err: {e}") + print(df4.tail(3).to_string()) + time.sleep(0.3) + +# ============ 6. 近 5 年退市股数量估计(小抽样,不扫全量) ============ +section("近 5 年退市股数量估计(基于 query_all_stock 差集 + 候选 basic)") +if len(df_all) > 0 and len(df_old) > 0: + print(f"2021-01-04 全市场: {len(df_old)} 只") + print(f"2026-07-18 全市场: {len(df_all)} 只") + print(f"2021 在 2026 不在的差集(候选期间退市/暂停/更名): {len(candidate_delisted)} 只(实际 {len(set(df_old['code']) - set(df_all['code']))} 只)") + print(f"注:这差集是『上限』——含真退市 + 更名/合并/暂停 + 历史数据缺日等。精确退市数要逐只 query_stock_basic 看 status=0 + outDate。") + +# ============ 7. logout ============ +section("LOGOUT") +bs.logout() +print("DONE_PROBE") diff --git a/scripts/data_platform/probe_delisted_ext.py b/scripts/data_platform/probe_delisted_ext.py new file mode 100644 index 0000000..8715d0f --- /dev/null +++ b/scripts/data_platform/probe_delisted_ext.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +"""P0 Task3 补充探针:早期退市股 K 线完整性 + 全市场 status=0 数量抽样估计""" +import sys, os, io +if sys.platform == "win32": + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace", write_through=True) +for k in ("http_proxy","https_proxy","all_proxy","HTTP_PROXY","HTTPS_PROXY","ALL_PROXY"): + os.environ.pop(k, None) + +import baostock as bs +import pandas as pd +import time + +lg = bs.login() +print(f"login: {lg.error_code} {lg.error_msg}") +if lg.error_code != '0': + print("ABORT"); sys.exit(1) + +# === 1. 早期退市股用更早 start_date 验证 K 线完整性 === +print("\n=== A. 早期退市股 K 线(start_date=2010-01-01) ===") +fields = "date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,peTTM,pbMRQ,psTTM,pcfNcfTTM,isST" +early_delisted = [ + ("sh.600432", "2018-07-13"), + ("sz.000033", "2017-07-07"), + ("sh.600632", "2004-11-18"), +] +for code, outdate in early_delisted: + rs = bs.query_history_k_data_plus( + code, fields, + start_date='2010-01-01', end_date='2026-07-18', + frequency="d", adjustflag="3" + ) + k = [] + while (rs.error_code == '0') & rs.next(): + k.append(rs.get_row_data()) + df = pd.DataFrame(k, columns=rs.fields) if k else pd.DataFrame(columns=rs.fields) + if len(df) > 0: + df['date_dt'] = pd.to_datetime(df['date'], errors='coerce') + mind = df['date_dt'].min(); maxd = df['date_dt'].max() + print(f" {code} outDate={outdate}: rows={len(df)} range={mind.date()}~{maxd.date()} | maxdate==outDate? {str(maxd.date())==outdate}") + print(f" tail(2):") + print(df.drop(columns=['date_dt']).tail(2).to_string()) + else: + print(f" {code} outDate={outdate}: rows=0 err={rs.error_msg}") + time.sleep(0.3) + +# === 2. 全市场 status=0 退市股数量抽样估计 === +# 从 2021-01-04 全市场 code 列表 + 扩展号段扫描 status=0 +print("\n=== B. 全市场退市股数量估计 ===") +# 先用 query_all_stock 拿基准列表(2021-01-04) +rs0 = bs.query_all_stock(day="2021-01-04") +base_codes = [] +while (rs0.error_code == '0') & rs0.next(): + base_codes.append(rs0.get_row_data()[0]) +print(f" 2021-01-04 all_stock code 数: {len(base_codes)}") + +# 对这些 code 全部查 basic,统计 status=0 + outDate 分布 +# 但 4687 只全扫会耗费 ~4687 query,Mac 单登录串行 ~30 分钟 +# 改为抽样:每 10 只取 1 只,共约 470 只 +sample_codes = base_codes[::10] +print(f" 抽样步长=10, 抽样数: {len(sample_codes)}") + +status0_cnt = 0 +status0_samples = [] +ipo_cnt = 0 +for i, code in enumerate(sample_codes): + rs2 = bs.query_stock_basic(code=code) + b = [] + while (rs2.error_code == '0') & rs2.next(): + b.append(rs2.get_row_data()) + if b: + df2 = pd.DataFrame(b, columns=rs2.fields) + rec = df2.iloc[0].to_dict() + if str(rec.get('status','')) == '0': + status0_cnt += 1 + status0_samples.append(rec) + elif str(rec.get('status','')) == '1': + ipo_cnt += 1 + if (i+1) % 50 == 0: + print(f" progress {i+1}/{len(sample_codes)} | status0={status0_cnt} status1={ipo_cnt}", flush=True) + time.sleep(0.05) # 50ms 间隔 = 20 qps,温柔 + +print(f"\n 抽样结果: {len(sample_codes)} 只中 status=0 {status0_cnt} 只, status=1 {ipo_cnt} 只") +print(f" 退市率(抽样): {status0_cnt/len(sample_codes)*100:.2f}%") +print(f" 外推全市场(基于 2021-01-04 的 {len(base_codes)} 只): {int(status0_cnt/len(sample_codes)*len(base_codes))} 只 status=0") +print(f" 注:2021-01-04 当日已退市的不会出现在列表里,所以这是『2021-01-04 还在交易,但之后退市』的估计") +print(f" status=0 样本前 10:") +for rec in status0_samples[:10]: + print(f" {rec.get('code')} {rec.get('code_name')} ipoDate={rec.get('ipoDate')} outDate={rec.get('outDate')}") + +bs.logout() +print("DONE_PROBE_EXT") diff --git a/scripts/data_platform/probe_early_delisted_kline.py b/scripts/data_platform/probe_early_delisted_kline.py new file mode 100644 index 0000000..4a5eb2e --- /dev/null +++ b/scripts/data_platform/probe_early_delisted_kline.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +"""最小探针:早期退市股 K 线完整性(3 query,瞬时)""" +import sys, os, io +if sys.platform == "win32": + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace", write_through=True) +for k in ("http_proxy","https_proxy","all_proxy","HTTP_PROXY","HTTPS_PROXY","ALL_PROXY"): + os.environ.pop(k, None) + +import baostock as bs +import pandas as pd + +lg = bs.login() +print(f"login: {lg.error_code} {lg.error_msg}", flush=True) +if lg.error_code != '0': + sys.exit(1) + +fields = "date,code,open,high,low,close,volume,amount,turn,pctChg,peTTM,pbMRQ,isST" +early = [ + ("sh.600432", "2018-07-13"), + ("sz.000033", "2017-07-07"), + ("sh.600632", "2004-11-18"), +] +for code, outdate in early: + rs = bs.query_history_k_data_plus( + code, fields, + start_date='2010-01-01', end_date='2026-07-18', + frequency="d", adjustflag="3" + ) + k = [] + while (rs.error_code == '0') & rs.next(): + k.append(rs.get_row_data()) + df = pd.DataFrame(k, columns=rs.fields) if k else pd.DataFrame(columns=rs.fields) + if len(df) > 0: + print(f"{code} outDate={outdate}: rows={len(df)} range={df['date'].min()}~{df['date'].max()}", flush=True) + print(f" maxdate==outDate? {df['date'].max()==outdate} | tail:", flush=True) + print(df.tail(2).to_string(), flush=True) + else: + print(f"{code} outDate={outdate}: rows=0 err={rs.error_msg}", flush=True) + +bs.logout() +print("DONE", flush=True) diff --git a/scripts/data_platform/probe_etf.py b/scripts/data_platform/probe_etf.py new file mode 100644 index 0000000..2de1838 --- /dev/null +++ b/scripts/data_platform/probe_etf.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""ETF universe + 前复权日线探针(P0 Task2.1)。 +VPS 跑: C:\\Python310\\python.exe -X utf8 probe_etf.py +""" +import sys +from xtquant import xtdata as xd + + +def main(): + etf = xd.get_stock_list_in_sector("沪深ETF") or [] + fund = xd.get_stock_list_in_sector("沪深基金") or [] + a = xd.get_stock_list_in_sector("沪深A股") or [] + u = list(set(a + etf + fund)) + print(f"A={len(a)} ETF={len(etf)} fund={len(fund)} union={len(u)}") + print(f"sample ETF: {etf[:5]}") + print(f"sample fund: {fund[:5]}") + + # 抽样: 510300.SH(沪深300ETF) / 513050.SH(中概互联网ETF) / 159919.SZ(300ETF) + samples = ["510300.SH", "513050.SH", "159919.SZ"] + r = xd.get_market_data_ex([], samples, period="1d", + start_time="20240101", end_time="20260721", + dividend_type="front") + for sym in samples: + df = r.get(sym) if r else None + bars = 0 if df is None else len(df) + tail_close = None if df is None or not len(df) else float(df["close"].iloc[-1]) + head_date = None if df is None or not len(df) else str(df.index[0])[:8] + tail_date = None if df is None or not len(df) else str(df.index[-1])[:8] + nan_close = None if df is None else bool(df["close"].isnull().any()) + print(f"{sym}: bars={bars} date=[{head_date}~{tail_date}] tail_close={tail_close} nan_close={nan_close}") + + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/probe_etf_v2.py b/scripts/data_platform/probe_etf_v2.py new file mode 100644 index 0000000..8c14c88 --- /dev/null +++ b/scripts/data_platform/probe_etf_v2.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""ETF 探针 v2: 先 download 再读,对比 dividend_type,确认 ETF 在 sector 中。""" +import sys +from xtquant import xtdata as xd + + +def main(): + etf = xd.get_stock_list_in_sector("沪深ETF") or [] + fund = xd.get_stock_list_in_sector("沪深基金") or [] + a = xd.get_stock_list_in_sector("沪深A股") or [] + u = list(set(a + etf + fund)) + print(f"A={len(a)} ETF={len(etf)} fund={len(fund)} union={len(u)}") + + # 1) 检查目标 samples 是否在 universe + for sym in ("510300.SH", "513050.SH", "159919.SZ"): + print(f" {sym} in A={sym in a} in ETF={sym in etf} in fund={sym in fund}") + + # 2) 重叠分析: ETF 与 fund 是否相同 + overlap = set(etf) & set(fund) + only_etf = set(etf) - set(fund) + only_fund = set(fund) - set(etf) + print(f"overlap(ETF&fund)={len(overlap)} only_etf={len(only_etf)} only_fund={len(only_fund)}") + if only_etf: + print(f" only_etf sample: {list(only_etf)[:5]}") + if only_fund: + print(f" only_fund sample: {list(only_fund)[:5]}") + + # 3) 先 download 再读 + samples = ["510300.SH", "513050.SH", "159919.SZ"] + print(f"\n=== download_history_data(1d, 20240101~20260721) ===") + for sym in samples: + try: + n = xd.download_history_data(sym, "1d", "20240101", "20260721") + print(f" {sym} download returned: {n}") + except Exception as e: + print(f" {sym} download err: {e}") + + # 4) 读两种 dividend_type + for dt in ("front", "none"): + print(f"\n=== get_market_data_ex dividend_type={dt} ===") + r = xd.get_market_data_ex([], samples, period="1d", + start_time="20240101", end_time="20260721", + dividend_type=dt) + for sym in samples: + df = r.get(sym) if r else None + bars = 0 if df is None else len(df) + tail_close = None if df is None or not len(df) else float(df["close"].iloc[-1]) + head_date = None if df is None or not len(df) else str(df.index[0])[:8] + tail_date = None if df is None or not len(df) else str(df.index[-1])[:8] + print(f" {sym}: bars={bars} date=[{head_date}~{tail_date}] tail_close={tail_close}") + + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/test_baostock_daily_constituent_sample.py b/scripts/data_platform/test_baostock_daily_constituent_sample.py new file mode 100755 index 0000000..0018871 --- /dev/null +++ b/scripts/data_platform/test_baostock_daily_constituent_sample.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""小样本验证脚本: 测试日K线和成分股下载 (各 10 只/1 指数) + +验证项: + 1. 日K线 10 只: parquet 生成 + 字段齐全 + 行数合理 + 日期范围对 + 2. 成分股 1 指数 1 年: parquet 生成 + 成分股数量合理 + 3. query 计数器工作 + +用法: + python test_baostock_daily_constituent_sample.py +""" + +import subprocess +import sys +from pathlib import Path + +# 颜色输出 +GREEN = "\033[92m" +RED = "\033[91m" +RESET = "\033[0m" + + +def run_test(name: str, cmd: list, expected_checks: dict): + """运行测试脚本并验证结果""" + print(f"\n{'=' * 60}") + print(f"测试: {name}") + print(f"命令: {' '.join(cmd)}") + print(f"{'=' * 60}") + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode not in [0, 2]: # 0=完成, 2=断路器(可接受) + print(f"{RED}❌ 失败: returncode={result.returncode}{RESET}") + print("STDOUT:", result.stdout[-500:] if len(result.stdout) > 500 else result.stdout) + print("STDERR:", result.stderr[-500:] if len(result.stderr) > 500 else result.stderr) + return False + + print(f"{GREEN}✅ 完成: returncode={result.returncode}{RESET}") + + # 检查输出 + output = result.stdout + result.stderr + + for check_key, check_pattern in expected_checks.items(): + if check_pattern in output: + print(f"{GREEN}✅ {check_key}: 找到'{check_pattern}'{RESET}") + else: + print(f"{RED}❌ {check_key}: 未找到'{check_pattern}'{RESET}") + return False + + return True + + +def check_parquet_files(out_dir: Path, pattern: str, min_count: int): + """检查 parquet 文件生成""" + print(f"\n检查 parquet 文件: {out_dir}/{pattern}") + parquet_files = list(out_dir.rglob(pattern)) + + if len(parquet_files) >= min_count: + print(f"{GREEN}✅ 找到 {len(parquet_files)} 个 parquet 文件 (≥{min_count}){RESET}") + # 显示前几个文件 + for f in parquet_files[:3]: + print(f" - {f.name}") + return True + else: + print(f"{RED}❌ 只找到 {len(parquet_files)} 个 parquet 文件 (<{min_count}){RESET}") + return False + + +def check_parquet_fields(file_path: Path, required_fields: list): + """检查 parquet 字段齐全""" + import pandas as pd + + print(f"\n检查字段: {file_path}") + try: + df = pd.read_parquet(file_path) + missing_fields = [f for f in required_fields if f not in df.columns] + + if missing_fields: + print(f"{RED}❌ 缺失字段: {missing_fields}{RESET}") + print(f"实际字段: {list(df.columns)}") + return False + else: + print(f"{GREEN}✅ 字段齐全: {len(required_fields)} 个必选字段都在{RESET}") + print(f"字段列表: {list(df.columns)}") + return True + except Exception as e: + print(f"{RED}❌ 读 parquet 失败: {e}{RESET}") + return False + + +def check_parquet_rowcount(file_path: Path, min_rows: int, max_rows: int): + """检查 parquet 行数合理""" + import pandas as pd + + print(f"\n检查行数: {file_path}") + try: + df = pd.read_parquet(file_path) + row_count = len(df) + + if min_rows <= row_count <= max_rows: + print(f"{GREEN}✅ 行数合理: {row_count} 行 (期望 {min_rows}-{max_rows}){RESET}") + return True + else: + print(f"{RED}❌ 行数异常: {row_count} 行 (期望 {min_rows}-{max_rows}){RESET}") + return False + except Exception as e: + print(f"{RED}❌ 读 parquet 失败: {e}{RESET}") + return False + + +def main(): + """主测试流程""" + print(f"\n{'=' * 60}") + print("BaoStock 脚本小样本验证") + print(f"{'=' * 60}") + + tests_passed = 0 + tests_failed = 0 + + # ======================= 测试 1: 日K线 10 只 ======================= + print(f"\n{'#' * 60}") + print("# 测试 1: 日K线下载 (10 只股票)") + print(f"{'#' * 60}") + + daily_cmd = [ + sys.executable, + "scripts/data_platform/baostock_daily_fullmarket_download.py", + "--limit", "10", + "--start", "2020-01-01", + "--end", "2020-12-31" + ] + + daily_checks = { + "登录成功": "baostock 登录成功", + "全市场A股": "全市场 A 股", + "处理10只": "limit=10 截断", + "query计数": "query 总计:", + } + + if run_test("日K线下载", daily_cmd, daily_checks): + tests_passed += 1 + + # 检查 parquet 文件 + daily_out_dir = Path("data/daily_baostock") + if check_parquet_files(daily_out_dir, "*.parquet", 10): + tests_passed += 1 + + # 检查第一个文件的字段和行数 + first_parquet = list(daily_out_dir.glob("*.parquet"))[0] + daily_fields = "date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST".split(",") + + if check_parquet_fields(first_parquet, daily_fields): + tests_passed += 1 + + # 2020-01-01~2020-12-31 约 244 个交易日 + if check_parquet_rowcount(first_parquet, 200, 300): + tests_passed += 1 + else: + tests_failed += 1 + else: + tests_failed += 1 + else: + tests_failed += 1 + else: + tests_failed += 1 + + # ======================= 测试 2: 成分股 1 指数 1 年 ======================= + print(f"\n{'#' * 60}") + print("# 测试 2: 成分股下载 (1 指数 1 年)") + print(f"{'#' * 60}") + + constituent_cmd = [ + sys.executable, + "scripts/data_platform/baostock_constituent_download.py", + "--indices", "hs300", + "--start", "2020-01-01", + "--end", "2020-12-31" + ] + + constituent_checks = { + "登录成功": "baostock 登录成功", + "快照日期": "快照日期列表:", + "每周一": "每周一快照", + } + + if run_test("成分股下载", constituent_cmd, constituent_checks): + tests_passed += 1 + + # 检查 parquet 文件 + constituent_out_dir = Path("data/constituent_baostock") + if check_parquet_files(constituent_out_dir, "hs300_*.parquet", 50): # 2020年约52个周一 + tests_passed += 1 + else: + tests_failed += 1 + else: + tests_failed += 1 + + # ======================= 测试结果汇总 ======================= + print(f"\n{'=' * 60}") + print("测试结果汇总") + print(f"{'=' * 60}") + print(f"通过: {tests_passed}") + print(f"失败: {tests_failed}") + + if tests_failed == 0: + print(f"\n{GREEN}✅ 所有测试通过!{RESET}\n") + return 0 + else: + print(f"\n{RED}❌ 有 {tests_failed} 个测试失败{RESET}\n") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/data_platform/test_mootdx_depth.py b/scripts/data_platform/test_mootdx_depth.py new file mode 100644 index 0000000..cb2cc02 --- /dev/null +++ b/scripts/data_platform/test_mootdx_depth.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""mootdx 分钟K线历史深度测试 —— 周一开盘后跑(非交易时段通达信全服务器返空) + +目的:对比 miniQMT 模拟端 1m/5m/15m 统一只给 ~12 个月(2025-07-16 起), +看通达信公共行情服务器能给多深的分钟历史。 + +用法: + venv311/bin/python scripts/data_platform/test_mootdx_depth.py +结果写 scripts/data_platform/_mootdx_depth_result.txt 并打印。 + +注意: +- 频率值表(mootdx 0.11.7 实测): 0=5m 1=15m 2=30m 3=60m 4=日 8=1分钟 9=日线 +- bars 返回【不复权】原始价;offset 硬上限 800,更深历史靠 start 分页 +- 非交易时段(收盘后/周末)通达信服务器 quotes+bars 全频率返空,连日线都不给 +""" +import socket +import sys +import datetime +from mootdx.quotes import Quotes + +_TDX_SERVERS = [ + ('119.97.185.59', 7709), ('124.70.133.119', 7709), ('116.205.183.150', 7709), + ('123.60.73.44', 7709), ('116.205.163.254', 7709), ('121.36.225.169', 7709), + ('123.60.70.228', 7709), ('124.71.9.153', 7709), ('110.41.147.114', 7709), + ('124.71.187.122', 7709), +] + + +def _probe(ip, port, timeout=2.0): + try: + with socket.create_connection((ip, port), timeout=timeout): + return True + except Exception: + return False + + +def _n(x): + """统一求长度, 规避空 DataFrame 的 bool 歧义坑""" + if x is None: + return 0 + try: + return len(x) + except Exception: + return 0 + + +def find_server(symbol='600519'): + """遍历服务器, 返回第一个能返回日线 bars 的(交易日内才有)""" + for ip, port in _TDX_SERVERS: + if not _probe(ip, port): + continue + try: + c = Quotes.factory(market='std', server=(ip, port)) + d = c.bars(symbol=symbol, frequency=9, offset=5) + if _n(d) > 0: + return ip, c + except Exception: + pass + return None, None + + +def test_depth(c, symbol, freq, name, max_pages=200): + """start 分页翻到底, 找最早/最新 datetime. 200页: 1m≈20月/5m≈8年/15m≈24年""" + start = 0 + total = 0 + pages = 0 + earliest = None + latest = None + while pages < max_pages: + try: + df = c.bars(symbol=symbol, frequency=freq, offset=800, start=start) + except Exception: + break + n = _n(df) + if n == 0: + break + total += n + try: + ft = str(df.iloc[0]['datetime']) + lt = str(df.iloc[-1]['datetime']) + if earliest is None or ft < earliest: + earliest = ft + if latest is None or lt > latest: + latest = lt + except Exception: + pass + if n < 800: + break + start += n + pages += 1 + return name, total, earliest, latest, pages + + +def main(): + out = ['mootdx 深度测试 @ %s' % datetime.datetime.now()] + ip, c = find_server() + if c is None: + out.append('!!! 没有服务器返回日线数据 —— 非交易时段(周末/收盘后)通达信全服务器返空') + out.append('!!! 请周一 09:30 开盘后重跑此脚本') + msg = '\n'.join(out) + print(msg) + with open('_mootdx_depth_result.txt', 'w') as f: + f.write(msg) + sys.exit(1) + + out.append('server: %s' % ip) + out.append('') + for sym in ['600519']: # 茅台(2001上市, 老股, 测深度上限最佳) + out.append('=== %s (茅台) ===' % sym) + for fr, nm in [(8, '1分钟'), (0, '5分钟'), (1, '15分钟')]: + name, total, earliest, latest, pages = test_depth(c, sym, fr, nm) + out.append(' %-6s: %7d 根 | 最早=%s | 最新=%s | 翻%d页' + % (name, total, earliest, latest, pages)) + out.append('') + out.append('对比: miniQMT 模拟端 1m/5m/15m 统一 ~12 个月(2025-07-16 起)') + msg = '\n'.join(out) + print(msg) + with open('_mootdx_depth_result.txt', 'w') as f: + f.write(msg) + + +if __name__ == '__main__': + main() diff --git a/scripts/data_platform/verify_delisted_in_db.py b/scripts/data_platform/verify_delisted_in_db.py new file mode 100644 index 0000000..53677f7 --- /dev/null +++ b/scripts/data_platform/verify_delisted_in_db.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""验证退市股是否早就在 daily_baostock_full(Task3 import +0 的根因确认)。""" +import sqlite3 + +DB = r"C:\sanguo_vnpy_v2\data\quant_trading.db" +c = sqlite3.connect(DB, timeout=60) +# 总况 +total = c.execute("SELECT COUNT(*), COUNT(DISTINCT symbol||exchange) FROM daily_baostock_full").fetchone() +print(f"TOTAL: rows={total[0]} distinct(sym+exc)={total[1]}") +# 抽样退市股(Task3 下过的) +for sym, exc in [("000005", "SZ"), ("000023", "SZ"), ("600811", "SH"), ("600074", "SH"), ("000004", "SZ")]: + r = c.execute( + "SELECT COUNT(*), MIN(date), MAX(date) FROM daily_baostock_full WHERE symbol=? AND exchange=?", + (sym, exc), + ).fetchone() + print(f" {sym}.{exc}: rows={r[0]} date {r[1]} ~ {r[2]}") +c.close() diff --git a/scripts/data_platform/verify_etf_data.py b/scripts/data_platform/verify_etf_data.py new file mode 100644 index 0000000..2ebf009 --- /dev/null +++ b/scripts/data_platform/verify_etf_data.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""验证主库 data(非 staging) 的 ETF + A股 max_date。 +检查: sh510300/sh513050/sz159919 max_date==2026-07-21;sh600519 max_date==2026-07-21(未被 staging 覆盖)。 +""" +import os +import sys +import pandas as pd + +DATA = r"C:\sanguo_vnpy_v2\data" + + +def check(sym_prefix, kind="qfq"): + """逐年合并所有年份的 parquet。""" + root = os.path.join(DATA, kind) + if not os.path.isdir(root): + return {"err": f"no {kind} dir"} + all_dfs = [] + for yr in sorted(os.listdir(root)): + if not yr.isdigit(): + continue + p = os.path.join(root, yr, f"{sym_prefix}_daily.parquet") + if os.path.exists(p): + all_dfs.append(pd.read_parquet(p)) + if not all_dfs: + return {"err": "not found"} + df = pd.concat(all_dfs).sort_values("date") + return { + "rows": len(df), + "min_date": str(df["date"].min().date()), + "max_date": str(df["date"].max().date()), + "tail_close": float(df["close"].iloc[-1]), + "nan_close": bool(df["close"].isnull().any()), + } + + +def main(): + print("=== ETF 抽样 (qfq) ===") + for sym in ("sh510300", "sh513050", "sz159919"): + print(f" {sym}: {check(sym, 'qfq')}") + print("\n=== A 股对照 (qfq, 验证未被 staging 覆盖) ===") + print(f" sh600519: {check('sh600519', 'qfq')}") + print(f"\n=== raw 同抽样(完整性对照) ===") + print(f" sh510300 raw: {check('sh510300', 'raw')}") + print(f" sh600519 raw: {check('sh600519', 'raw')}") + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/data_platform/verify_etf_staging.py b/scripts/data_platform/verify_etf_staging.py new file mode 100644 index 0000000..525c176 --- /dev/null +++ b/scripts/data_platform/verify_etf_staging.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""验证 ETF staging parquet (P0 Task2.4)。 +检查: 股票/ETF/fund 计数, 抽样 close 非 NaN, 日期范围, 行数。 + +VPS 跑: C:\\Python310\\python.exe -X utf8 verify_etf_staging.py +""" +import os +import glob +import sys +import pandas as pd + +STAGING = r"C:\sanguo_vnpy_v2\data\_staging_xtdata" + +def count_symbols(kind): + """kind='qfq' or 'raw' — count distinct symbol files across years.""" + pattern = os.path.join(STAGING, kind, "*", "*_daily.parquet") + files = glob.glob(pattern) + syms = set(os.path.basename(f).replace("_daily.parquet", "") for f in files) + return len(syms), files + +def sample_check(sym_prefix, kind="qfq"): + """读最新年份 parquet 看尾部 close.""" + # 逐年合并所有年份的 parquet + years = sorted(d for d in os.listdir(os.path.join(STAGING, kind)) + if d.isdigit()) if os.path.exists(os.path.join(STAGING, kind)) else [] + if not years: + return None + all_dfs = [] + for yr in years: + p = os.path.join(STAGING, kind, yr, f"{sym_prefix}_daily.parquet") + if os.path.exists(p): + df = pd.read_parquet(p) + all_dfs.append(df) + if not all_dfs: + return None + df = pd.concat(all_dfs).sort_values("date") + return { + "rows": len(df), + "min_date": str(df["date"].min().date()), + "max_date": str(df["date"].max().date()), + "tail_close": float(df["close"].iloc[-1]), + "nan_close": bool(df["close"].isnull().any()), + } + +def main(): + for kind in ("raw", "qfq"): + n, files = count_symbols(kind) + print(f"=== {kind}: {n} distinct symbols, {len(files)} year-files ===") + if not n: + continue + # 分类: sh* sz* 前缀 + sh = sum(1 for f in files if os.path.basename(f).startswith("sh")) + sz = sum(1 for f in files if os.path.basename(f).startswith("sz")) + print(f" sh-prefix: {sh} sz-prefix: {sz}") + # 总行数估算(抽样第一个年份目录) + years = sorted(set(os.path.dirname(f).split(os.sep)[-1] for f in files)) + print(f" years: {years[0]}~{years[-1]} ({len(years)} years)") + + print("\n=== 抽样 ETF (qfq) ===") + for sym_prefix in ("sh510300", "sh513050", "sz159919"): + info = sample_check(sym_prefix, "qfq") + if info is None: + print(f" {sym_prefix}: NOT FOUND") + else: + print(f" {sym_prefix}: {info}") + + print("\n=== 抽样 A 股对照 (sh600519 茅台 qfq) ===") + info = sample_check("sh600519", "qfq") + print(f" sh600519: {info}") + + sys.stdout.flush() + +if __name__ == "__main__": + main()