From e91b103f7a0fd950571f1ef41799f62a344e296a Mon Sep 17 00:00:00 2001 From: claude_dev Date: Wed, 29 Jul 2026 11:56:44 +0800 Subject: [PATCH] =?UTF-8?q?docs(data):=20Phase2=20=E5=BD=92=E6=A1=A3?= =?UTF-8?q?=E6=97=A7=E5=9B=9E=E5=A1=AB/import=E9=93=BE+15m=E7=81=8C?= =?UTF-8?q?=E5=BA=93=E9=93=BE(VPS=20schtask=E5=AE=9E=E8=AF=81=E7=BA=BF?= =?UTF-8?q?=E5=A4=B4=E6=AD=BB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VPS schtasks /query: 调用这些脚本的 schtask 全已禁用(baostock_day1/2, xtdata-build, index-hist, bs-daily-increment 等),且不被8个活跃wrapper引用。 活跃白名单(保留): bs_eod/xt_eod/sina_index_eod/akshare_static_download/ index_const_hist_download/merge_constituent/migrate_constituent/parse_csindex_announce - _archive/backfill_legacy/: 31个(旧回填import链+旧baostock下载+禁用wrapper+Mac .sh链) - _archive/backfill_15m/: 7个(15min灌库链+hardening测试,backfill已完成) - 关键实证: index_const_hist_download 被 index_monthly_wrapper(活跃)调用→保留不归档 - 无残留import(fetch_with_fallback是sanguo_data函数非fallback模块) --- scripts/data_platform/_archive/README.md | 31 ++++++++++++++++--- .../backfill_15m}/backfill_15min_baostock.py | 0 .../baostock_15min_fullmarket_download.py | 0 .../baostock_15min_sample_to_db.py | 0 .../backfill_15m}/download_15m_xtdata.py | 0 .../backfill_15m}/download_minute.py | 0 .../backfill_15m}/refresh_15min_daily.py | 0 .../test_backfill_15min_hardening.py | 0 .../backfill_legacy}/_launch_15m.bat | 0 .../backfill_legacy}/_run_build.ps1 | 0 .../backfill_legacy}/_run_daily.ps1 | 0 .../backfill_legacy}/_run_import.ps1 | 0 .../baostock_constituent_download.py | 0 .../baostock_daily_fullmarket_download.py | 0 .../baostock_day1_wrapper.ps1 | 0 .../baostock_day2a_wrapper.ps1 | 0 .../baostock_day2b_wrapper.ps1 | 0 .../baostock_delisted_download.py | 0 .../backfill_legacy}/baostock_download.py | 0 .../baostock_static_download.py | 0 .../build_daily_from_xtdata.py | 0 .../backfill_legacy}/daily_all_update.py | 0 .../backfill_legacy}/daily_update.sh | 0 .../backfill_legacy}/daily_update_all.sh | 0 .../daily_update_static_wrapper.ps1 | 0 .../backfill_legacy}/full_deploy_5yr.sh | 0 .../backfill_legacy}/import_baostock_to_db.py | 0 .../backfill_legacy}/import_bs_wrapper.ps1 | 0 .../backfill_legacy}/import_delisted_to_db.py | 0 .../backfill_legacy}/import_vnpy_daily.py | 0 .../import_vnpy_daily_fast.py | 0 .../backfill_legacy}/import_vnpy_minute.py | 0 .../import_vnpy_minute_fast.py | 0 .../index_const_hist_wrapper.ps1 | 0 .../backfill_legacy}/merge_daily_baostock.py | 0 .../migrate_daily_baostock.py | 0 .../backfill_legacy}/raw_redownload.py | 0 .../backfill_legacy}/relaunch_15m_wrapper.ps1 | 0 .../backfill_legacy}/run_daily_update.sh | 0 39 files changed, 27 insertions(+), 4 deletions(-) rename scripts/data_platform/{ => _archive/backfill_15m}/backfill_15min_baostock.py (100%) rename scripts/data_platform/{ => _archive/backfill_15m}/baostock_15min_fullmarket_download.py (100%) rename scripts/data_platform/{ => _archive/backfill_15m}/baostock_15min_sample_to_db.py (100%) rename scripts/data_platform/{ => _archive/backfill_15m}/download_15m_xtdata.py (100%) rename scripts/data_platform/{ => _archive/backfill_15m}/download_minute.py (100%) rename scripts/data_platform/{ => _archive/backfill_15m}/refresh_15min_daily.py (100%) rename {tests/data => scripts/data_platform/_archive/backfill_15m}/test_backfill_15min_hardening.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/_launch_15m.bat (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/_run_build.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/_run_daily.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/_run_import.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_constituent_download.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_daily_fullmarket_download.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_day1_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_day2a_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_day2b_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_delisted_download.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_download.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/baostock_static_download.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/build_daily_from_xtdata.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/daily_all_update.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/daily_update.sh (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/daily_update_all.sh (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/daily_update_static_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/full_deploy_5yr.sh (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_baostock_to_db.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_bs_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_delisted_to_db.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_vnpy_daily.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_vnpy_daily_fast.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_vnpy_minute.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/import_vnpy_minute_fast.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/index_const_hist_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/merge_daily_baostock.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/migrate_daily_baostock.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/raw_redownload.py (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/relaunch_15m_wrapper.ps1 (100%) rename scripts/data_platform/{ => _archive/backfill_legacy}/run_daily_update.sh (100%) diff --git a/scripts/data_platform/_archive/README.md b/scripts/data_platform/_archive/README.md index 27a4ae1..dfe9b4d 100644 --- a/scripts/data_platform/_archive/README.md +++ b/scripts/data_platform/_archive/README.md @@ -16,8 +16,31 @@ 归档前已验证：**零 import、无活跃 wrapper 引用**。 -## backfill_15m/ — 15min 一次性灌库链（Phase 2 待归档） +## backfill_legacy/ — 旧回填/import 链 + 禁用 wrapper + Mac .sh 链（2026-07-29 归档） -⚠️ 未归档。`backfill_15min_baostock` 被 `tests/data/test_backfill_15min_hardening.py` 正式 import， -`refresh_15min_daily` / `download_minute` / `download_15m_xtdata` / `raw_redownload` / `audit_data_layout` -存在交叉引用或 ops wrapper 依赖，需 Phase 2 评估后统一处理。 +31 个文件。方案 A 后由 bs_eod/xt_eod 接管的旧路径。**VPS schtasks /query 实证**：调用这些脚本的 +schtask（baostock_day1/day2、xtdata-build、index-hist、bs-daily-increment、bs-akshare、bs-static、 +bs15min、daily-update 等）**全部已禁用**，且不被 8 个活跃 wrapper 引用。 + +- 旧回填/import：`build_daily_from_xtdata` / `import_vnpy_*` / `import_baostock_to_db` / `merge_daily_baostock` / `migrate_daily_baostock` / `raw_redownload` / `daily_all_update` +- 旧 baostock 下载：`baostock_download` / `baostock_constituent_download` / `baostock_static_download` / `baostock_delisted_download` / `baostock_daily_fullmarket_download` +- 禁用 wrapper：`baostock_day1/day2a/day2b_wrapper` / `import_bs_wrapper` / `daily_update_static_wrapper` / `index_const_hist_wrapper` / `_run_import` / `_run_build` / `_run_daily` / `relaunch_15m_wrapper` +- Mac 旧 .sh 链：`daily_update.sh` / `daily_update_all.sh` / `run_daily_update.sh` / `full_deploy_5yr.sh` / `_launch_15m.bat` + +## backfill_15m/ — 15min 一次性灌库链 + 测试（2026-07-29 归档） + +7 个文件。15min 全市场灌库已于 2026-07-20 完成，方案 A 后由 `sanguo-bs-eod` 每日增量接管。 +`backfill_15min_baostock` + `baostock_15min_fullmarket_download` + `baostock_15min_sample_to_db` + +`refresh_15min_daily` + `download_minute` + `download_15m_xtdata` + `test_backfill_15min_hardening` +（hardening 测试随被测脚本一并归档，backfill 不再执行）。 + +## ⚠️ 归档时关键实证 + +`index_const_hist_download.py` 被 `index_monthly_wrapper.ps1`（sanguo-index 月度，**活跃**）调用 +→ **保留在顶层，不归档**。归档前须查活跃 wrapper 调用集，勿凭"一次性脚本"直觉判断。 + +## 保留在顶层（活跃生产，不归档） + +`bs_eod` / `xt_eod` / `sina_index_eod` / `akshare_static_download` / `index_const_hist_download` / +`merge_constituent` / `migrate_constituent` / `parse_csindex_announce` / `validator` / `validate_import` / +`verify_*` / `cleanup_staging` / `backup_db` / `dbbardata_utils` / 活跃 wrapper / `register_schtasks` / ops 工具 diff --git a/scripts/data_platform/backfill_15min_baostock.py b/scripts/data_platform/_archive/backfill_15m/backfill_15min_baostock.py similarity index 100% rename from scripts/data_platform/backfill_15min_baostock.py rename to scripts/data_platform/_archive/backfill_15m/backfill_15min_baostock.py diff --git a/scripts/data_platform/baostock_15min_fullmarket_download.py b/scripts/data_platform/_archive/backfill_15m/baostock_15min_fullmarket_download.py similarity index 100% rename from scripts/data_platform/baostock_15min_fullmarket_download.py rename to scripts/data_platform/_archive/backfill_15m/baostock_15min_fullmarket_download.py diff --git a/scripts/data_platform/baostock_15min_sample_to_db.py b/scripts/data_platform/_archive/backfill_15m/baostock_15min_sample_to_db.py similarity index 100% rename from scripts/data_platform/baostock_15min_sample_to_db.py rename to scripts/data_platform/_archive/backfill_15m/baostock_15min_sample_to_db.py diff --git a/scripts/data_platform/download_15m_xtdata.py b/scripts/data_platform/_archive/backfill_15m/download_15m_xtdata.py similarity index 100% rename from scripts/data_platform/download_15m_xtdata.py rename to scripts/data_platform/_archive/backfill_15m/download_15m_xtdata.py diff --git a/scripts/data_platform/download_minute.py b/scripts/data_platform/_archive/backfill_15m/download_minute.py similarity index 100% rename from scripts/data_platform/download_minute.py rename to scripts/data_platform/_archive/backfill_15m/download_minute.py diff --git a/scripts/data_platform/refresh_15min_daily.py b/scripts/data_platform/_archive/backfill_15m/refresh_15min_daily.py similarity index 100% rename from scripts/data_platform/refresh_15min_daily.py rename to scripts/data_platform/_archive/backfill_15m/refresh_15min_daily.py diff --git a/tests/data/test_backfill_15min_hardening.py b/scripts/data_platform/_archive/backfill_15m/test_backfill_15min_hardening.py similarity index 100% rename from tests/data/test_backfill_15min_hardening.py rename to scripts/data_platform/_archive/backfill_15m/test_backfill_15min_hardening.py diff --git a/scripts/data_platform/_launch_15m.bat b/scripts/data_platform/_archive/backfill_legacy/_launch_15m.bat similarity index 100% rename from scripts/data_platform/_launch_15m.bat rename to scripts/data_platform/_archive/backfill_legacy/_launch_15m.bat diff --git a/scripts/data_platform/_run_build.ps1 b/scripts/data_platform/_archive/backfill_legacy/_run_build.ps1 similarity index 100% rename from scripts/data_platform/_run_build.ps1 rename to scripts/data_platform/_archive/backfill_legacy/_run_build.ps1 diff --git a/scripts/data_platform/_run_daily.ps1 b/scripts/data_platform/_archive/backfill_legacy/_run_daily.ps1 similarity index 100% rename from scripts/data_platform/_run_daily.ps1 rename to scripts/data_platform/_archive/backfill_legacy/_run_daily.ps1 diff --git a/scripts/data_platform/_run_import.ps1 b/scripts/data_platform/_archive/backfill_legacy/_run_import.ps1 similarity index 100% rename from scripts/data_platform/_run_import.ps1 rename to scripts/data_platform/_archive/backfill_legacy/_run_import.ps1 diff --git a/scripts/data_platform/baostock_constituent_download.py b/scripts/data_platform/_archive/backfill_legacy/baostock_constituent_download.py similarity index 100% rename from scripts/data_platform/baostock_constituent_download.py rename to scripts/data_platform/_archive/backfill_legacy/baostock_constituent_download.py diff --git a/scripts/data_platform/baostock_daily_fullmarket_download.py b/scripts/data_platform/_archive/backfill_legacy/baostock_daily_fullmarket_download.py similarity index 100% rename from scripts/data_platform/baostock_daily_fullmarket_download.py rename to scripts/data_platform/_archive/backfill_legacy/baostock_daily_fullmarket_download.py diff --git a/scripts/data_platform/baostock_day1_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/baostock_day1_wrapper.ps1 similarity index 100% rename from scripts/data_platform/baostock_day1_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/baostock_day1_wrapper.ps1 diff --git a/scripts/data_platform/baostock_day2a_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/baostock_day2a_wrapper.ps1 similarity index 100% rename from scripts/data_platform/baostock_day2a_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/baostock_day2a_wrapper.ps1 diff --git a/scripts/data_platform/baostock_day2b_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/baostock_day2b_wrapper.ps1 similarity index 100% rename from scripts/data_platform/baostock_day2b_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/baostock_day2b_wrapper.ps1 diff --git a/scripts/data_platform/baostock_delisted_download.py b/scripts/data_platform/_archive/backfill_legacy/baostock_delisted_download.py similarity index 100% rename from scripts/data_platform/baostock_delisted_download.py rename to scripts/data_platform/_archive/backfill_legacy/baostock_delisted_download.py diff --git a/scripts/data_platform/baostock_download.py b/scripts/data_platform/_archive/backfill_legacy/baostock_download.py similarity index 100% rename from scripts/data_platform/baostock_download.py rename to scripts/data_platform/_archive/backfill_legacy/baostock_download.py diff --git a/scripts/data_platform/baostock_static_download.py b/scripts/data_platform/_archive/backfill_legacy/baostock_static_download.py similarity index 100% rename from scripts/data_platform/baostock_static_download.py rename to scripts/data_platform/_archive/backfill_legacy/baostock_static_download.py diff --git a/scripts/data_platform/build_daily_from_xtdata.py b/scripts/data_platform/_archive/backfill_legacy/build_daily_from_xtdata.py similarity index 100% rename from scripts/data_platform/build_daily_from_xtdata.py rename to scripts/data_platform/_archive/backfill_legacy/build_daily_from_xtdata.py diff --git a/scripts/data_platform/daily_all_update.py b/scripts/data_platform/_archive/backfill_legacy/daily_all_update.py similarity index 100% rename from scripts/data_platform/daily_all_update.py rename to scripts/data_platform/_archive/backfill_legacy/daily_all_update.py diff --git a/scripts/data_platform/daily_update.sh b/scripts/data_platform/_archive/backfill_legacy/daily_update.sh similarity index 100% rename from scripts/data_platform/daily_update.sh rename to scripts/data_platform/_archive/backfill_legacy/daily_update.sh diff --git a/scripts/data_platform/daily_update_all.sh b/scripts/data_platform/_archive/backfill_legacy/daily_update_all.sh similarity index 100% rename from scripts/data_platform/daily_update_all.sh rename to scripts/data_platform/_archive/backfill_legacy/daily_update_all.sh diff --git a/scripts/data_platform/daily_update_static_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/daily_update_static_wrapper.ps1 similarity index 100% rename from scripts/data_platform/daily_update_static_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/daily_update_static_wrapper.ps1 diff --git a/scripts/data_platform/full_deploy_5yr.sh b/scripts/data_platform/_archive/backfill_legacy/full_deploy_5yr.sh similarity index 100% rename from scripts/data_platform/full_deploy_5yr.sh rename to scripts/data_platform/_archive/backfill_legacy/full_deploy_5yr.sh diff --git a/scripts/data_platform/import_baostock_to_db.py b/scripts/data_platform/_archive/backfill_legacy/import_baostock_to_db.py similarity index 100% rename from scripts/data_platform/import_baostock_to_db.py rename to scripts/data_platform/_archive/backfill_legacy/import_baostock_to_db.py diff --git a/scripts/data_platform/import_bs_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/import_bs_wrapper.ps1 similarity index 100% rename from scripts/data_platform/import_bs_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/import_bs_wrapper.ps1 diff --git a/scripts/data_platform/import_delisted_to_db.py b/scripts/data_platform/_archive/backfill_legacy/import_delisted_to_db.py similarity index 100% rename from scripts/data_platform/import_delisted_to_db.py rename to scripts/data_platform/_archive/backfill_legacy/import_delisted_to_db.py diff --git a/scripts/data_platform/import_vnpy_daily.py b/scripts/data_platform/_archive/backfill_legacy/import_vnpy_daily.py similarity index 100% rename from scripts/data_platform/import_vnpy_daily.py rename to scripts/data_platform/_archive/backfill_legacy/import_vnpy_daily.py diff --git a/scripts/data_platform/import_vnpy_daily_fast.py b/scripts/data_platform/_archive/backfill_legacy/import_vnpy_daily_fast.py similarity index 100% rename from scripts/data_platform/import_vnpy_daily_fast.py rename to scripts/data_platform/_archive/backfill_legacy/import_vnpy_daily_fast.py diff --git a/scripts/data_platform/import_vnpy_minute.py b/scripts/data_platform/_archive/backfill_legacy/import_vnpy_minute.py similarity index 100% rename from scripts/data_platform/import_vnpy_minute.py rename to scripts/data_platform/_archive/backfill_legacy/import_vnpy_minute.py diff --git a/scripts/data_platform/import_vnpy_minute_fast.py b/scripts/data_platform/_archive/backfill_legacy/import_vnpy_minute_fast.py similarity index 100% rename from scripts/data_platform/import_vnpy_minute_fast.py rename to scripts/data_platform/_archive/backfill_legacy/import_vnpy_minute_fast.py diff --git a/scripts/data_platform/index_const_hist_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/index_const_hist_wrapper.ps1 similarity index 100% rename from scripts/data_platform/index_const_hist_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/index_const_hist_wrapper.ps1 diff --git a/scripts/data_platform/merge_daily_baostock.py b/scripts/data_platform/_archive/backfill_legacy/merge_daily_baostock.py similarity index 100% rename from scripts/data_platform/merge_daily_baostock.py rename to scripts/data_platform/_archive/backfill_legacy/merge_daily_baostock.py diff --git a/scripts/data_platform/migrate_daily_baostock.py b/scripts/data_platform/_archive/backfill_legacy/migrate_daily_baostock.py similarity index 100% rename from scripts/data_platform/migrate_daily_baostock.py rename to scripts/data_platform/_archive/backfill_legacy/migrate_daily_baostock.py diff --git a/scripts/data_platform/raw_redownload.py b/scripts/data_platform/_archive/backfill_legacy/raw_redownload.py similarity index 100% rename from scripts/data_platform/raw_redownload.py rename to scripts/data_platform/_archive/backfill_legacy/raw_redownload.py diff --git a/scripts/data_platform/relaunch_15m_wrapper.ps1 b/scripts/data_platform/_archive/backfill_legacy/relaunch_15m_wrapper.ps1 similarity index 100% rename from scripts/data_platform/relaunch_15m_wrapper.ps1 rename to scripts/data_platform/_archive/backfill_legacy/relaunch_15m_wrapper.ps1 diff --git a/scripts/data_platform/run_daily_update.sh b/scripts/data_platform/_archive/backfill_legacy/run_daily_update.sh similarity index 100% rename from scripts/data_platform/run_daily_update.sh rename to scripts/data_platform/_archive/backfill_legacy/run_daily_update.sh