refactor(data): 5m回灌改每日分片——用户拍板2026-08-20'每天固定下一些',废弃--full一趟灌满——①每日一跑=回灌片(若有)+每日增量(回灌期也跑,近端7天始终新鲜)②片=半年窗口newest-first(近端先到先可用,回测最常用的近期数据第一天就有),state(bs_5m_state.json next_end)逐片向2020-01-03推进;整片扫完才推进,断点/达限次日重拉同片(OR REPLACE幂等)③全程~14片≈两周补完,每片~5.5k query/~3200万行/3-4h;片+增量每日~11k query,NAS IP独立48k配额内④DAILY_LIMIT守卫改跨窗口累计计数(swept);删FULL_TIMEOUT(半年片5.8k行/股,60s超时足够);+3测试(首片=最近半年/逐片向2020走+完结盖章/损坏state重头),main测试改双sweep断言(片+增量4调用/完结后单sweep2调用);data_platform 158绿 [nas]
CI/CD / test (push) Successful in 18s
CI/CD / nas-deploy (push) Successful in 36s
CI/CD / nas-verify (push) Successful in 17s

This commit is contained in:
2026-08-20 10:42:34 +08:00
parent 20ff70e85d
commit 9fa2f933ee
2 changed files with 149 additions and 56 deletions
+40 -6
View File
@@ -10,6 +10,7 @@ INSERT OR IGNORE 去重(merge_increment.py), NAS 回测 reader 零改动可见 5
--full 全区间也一样); VPS 侧 bs_eod/bs_fund 用 VPS 自己的 IP 配额, 互不相干。
"""
import datetime as dt
import json
import os
import sqlite3
import sys
@@ -138,13 +139,32 @@ def test_process_one_success_writes(tmp_db_conn):
assert n == 1
# ---------- 窗口 ----------
# ---------- 回灌分片(newest-first, 每天固定一片) ----------
def test_calc_window_full_vs_daily():
def test_next_chunk_first_is_recent_half_year(tmp_path, monkeypatch):
"""无 state → 首片 = 最近半年(today-183, today], 近端先到先可用。"""
monkeypatch.setattr(b5, "DB", tmp_path / "dbbardata_5m.db")
today = dt.date(2026, 8, 20)
assert b5.calc_window(today, full=True) == ("2020-01-03", "2026-08-20")
s, e = b5.calc_window(today, full=False)
assert s == "2026-08-13" and e == "2026-08-20" # LOOKBACK=7
assert b5.next_chunk(today) == ("2026-02-18", "2026-08-20")
def test_next_chunk_walks_backward_and_finishes(tmp_path, monkeypatch):
"""state 逐片向 2020 走; 尾片与 FULL_START 对齐; 越过即 None(回灌完成)。"""
monkeypatch.setattr(b5, "DB", tmp_path / "dbbardata_5m.db")
b5.save_chunk_state({"next_end": "2020-05-01"})
assert b5.next_chunk(dt.date(2026, 8, 20)) == ("2020-01-03", "2020-05-01")
b5.save_chunk_state({"next_end": "2019-12-31"}) # 已越过起点
assert b5.next_chunk(dt.date(2026, 8, 20)) is None
b5.save_chunk_state({"next_end": None}) # 完结盖章
assert b5.next_chunk(dt.date(2026, 8, 20)) is None
def test_chunk_state_corrupt_falls_back_to_recent(tmp_path, monkeypatch):
"""state 损坏 → 从最近半年重头(OR REPLACE 幂等无伤)。"""
monkeypatch.setattr(b5, "DB", tmp_path / "dbbardata_5m.db")
(tmp_path / "bs_5m_state.json").write_text("{broken", encoding="utf-8")
st = b5.load_chunk_state()
assert st["next_end"] == dt.date.today().isoformat()
# ---------- 单实例锁(防 DSM 定时与全量回灌撞车) ----------
@@ -182,7 +202,11 @@ def test_main_runs_and_exits_0(main_env):
with pytest.raises(SystemExit) as e:
b5.main()
assert e.value.code == 0
assert m_proc.call_count == 2
# 双 sweep: 回灌片(首跑=最近半年) + 每日增量, 各 2 股 → 4 次调用
assert m_proc.call_count == 4
# 整片扫完 → state 推进到片起点
st = json.loads((main_env / "bs_5m_state.json").read_text(encoding="utf-8"))
assert st["next_end"] == (dt.date.today() - dt.timedelta(days=b5.CHUNK_DAYS)).isoformat()
# 数据库文件+schema 已就位
conn = sqlite3.connect(str(b5.DB))
assert conn.execute(
@@ -194,6 +218,16 @@ def test_main_runs_and_exits_0(main_env):
assert not (main_env / "bs_5m.lock").exists()
def test_main_increment_only_when_backfill_done(main_env, monkeypatch):
"""回灌完结盖章(next_end=None) → 只跑每日增量(单 sweep)。"""
b5.save_chunk_state({"next_end": None})
with patch.object(b5, "_process_one", return_value=48) as m_proc:
with pytest.raises(SystemExit) as e:
b5.main()
assert e.value.code == 0
assert m_proc.call_count == 2 # 仅增量 2 股
def test_main_exit2_when_login_fails(main_env, monkeypatch):
monkeypatch.setattr(b5, "login_with_retry", MagicMock(return_value=False))
with pytest.raises(SystemExit) as e: