fix(data): 账本启动对账自愈——分区有/账本无缺口永久闭合 [nas]
09-07 巡检 verify FAIL 实锤: 旧代码部分段(news/fulltext/pdf)无 flush 计数器, docker stop 的腿把内存账本丢掉(ann 差4.5k/news 差9.9w;分区数据完好 dups=0, 近窗索引兜底着去重但账本缺口30天后成理论重拉窗)。修=两 lane 启动时 _reconcile_ledgers: 近30日分区 key 对账补进账本并落盘,幂等; 新代码批量提交已让缺口收敛到单commit周期,对账再兜历史+未来任何崩溃窗 测试 55(新增对账自愈幂等钉);全套 272 绿
This commit is contained in:
@@ -218,6 +218,24 @@ def test_commit_batches_data_before_markers(root, monkeypatch):
|
||||
assert cd.IdLedger("ann_meta").has_any(["m1"]) == [True]
|
||||
|
||||
|
||||
def test_reconcile_ledgers_heals_crash_gap(root):
|
||||
"""启动对账自愈(09-07 巡检实锤旧代码丢内存账本 ann 差4.5k/news 差9.9w):
|
||||
分区有而账本无 → 启动补齐;账本⊇近30日分区永久成立。"""
|
||||
import pandas as pd
|
||||
today = dt.date.today().isoformat()
|
||||
d = root / "ann_meta" / f"dt={today}"
|
||||
d.mkdir(parents=True)
|
||||
pd.DataFrame({"announcement_id": ["r1", "r2"]}).to_parquet(d / "part-0.parquet")
|
||||
ledgers = {"ann_meta": cd.IdLedger("ann_meta")} # 空账本
|
||||
recents = {"ann_meta": cd.RecentIndex("ann_meta", days=7)}
|
||||
assert ledgers["ann_meta"].has_any(["r1", "r2"]) == [False, False]
|
||||
cd._reconcile_ledgers(ledgers, recents)
|
||||
assert cd.IdLedger("ann_meta").has_any(["r1", "r2"]) == [True, True] # 落盘可复读
|
||||
# 幂等: 再对账零新增
|
||||
cd._reconcile_ledgers(ledgers, recents)
|
||||
assert cd.IdLedger("ann_meta").has_any(["r1", "r2"]) == [True, True]
|
||||
|
||||
|
||||
# ---------- unit marker ----------
|
||||
|
||||
def test_marker_done_and_skip(root):
|
||||
|
||||
Reference in New Issue
Block a user