fix(data): 掉电防御两刀+funnel配置三层——写侧根因=DayBuffer/IdLedger flush 已tmp+rename但无fsync,掉电下rename进日志数据块滞页缓存=0字节终局残骸(news_meta dt=2026-10-05 part-1实锤,funnel walk裸崩),修=_fsync_file先fsync再replace;读侧纵深=partition_reader对0B/截断坏件skip+ERROR不裸崩(失败分级新增分区残骸档,全消费方受益);LLM配置三层条件切(factor P5转达):pipeline库存在才config_store.resolve_effective_config(只读位_connect会makedirs+建表抛错,/app只读挂载必炸),否则env两层兜底零行为变化;配套账本orphan剪除生产修复(961913→961912)+0B残骸清除已落NAS; fusion spec §20.16台账 [nas] [vps]
CI/CD / test (push) Failing after 49s
CI/CD / nas-deploy (push) Has been cancelled
CI/CD / nas-verify (push) Has been cancelled

This commit is contained in:
2026-10-05 12:42:02 +08:00
parent 7cd3f49c76
commit aba1c3de80
6 changed files with 102 additions and 2 deletions
+13
View File
@@ -330,6 +330,17 @@ def _row_key(domain, row):
return "|".join(str(row.get(c) or "") for c in ID_KEYS[domain])
def _fsync_file(path):
"""10-05 NAS 掉电案: os.replace 的 rename 进日志但数据块可能仍在页缓存
(延迟分配) → 掉电=0 字节终局残骸(news_meta dt=2026-10-05 part-1 实锤,
funnel walk 一读即裸崩)。fsync 数据块后再 rename, 进程崩与掉电双安全。"""
fd = os.open(str(path), os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
class IdLedger:
"""全局 id 账本(state/ids_<domain>.parquet, int64 hash 有序数组)。
@@ -364,6 +375,7 @@ class IdLedger:
[self._arr, np.array(self._pending, dtype=np.int64)]))
tmp = self.path.with_suffix(".tmp")
pd.DataFrame({"h": merged}).to_parquet(tmp, index=False)
_fsync_file(tmp) # 10-05: 掉电防 0 字节残骸
os.replace(tmp, self.path)
self._arr = merged
self._pending = []
@@ -443,6 +455,7 @@ class DayBuffer:
tmp = d / f".part-{self.next_idx}.{os.getpid()}.tmp"
try:
pd.DataFrame(self.rows).to_parquet(tmp, index=False)
_fsync_file(tmp) # 10-05: 掉电防 0 字节终局残骸
os.replace(tmp, d / f"part-{self.next_idx}.parquet")
except Exception:
tmp.unlink(missing_ok=True)
+10 -1
View File
@@ -338,7 +338,16 @@ def main(argv=None):
if not args.no_llm and (pending or fresh):
try:
from sanguo_api.llm import resolve_config, LLMClient
cfg = resolve_config()
# 10-05(factor P5 转达): LLM 配置三层 DB>env>默认 ——
# pipeline 库存在才走 config_store 三层(不存在/只读位
# _connect 会 makedirs+建表抛错, funnel 容器 /app 只读挂载
# 必炸), 否则 env 两层兜底=生产行为零变化
if args.pipeline_db and os.path.exists(args.pipeline_db):
from sanguo_api.llm.config_store import (
resolve_effective_config)
cfg = resolve_effective_config(args.pipeline_db)
else:
cfg = resolve_config()
except Exception as e: # noqa: BLE001 配置缺失
log.error("LLM 配置缺失(env 三件见 runbook P4-1 节): %s", e)
config_error = True