fix(data): 掉电防御两刀+funnel配置三层——写侧根因=DayBuffer/IdLedger flush 已tmp+rename但无fsync,掉电下rename进日志数据块滞页缓存=0字节终局残骸(news_meta dt=2026-10-05 part-1实锤,funnel walk裸崩),修=_fsync_file先fsync再replace;读侧纵深=partition_reader对0B/截断坏件skip+ERROR不裸崩(失败分级新增分区残骸档,全消费方受益);LLM配置三层条件切(factor P5转达):pipeline库存在才config_store.resolve_effective_config(只读位_connect会makedirs+建表抛错,/app只读挂载必炸),否则env两层兜底零行为变化;配套账本orphan剪除生产修复(961913→961912)+0B残骸清除已落NAS; fusion spec §20.16台账 [nas] [vps]
This commit is contained in:
@@ -330,6 +330,17 @@ def _row_key(domain, row):
|
||||
return "|".join(str(row.get(c) or "") for c in ID_KEYS[domain])
|
||||
|
||||
|
||||
def _fsync_file(path):
|
||||
"""10-05 NAS 掉电案: os.replace 的 rename 进日志但数据块可能仍在页缓存
|
||||
(延迟分配) → 掉电=0 字节终局残骸(news_meta dt=2026-10-05 part-1 实锤,
|
||||
funnel walk 一读即裸崩)。fsync 数据块后再 rename, 进程崩与掉电双安全。"""
|
||||
fd = os.open(str(path), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
class IdLedger:
|
||||
"""全局 id 账本(state/ids_<domain>.parquet, int64 hash 有序数组)。
|
||||
|
||||
@@ -364,6 +375,7 @@ class IdLedger:
|
||||
[self._arr, np.array(self._pending, dtype=np.int64)]))
|
||||
tmp = self.path.with_suffix(".tmp")
|
||||
pd.DataFrame({"h": merged}).to_parquet(tmp, index=False)
|
||||
_fsync_file(tmp) # 10-05: 掉电防 0 字节残骸
|
||||
os.replace(tmp, self.path)
|
||||
self._arr = merged
|
||||
self._pending = []
|
||||
@@ -443,6 +455,7 @@ class DayBuffer:
|
||||
tmp = d / f".part-{self.next_idx}.{os.getpid()}.tmp"
|
||||
try:
|
||||
pd.DataFrame(self.rows).to_parquet(tmp, index=False)
|
||||
_fsync_file(tmp) # 10-05: 掉电防 0 字节终局残骸
|
||||
os.replace(tmp, d / f"part-{self.next_idx}.parquet")
|
||||
except Exception:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
@@ -338,7 +338,16 @@ def main(argv=None):
|
||||
if not args.no_llm and (pending or fresh):
|
||||
try:
|
||||
from sanguo_api.llm import resolve_config, LLMClient
|
||||
cfg = resolve_config()
|
||||
# 10-05(factor P5 转达): LLM 配置三层 DB>env>默认 ——
|
||||
# pipeline 库存在才走 config_store 三层(不存在/只读位
|
||||
# _connect 会 makedirs+建表抛错, funnel 容器 /app 只读挂载
|
||||
# 必炸), 否则 env 两层兜底=生产行为零变化
|
||||
if args.pipeline_db and os.path.exists(args.pipeline_db):
|
||||
from sanguo_api.llm.config_store import (
|
||||
resolve_effective_config)
|
||||
cfg = resolve_effective_config(args.pipeline_db)
|
||||
else:
|
||||
cfg = resolve_config()
|
||||
except Exception as e: # noqa: BLE001 配置缺失
|
||||
log.error("LLM 配置缺失(env 三件见 runbook P4-1 节): %s", e)
|
||||
config_error = True
|
||||
|
||||
Reference in New Issue
Block a user