claude_dev
62b8fef18a
- C1 (Critical): 添加 PyYAML 解析 frontmatter，支持多行值和列表 - M1 (Major): 添加 FTS5 查询验证，防止注入攻击 - M2/M3 (Major): 工具类使用显式 wiki_vault_path 参数 - M4 (Major): 集成 aiofiles 实现真正的异步文件 I/O - M5 (Major): 改用 logger.exception() 记录完整堆栈 Co-Authored-By: Claude Dev <noreply@anthropic.com>
228 lines
8.0 KiB
Python
228 lines
8.0 KiB
Python
"""
|
||
Service Layer - IndexerService（索引服务）
|
||
|
||
管理 wiki 页面索引，支持增量更新和全量重建。
|
||
|
||
参考设计文档：第 2.3 节
|
||
"""
|
||
|
||
import logging
|
||
import asyncio
|
||
from pathlib import Path
|
||
from typing import List, Set, Optional
|
||
from datetime import datetime
|
||
import aiofiles
|
||
from ..storage import Database, WikiPage, compute_content_hash
|
||
from .parser import ParserService
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class IndexerService:
|
||
"""索引服务 - 管理页面索引"""
|
||
|
||
def __init__(self, db: Database, parser: ParserService, wiki_vault_path: str):
|
||
self.db = db
|
||
self.parser = parser
|
||
self.wiki_vault_path = Path(wiki_vault_path)
|
||
self._lock = asyncio.Lock()
|
||
|
||
async def index_page(self, page_path: str) -> Optional[WikiPage]:
|
||
"""索引单个页面"""
|
||
try:
|
||
full_path = self.wiki_vault_path / page_path
|
||
|
||
# 读取文件内容（使用 aiofiles 实现真正的异步 I/O）
|
||
async with aiofiles.open(full_path, 'r', encoding='utf-8') as f:
|
||
content = await f.read()
|
||
|
||
# 解析页面
|
||
parsed = self.parser.parse_wiki_page(page_path, content)
|
||
|
||
# 计算内容哈希
|
||
content_hash = compute_content_hash(content)
|
||
|
||
# 检查是否需要更新
|
||
stored_hash = await self.db.get_page_hash(page_path)
|
||
if stored_hash == content_hash:
|
||
logger.debug(f"Page unchanged, skipping: {page_path}")
|
||
return None
|
||
|
||
# 创建 WikiPage 对象
|
||
page = WikiPage(
|
||
path=parsed["path"],
|
||
title=parsed["title"],
|
||
category=parsed["category"],
|
||
tags=parsed["tags"],
|
||
summary=parsed["summary"],
|
||
content_hash=content_hash,
|
||
lifecycle="draft", # 默认为 draft，后续可通过 wiki_lint 提升
|
||
source_tool=parsed["source_tool"],
|
||
created_at=parsed["created_at"],
|
||
updated_at=parsed["updated_at"],
|
||
indexed_at=parsed["indexed_at"]
|
||
)
|
||
|
||
# 更新数据库
|
||
await self.db.upsert_page(page)
|
||
|
||
# 更新 FTS5 索引
|
||
await self.db.update_fts_content(
|
||
page_path,
|
||
page.title,
|
||
content,
|
||
page.summary
|
||
)
|
||
|
||
# 更新链接关系
|
||
await self.db.delete_links(page_path)
|
||
for link in parsed.get("links", []):
|
||
await self.db.upsert_link(page_path, link)
|
||
|
||
# 更新标签
|
||
for tag in page.tags:
|
||
await self.db.upsert_tag(tag)
|
||
|
||
logger.info(f"Indexed page: {page_path}")
|
||
return page
|
||
|
||
except Exception as e:
|
||
logger.error(f"Failed to index page {page_path}: {e}")
|
||
return None
|
||
|
||
async def index_batch(self, page_paths: List[str]) -> dict:
|
||
"""批量索引页面"""
|
||
results = {
|
||
"success": 0,
|
||
"failed": 0,
|
||
"skipped": 0,
|
||
"pages": []
|
||
}
|
||
|
||
for page_path in page_paths:
|
||
try:
|
||
page = await self.index_page(page_path)
|
||
if page:
|
||
results["success"] += 1
|
||
results["pages"].append(page_path)
|
||
else:
|
||
results["skipped"] += 1
|
||
except Exception as e:
|
||
logger.error(f"Failed to index {page_path}: {e}")
|
||
results["failed"] += 1
|
||
|
||
return results
|
||
|
||
async def get_dirty_pages(self) -> List[str]:
|
||
"""获取需要重新索引的页面（增量更新）"""
|
||
all_pages = self.scan_wiki_vault()
|
||
dirty_pages = []
|
||
|
||
for page_path in all_pages:
|
||
full_path = self.wiki_vault_path / page_path
|
||
try:
|
||
async with aiofiles.open(full_path, 'r', encoding='utf-8') as f:
|
||
content = await f.read()
|
||
current_hash = compute_content_hash(content)
|
||
stored_hash = await self.db.get_page_hash(page_path)
|
||
if stored_hash != current_hash:
|
||
dirty_pages.append(page_path)
|
||
except Exception as e:
|
||
logger.warning(f"Failed to check {page_path}: {e}")
|
||
|
||
return dirty_pages
|
||
|
||
async def incremental_update(self) -> dict:
|
||
"""增量更新索引 - 只处理变化的页面"""
|
||
logger.info("Starting incremental update...")
|
||
|
||
async with self._lock:
|
||
# 1. 获取所有 wiki 页面
|
||
all_pages = self.scan_wiki_vault()
|
||
|
||
# 2. 检查每个页面的哈希
|
||
for page_path in all_pages:
|
||
try:
|
||
full_path = self.wiki_vault_path / page_path
|
||
async with aiofiles.open(full_path, 'r', encoding='utf-8') as f:
|
||
content = await f.read()
|
||
current_hash = compute_content_hash(content)
|
||
stored_hash = await self.db.get_page_hash(page_path)
|
||
|
||
if stored_hash != current_hash:
|
||
await self.index_page(page_path)
|
||
except Exception as e:
|
||
logger.error(f"Failed to update {page_path}: {e}")
|
||
|
||
# 3. 处理删除的页面
|
||
indexed_paths = await self.db.get_all_indexed_paths()
|
||
for path in indexed_paths:
|
||
if path not in all_pages:
|
||
await self.db.delete_page(path)
|
||
await self.db.delete_fts_content(path)
|
||
logger.info(f"Deleted from index: {path}")
|
||
|
||
# 更新元数据
|
||
await self.db.set_meta("last_incremental_update", datetime.now().isoformat())
|
||
|
||
stats = await self.db.get_stats()
|
||
logger.info(f"Incremental update completed: {stats}")
|
||
return stats
|
||
|
||
async def rebuild_index(self) -> dict:
|
||
"""全量重建索引"""
|
||
logger.info("Starting full index rebuild...")
|
||
|
||
async with self._lock:
|
||
# 清空现有索引
|
||
await self.db.execute("DELETE FROM wiki_pages")
|
||
await self.db.execute("DELETE FROM wiki_content")
|
||
await self.db.execute("DELETE FROM wiki_fts")
|
||
await self.db.execute("DELETE FROM wiki_links")
|
||
await self.db.execute("DELETE FROM wiki_tags")
|
||
await self.db.execute("DELETE FROM wiki_page_tags")
|
||
logger.info("Cleared existing index")
|
||
|
||
# 扫描并索引所有页面
|
||
all_pages = self.scan_wiki_vault()
|
||
results = await self.index_batch(all_pages)
|
||
|
||
# 更新元数据
|
||
await self.db.set_meta("last_full_reindex", datetime.now().isoformat())
|
||
|
||
stats = await self.db.get_stats()
|
||
logger.info(f"Full rebuild completed: {stats}")
|
||
return {"results": results, "stats": stats}
|
||
|
||
def scan_wiki_vault(self) -> List[str]:
|
||
"""扫描 wiki vault，返回所有 .md 文件的相对路径"""
|
||
if not self.wiki_vault_path.exists():
|
||
logger.error(f"Wiki vault path does not exist: {self.wiki_vault_path}")
|
||
return []
|
||
|
||
markdown_files = list(self.wiki_vault_path.rglob("*.md"))
|
||
relative_paths = []
|
||
|
||
for md_file in markdown_files:
|
||
try:
|
||
relative_path = md_file.relative_to(self.wiki_vault_path).as_posix()
|
||
relative_paths.append(relative_path)
|
||
except ValueError:
|
||
logger.warning(f"Failed to get relative path for: {md_file}")
|
||
|
||
return sorted(relative_paths)
|
||
|
||
async def get_index_status(self) -> dict:
|
||
"""获取索引状态"""
|
||
stats = await self.db.get_stats()
|
||
dirty_pages = await self.get_dirty_pages()
|
||
|
||
return {
|
||
"total_pages": stats.get("total_pages", 0),
|
||
"total_links": stats.get("total_links", 0),
|
||
"total_tags": stats.get("total_tags", 0),
|
||
"last_indexed": stats.get("last_indexed"),
|
||
"dirty_pages": len(dirty_pages),
|
||
"dirty_page_list": dirty_pages[:10] # 只返回前 10 个
|
||
}
|