""" Service Layer - IndexerService（索引服务） 管理 wiki 页面索引，支持增量更新和全量重建。 参考设计文档：第 2.3 节 """ import logging import asyncio from pathlib import Path from typing import List, Set, Optional from datetime import datetime from ..storage import Database, WikiPage, compute_content_hash from .parser import ParserService logger = logging.getLogger(__name__) class IndexerService: """索引服务 - 管理页面索引""" def __init__(self, db: Database, parser: ParserService, wiki_vault_path: str): self.db = db self.parser = parser self.wiki_vault_path = Path(wiki_vault_path) self._lock = asyncio.Lock() async def index_page(self, page_path: str) -> Optional[WikiPage]: """索引单个页面""" try: full_path = self.wiki_vault_path / page_path # 读取文件内容 with open(full_path, 'r', encoding='utf-8') as f: content = f.read() # 解析页面 parsed = self.parser.parse_wiki_page(page_path, content) # 计算内容哈希 content_hash = compute_content_hash(content) # 检查是否需要更新 stored_hash = await self.db.get_page_hash(page_path) if stored_hash == content_hash: logger.debug(f"Page unchanged, skipping: {page_path}") return None # 创建 WikiPage 对象 page = WikiPage( path=parsed["path"], title=parsed["title"], category=parsed["category"], tags=parsed["tags"], summary=parsed["summary"], content_hash=content_hash, lifecycle="draft", # 默认为 draft，后续可通过 wiki_lint 提升 source_tool=parsed["source_tool"], created_at=parsed["created_at"], updated_at=parsed["updated_at"], indexed_at=parsed["indexed_at"] ) # 更新数据库 await self.db.upsert_page(page) # 更新 FTS5 索引 await self.db.update_fts_content( page_path, page.title, content, page.summary ) # 更新链接关系 await self.db.delete_links(page_path) for link in parsed.get("links", []): await self.db.upsert_link(page_path, link) # 更新标签 for tag in page.tags: await self.db.upsert_tag(tag) logger.info(f"Indexed page: {page_path}") return page except Exception as e: logger.error(f"Failed to index page {page_path}: {e}") return None async def index_batch(self, page_paths: List[str]) -> dict: """批量索引页面""" results = { "success": 0, "failed": 0, "skipped": 0, "pages": [] } for page_path in page_paths: try: page = await self.index_page(page_path) if page: results["success"] += 1 results["pages"].append(page_path) else: results["skipped"] += 1 except Exception as e: logger.error(f"Failed to index {page_path}: {e}") results["failed"] += 1 return results async def get_dirty_pages(self) -> List[str]: """获取需要重新索引的页面（增量更新）""" all_pages = self.scan_wiki_vault() dirty_pages = [] for page_path in all_pages: full_path = self.wiki_vault_path / page_path try: with open(full_path, 'r', encoding='utf-8') as f: content = f.read() current_hash = compute_content_hash(content) stored_hash = await self.db.get_page_hash(page_path) if stored_hash != current_hash: dirty_pages.append(page_path) except Exception as e: logger.warning(f"Failed to check {page_path}: {e}") return dirty_pages async def incremental_update(self) -> dict: """增量更新索引 - 只处理变化的页面""" logger.info("Starting incremental update...") async with self._lock: # 1. 获取所有 wiki 页面 all_pages = self.scan_wiki_vault() # 2. 检查每个页面的哈希 for page_path in all_pages: try: full_path = self.wiki_vault_path / page_path with open(full_path, 'r', encoding='utf-8') as f: content = f.read() current_hash = compute_content_hash(content) stored_hash = await self.db.get_page_hash(page_path) if stored_hash != current_hash: await self.index_page(page_path) except Exception as e: logger.error(f"Failed to update {page_path}: {e}") # 3. 处理删除的页面 indexed_paths = await self.db.get_all_indexed_paths() for path in indexed_paths: if path not in all_pages: await self.db.delete_page(path) await self.db.delete_fts_content(path) logger.info(f"Deleted from index: {path}") # 更新元数据 await self.db.set_meta("last_incremental_update", datetime.now().isoformat()) stats = await self.db.get_stats() logger.info(f"Incremental update completed: {stats}") return stats async def rebuild_index(self) -> dict: """全量重建索引""" logger.info("Starting full index rebuild...") async with self._lock: # 清空现有索引 await self.db.execute("DELETE FROM wiki_pages") await self.db.execute("DELETE FROM wiki_content") await self.db.execute("DELETE FROM wiki_fts") await self.db.execute("DELETE FROM wiki_links") await self.db.execute("DELETE FROM wiki_tags") await self.db.execute("DELETE FROM wiki_page_tags") logger.info("Cleared existing index") # 扫描并索引所有页面 all_pages = self.scan_wiki_vault() results = await self.index_batch(all_pages) # 更新元数据 await self.db.set_meta("last_full_reindex", datetime.now().isoformat()) stats = await self.db.get_stats() logger.info(f"Full rebuild completed: {stats}") return {"results": results, "stats": stats} def scan_wiki_vault(self) -> List[str]: """扫描 wiki vault，返回所有 .md 文件的相对路径""" if not self.wiki_vault_path.exists(): logger.error(f"Wiki vault path does not exist: {self.wiki_vault_path}") return [] markdown_files = list(self.wiki_vault_path.rglob("*.md")) relative_paths = [] for md_file in markdown_files: try: relative_path = md_file.relative_to(self.wiki_vault_path).as_posix() relative_paths.append(relative_path) except ValueError: logger.warning(f"Failed to get relative path for: {md_file}") return sorted(relative_paths) async def get_index_status(self) -> dict: """获取索引状态""" stats = await self.db.get_stats() dirty_pages = await self.get_dirty_pages() return { "total_pages": stats.get("total_pages", 0), "total_links": stats.get("total_links", 0), "total_tags": stats.get("total_tags", 0), "last_indexed": stats.get("last_indexed"), "dirty_pages": len(dirty_pages), "dirty_page_list": dirty_pages[:10] # 只返回前 10 个 }