claude_dev
dfd8421dc6
**Storage Layer:** - Database 类（aiosqlite + WAL + 并发保护） - WikiPage 数据模型 - fix_dirty_states 恢复机制 - FTS5 全文搜索支持 - 完整的 SQLite 表结构 **Service Layer:** - CacheService（LRU 缓存 + TTL + 大小限制） - QueryService（查询服务） - ParserService（Markdown 解析） - IndexerService（索引服务） - GraphService（链接图服务） **Tool Layer (8 个 MCP Tools):** - wiki_query - FTS5 全文搜索 + 标签过滤 - memory_bridge - 按工具来源浏览 - wiki_status - 索引状态 - wiki_lint - 健康审计 - cross_linker - 缺失链接发现 - tag_taxonomy - 标签一致性 - wiki_synthesize - 跨概念综合分析 - daily_update - 日常维护 + hot.md 生成 **MCP Protocol Layer:** - MCP 协议解析和封装 - 工具注册和路由 - 错误处理和日志 - stdio 模式支持 **配置和部署:** - requirements.txt - config.example.yaml - ecosystem.config.cjs (PM2) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
227 lines
7.9 KiB
Python
227 lines
7.9 KiB
Python
"""
|
|
Service Layer - IndexerService（索引服务）
|
|
|
|
管理 wiki 页面索引，支持增量更新和全量重建。
|
|
|
|
参考设计文档：第 2.3 节
|
|
"""
|
|
|
|
import logging
|
|
import asyncio
|
|
from pathlib import Path
|
|
from typing import List, Set, Optional
|
|
from datetime import datetime
|
|
from ..storage import Database, WikiPage, compute_content_hash
|
|
from .parser import ParserService
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class IndexerService:
|
|
"""索引服务 - 管理页面索引"""
|
|
|
|
def __init__(self, db: Database, parser: ParserService, wiki_vault_path: str):
|
|
self.db = db
|
|
self.parser = parser
|
|
self.wiki_vault_path = Path(wiki_vault_path)
|
|
self._lock = asyncio.Lock()
|
|
|
|
async def index_page(self, page_path: str) -> Optional[WikiPage]:
|
|
"""索引单个页面"""
|
|
try:
|
|
full_path = self.wiki_vault_path / page_path
|
|
|
|
# 读取文件内容
|
|
with open(full_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
# 解析页面
|
|
parsed = self.parser.parse_wiki_page(page_path, content)
|
|
|
|
# 计算内容哈希
|
|
content_hash = compute_content_hash(content)
|
|
|
|
# 检查是否需要更新
|
|
stored_hash = await self.db.get_page_hash(page_path)
|
|
if stored_hash == content_hash:
|
|
logger.debug(f"Page unchanged, skipping: {page_path}")
|
|
return None
|
|
|
|
# 创建 WikiPage 对象
|
|
page = WikiPage(
|
|
path=parsed["path"],
|
|
title=parsed["title"],
|
|
category=parsed["category"],
|
|
tags=parsed["tags"],
|
|
summary=parsed["summary"],
|
|
content_hash=content_hash,
|
|
lifecycle="draft", # 默认为 draft，后续可通过 wiki_lint 提升
|
|
source_tool=parsed["source_tool"],
|
|
created_at=parsed["created_at"],
|
|
updated_at=parsed["updated_at"],
|
|
indexed_at=parsed["indexed_at"]
|
|
)
|
|
|
|
# 更新数据库
|
|
await self.db.upsert_page(page)
|
|
|
|
# 更新 FTS5 索引
|
|
await self.db.update_fts_content(
|
|
page_path,
|
|
page.title,
|
|
content,
|
|
page.summary
|
|
)
|
|
|
|
# 更新链接关系
|
|
await self.db.delete_links(page_path)
|
|
for link in parsed.get("links", []):
|
|
await self.db.upsert_link(page_path, link)
|
|
|
|
# 更新标签
|
|
for tag in page.tags:
|
|
await self.db.upsert_tag(tag)
|
|
|
|
logger.info(f"Indexed page: {page_path}")
|
|
return page
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to index page {page_path}: {e}")
|
|
return None
|
|
|
|
async def index_batch(self, page_paths: List[str]) -> dict:
|
|
"""批量索引页面"""
|
|
results = {
|
|
"success": 0,
|
|
"failed": 0,
|
|
"skipped": 0,
|
|
"pages": []
|
|
}
|
|
|
|
for page_path in page_paths:
|
|
try:
|
|
page = await self.index_page(page_path)
|
|
if page:
|
|
results["success"] += 1
|
|
results["pages"].append(page_path)
|
|
else:
|
|
results["skipped"] += 1
|
|
except Exception as e:
|
|
logger.error(f"Failed to index {page_path}: {e}")
|
|
results["failed"] += 1
|
|
|
|
return results
|
|
|
|
async def get_dirty_pages(self) -> List[str]:
|
|
"""获取需要重新索引的页面（增量更新）"""
|
|
all_pages = self.scan_wiki_vault()
|
|
dirty_pages = []
|
|
|
|
for page_path in all_pages:
|
|
full_path = self.wiki_vault_path / page_path
|
|
try:
|
|
with open(full_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
current_hash = compute_content_hash(content)
|
|
stored_hash = await self.db.get_page_hash(page_path)
|
|
if stored_hash != current_hash:
|
|
dirty_pages.append(page_path)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to check {page_path}: {e}")
|
|
|
|
return dirty_pages
|
|
|
|
async def incremental_update(self) -> dict:
|
|
"""增量更新索引 - 只处理变化的页面"""
|
|
logger.info("Starting incremental update...")
|
|
|
|
async with self._lock:
|
|
# 1. 获取所有 wiki 页面
|
|
all_pages = self.scan_wiki_vault()
|
|
|
|
# 2. 检查每个页面的哈希
|
|
for page_path in all_pages:
|
|
try:
|
|
full_path = self.wiki_vault_path / page_path
|
|
with open(full_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
current_hash = compute_content_hash(content)
|
|
stored_hash = await self.db.get_page_hash(page_path)
|
|
|
|
if stored_hash != current_hash:
|
|
await self.index_page(page_path)
|
|
except Exception as e:
|
|
logger.error(f"Failed to update {page_path}: {e}")
|
|
|
|
# 3. 处理删除的页面
|
|
indexed_paths = await self.db.get_all_indexed_paths()
|
|
for path in indexed_paths:
|
|
if path not in all_pages:
|
|
await self.db.delete_page(path)
|
|
await self.db.delete_fts_content(path)
|
|
logger.info(f"Deleted from index: {path}")
|
|
|
|
# 更新元数据
|
|
await self.db.set_meta("last_incremental_update", datetime.now().isoformat())
|
|
|
|
stats = await self.db.get_stats()
|
|
logger.info(f"Incremental update completed: {stats}")
|
|
return stats
|
|
|
|
async def rebuild_index(self) -> dict:
|
|
"""全量重建索引"""
|
|
logger.info("Starting full index rebuild...")
|
|
|
|
async with self._lock:
|
|
# 清空现有索引
|
|
await self.db.execute("DELETE FROM wiki_pages")
|
|
await self.db.execute("DELETE FROM wiki_content")
|
|
await self.db.execute("DELETE FROM wiki_fts")
|
|
await self.db.execute("DELETE FROM wiki_links")
|
|
await self.db.execute("DELETE FROM wiki_tags")
|
|
await self.db.execute("DELETE FROM wiki_page_tags")
|
|
logger.info("Cleared existing index")
|
|
|
|
# 扫描并索引所有页面
|
|
all_pages = self.scan_wiki_vault()
|
|
results = await self.index_batch(all_pages)
|
|
|
|
# 更新元数据
|
|
await self.db.set_meta("last_full_reindex", datetime.now().isoformat())
|
|
|
|
stats = await self.db.get_stats()
|
|
logger.info(f"Full rebuild completed: {stats}")
|
|
return {"results": results, "stats": stats}
|
|
|
|
def scan_wiki_vault(self) -> List[str]:
|
|
"""扫描 wiki vault，返回所有 .md 文件的相对路径"""
|
|
if not self.wiki_vault_path.exists():
|
|
logger.error(f"Wiki vault path does not exist: {self.wiki_vault_path}")
|
|
return []
|
|
|
|
markdown_files = list(self.wiki_vault_path.rglob("*.md"))
|
|
relative_paths = []
|
|
|
|
for md_file in markdown_files:
|
|
try:
|
|
relative_path = md_file.relative_to(self.wiki_vault_path).as_posix()
|
|
relative_paths.append(relative_path)
|
|
except ValueError:
|
|
logger.warning(f"Failed to get relative path for: {md_file}")
|
|
|
|
return sorted(relative_paths)
|
|
|
|
async def get_index_status(self) -> dict:
|
|
"""获取索引状态"""
|
|
stats = await self.db.get_stats()
|
|
dirty_pages = await self.get_dirty_pages()
|
|
|
|
return {
|
|
"total_pages": stats.get("total_pages", 0),
|
|
"total_links": stats.get("total_links", 0),
|
|
"total_tags": stats.get("total_tags", 0),
|
|
"last_indexed": stats.get("last_indexed"),
|
|
"dirty_pages": len(dirty_pages),
|
|
"dirty_page_list": dirty_pages[:10] # 只返回前 10 个
|
|
}
|