feat(§03): 实现 MCP Server 核心 - Storage/Service/Tool/MCP Protocol Layers

**Storage Layer:**
- Database 类(aiosqlite + WAL + 并发保护)
- WikiPage 数据模型
- fix_dirty_states 恢复机制
- FTS5 全文搜索支持
- 完整的 SQLite 表结构

**Service Layer:**
- CacheService(LRU 缓存 + TTL + 大小限制)
- QueryService(查询服务)
- ParserService(Markdown 解析)
- IndexerService(索引服务)
- GraphService(链接图服务)

**Tool Layer (8 个 MCP Tools):**
- wiki_query - FTS5 全文搜索 + 标签过滤
- memory_bridge - 按工具来源浏览
- wiki_status - 索引状态
- wiki_lint - 健康审计
- cross_linker - 缺失链接发现
- tag_taxonomy - 标签一致性
- wiki_synthesize - 跨概念综合分析
- daily_update - 日常维护 + hot.md 生成

**MCP Protocol Layer:**
- MCP 协议解析和封装
- 工具注册和路由
- 错误处理和日志
- stdio 模式支持

**配置和部署:**
- requirements.txt
- config.example.yaml
- ecosystem.config.cjs (PM2)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-26 11:32:43 +08:00
parent 67a53d82de
commit dfd8421dc6
22 changed files with 2561 additions and 0 deletions
+7
View File
@@ -0,0 +1,7 @@
"""
MCP Server - Wiki MCP Server
混合架构 Wiki 系统的 MCP Server 实现。
"""
__version__ = "1.0.0"
+218
View File
@@ -0,0 +1,218 @@
"""
MCP Server - Wiki MCP Server 主入口
实现 MCP 协议解析、工具注册和路由。
参考设计文档:第 2.1 节
"""
import asyncio
import json
import logging
import sys
from typing import Any, Callable, Dict, Optional
from pathlib import Path
# MCP SDK
try:
from mcp.server.models import InitializationOptions
from mcp.server import Server, NotificationOptions
from mcp.types import Tool, TextContent
MCP_AVAILABLE = True
except ImportError:
MCP_AVAILABLE = False
logger = logging.getLogger(__name__)
logger.warning("MCP SDK not available, using mock implementation")
# 本地模块
from .storage import Database, fix_dirty_states
from .services import CacheService, QueryService, ParserService, IndexerService, GraphService
from .tools import (
WikiQueryTool, MemoryBridgeTool, WikiStatusTool,
WikiLintTool, CrossLinkerTool, TagTaxonomyTool,
WikiSynthesizeTool, DailyUpdateTool
)
# 配置
DEFAULT_WIKI_VAULT = "/Volumes/KnowledgeBase/wiki-vault"
DEFAULT_INDEX_PATH = "~/.sanguo-llmwiki/index.db"
DEFAULT_LOG_LEVEL = "INFO"
# 配置日志
logging.basicConfig(
level=getattr(logging, DEFAULT_LOG_LEVEL),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
class MCPServer:
"""MCP 服务器封装类"""
def __init__(self, wiki_vault_path: str, index_path: str):
self.wiki_vault_path = wiki_vault_path
self.index_path = index_path
self.db: Optional[Database] = None
self.tools: Dict[str, Callable] = {}
self.tool_schemas: Dict[str, dict] = {}
# 如果 MCP SDK 可用,创建 Server 实例
if MCP_AVAILABLE:
self.server = Server("wiki-mcp")
self._setup_handlers()
else:
self.server = None
logger.warning("Running in mock mode (MCP SDK not available)")
async def initialize(self) -> None:
"""初始化数据库和服务"""
logger.info("Initializing Wiki MCP Server...")
# 初始化数据库
self.db = Database(self.index_path)
await self.db.connect()
# 恢复脏状态
await fix_dirty_states(self.db)
# 初始化服务
self.cache = CacheService(max_size=1000)
self.query_service = QueryService(self.db, self.cache)
self.parser_service = ParserService()
self.indexer_service = IndexerService(self.db, self.parser_service, self.wiki_vault_path)
self.graph_service = GraphService(self.db)
# 初始化工具
self._init_tools()
logger.info("Wiki MCP Server initialized")
def _init_tools(self) -> None:
"""初始化所有工具"""
wiki_query = WikiQueryTool(self.query_service)
memory_bridge = MemoryBridgeTool(self.query_service)
wiki_status = WikiStatusTool(self.query_service, self.indexer_service, self.graph_service)
wiki_lint = WikiLintTool(self.parser_service, self.query_service, self.graph_service)
cross_linker = CrossLinkerTool(self.graph_service)
tag_taxonomy = TagTaxonomyTool(self.query_service)
wiki_synthesize = WikiSynthesizeTool(self.query_service, self.graph_service)
daily_update = DailyUpdateTool(self.query_service, self.indexer_service)
# 注册工具
self.tools = {
"wiki_query": wiki_query.handle,
"memory_bridge": memory_bridge.handle,
"wiki_status": wiki_status.handle,
"wiki_lint": wiki_lint.handle,
"cross_linker": cross_linker.handle,
"tag_taxonomy": tag_taxonomy.handle,
"wiki_synthesize": wiki_synthesize.handle,
"daily_update": daily_update.handle
}
# 注册工具 schema
self.tool_schemas = {
"wiki_query": wiki_query.get_schema(),
"memory_bridge": memory_bridge.get_schema(),
"wiki_status": wiki_status.get_schema(),
"wiki_lint": wiki_lint.get_schema(),
"cross_linker": cross_linker.get_schema(),
"tag_taxonomy": tag_taxonomy.get_schema(),
"wiki_synthesize": wiki_synthesize.get_schema(),
"daily_update": daily_update.get_schema()
}
logger.info(f"Registered {len(self.tools)} tools")
def _setup_handlers(self) -> None:
"""设置 MCP 处理器"""
if not self.server:
return
@self.server.list_tools()
async def list_tools() -> list[Tool]:
"""列出所有可用工具"""
return [
Tool(
name=schema["name"],
description=schema["description"],
inputSchema=schema["inputSchema"]
)
for schema in self.tool_schemas.values()
]
@self.server.call_tool()
async def call_tool(name: str, arguments: dict) -> list[TextContent]:
"""调用工具"""
if name not in self.tools:
raise ValueError(f"Unknown tool: {name}")
try:
result = await self.tools[name](**arguments)
return [TextContent(type="text", text=json.dumps(result, ensure_ascii=False, indent=2))]
except Exception as e:
logger.error(f"Error calling tool {name}: {e}")
return [TextContent(type="text", text=json.dumps({
"success": False,
"error": str(e)
}, ensure_ascii=False))]
async def run(self) -> None:
"""运行 MCP 服务器"""
if not self.server:
logger.error("MCP Server not available (MCP SDK not installed)")
return
# 运行服务器
async with self.server.stdio_stdio() as (read_stream, write_stream):
await self.server.run(
read_stream,
write_stream,
InitializationOptions(
server_name="wiki-mcp",
server_version="1.0.0",
capabilities=self.server.get_capabilities(
notification_options=NotificationOptions(),
experimental_capabilities={}
)
)
)
async def close(self) -> None:
"""关闭服务器"""
if self.db:
await self.db.close()
logger.info("Database connection closed")
async def main():
"""主入口"""
import os
# 从环境变量读取配置
wiki_vault_path = os.environ.get("WIKI_VAULT_PATH", DEFAULT_WIKI_VAULT)
index_path = os.path.expanduser(os.environ.get("WIKI_INDEX_PATH", DEFAULT_INDEX_PATH))
log_level = os.environ.get("LOG_LEVEL", DEFAULT_LOG_LEVEL)
# 设置日志级别
logging.getLogger().setLevel(getattr(logging, log_level))
# 检查 wiki vault 路径
if not Path(wiki_vault_path).exists():
logger.error(f"Wiki vault path does not exist: {wiki_vault_path}")
sys.exit(1)
# 创建并运行服务器
server = MCPServer(wiki_vault_path, index_path)
try:
await server.initialize()
await server.run()
except KeyboardInterrupt:
logger.info("Interrupted by user")
finally:
await server.close()
if __name__ == "__main__":
asyncio.run(main())
+19
View File
@@ -0,0 +1,19 @@
"""
Service Layer - 服务层
提供各种服务:缓存、查询、索引、解析、图服务。
"""
from .cache import CacheService
from .query import QueryService
from .parser import ParserService
from .indexer import IndexerService
from .graph import GraphService
__all__ = [
"CacheService",
"QueryService",
"ParserService",
"IndexerService",
"GraphService"
]
+91
View File
@@ -0,0 +1,91 @@
"""
Service Layer - CacheService(缓存服务)
使用 LRU 缓存 + TTL 过期策略,防止内存泄漏。
参考设计文档:第 2.3 节
"""
import asyncio
import time
from collections import OrderedDict
from typing import Optional, Any
import fnmatch
import logging
logger = logging.getLogger(__name__)
class CacheService:
"""缓存服务 - LRU 缓存 + TTL 过期"""
def __init__(self, max_size: int = 1000):
self.cache: OrderedDict[str, tuple] = OrderedDict() # key -> (value, expire_time)
self.max_size = max_size
self.lock = asyncio.Lock()
async def get(self, key: str) -> Optional[Any]:
"""获取缓存值(异步,带锁)"""
async with self.lock:
if key not in self.cache:
return None
value, expire_time = self.cache[key]
# 检查是否过期
if expire_time and time.time() > expire_time:
del self.cache[key]
logger.debug(f"Cache expired: {key}")
return None
# LRU: 移到末尾
self.cache.move_to_end(key)
return value
async def set(self, key: str, value: Any, ttl: int = 3600) -> None:
"""设置缓存值(异步,带锁)"""
async with self.lock:
expire_time = time.time() + ttl if ttl else None
# 如果缓存已满,删除最旧的条目
if len(self.cache) >= self.max_size and key not in self.cache:
self.cache.popitem(last=False) # FIFO 删除
logger.debug(f"Cache full, evicted oldest entry")
self.cache[key] = (value, expire_time)
self.cache.move_to_end(key)
logger.debug(f"Cache set: {key} (TTL={ttl}s)")
async def invalidate(self, pattern: str) -> int:
"""按模式清除缓存(支持 * 通配符)"""
async with self.lock:
if pattern == "*":
count = len(self.cache)
self.cache.clear()
logger.info(f"Cache cleared: {count} entries")
return count
keys_to_delete = [k for k in self.cache.keys() if fnmatch.fnmatch(k, pattern)]
for key in keys_to_delete:
del self.cache[key]
logger.info(f"Cache invalidated: {len(keys_to_delete)} entries matching '{pattern}'")
return len(keys_to_delete)
async def get_stats(self) -> dict:
"""获取缓存统计信息"""
async with self.lock:
now = time.time()
expired_count = sum(1 for _, expire_time in self.cache.values() if expire_time and now > expire_time)
return {
"size": len(self.cache),
"max_size": self.max_size,
"expired_count": expired_count,
"utilization": len(self.cache) / self.max_size if self.max_size > 0 else 0
}
async def clear(self) -> None:
"""清空缓存"""
async with self.lock:
self.cache.clear()
logger.info("Cache cleared")
+137
View File
@@ -0,0 +1,137 @@
"""
Service Layer - GraphService(图服务)
管理页面间的链接关系,查找孤立页面和缺失链接。
参考设计文档:第 2.3 节
"""
import logging
from typing import Set, List, Tuple
from ..storage import Database
logger = logging.getLogger(__name__)
class GraphService:
"""图服务 - 管理链接关系"""
def __init__(self, db: Database):
self.db = db
async def get_links(self, path: str) -> Set[str]:
"""获取页面的出链"""
return await self.db.get_links(path)
async def get_backlinks(self, path: str) -> Set[str]:
"""获取页面的反向链接"""
return await self.db.get_backlinks(path)
async def find_orphans(self) -> Set[str]:
"""查找孤立页面(无反向链接)"""
all_pages = await self.db.get_all_pages()
orphans = set()
for page in all_pages:
backlinks = await self.db.get_backlinks(page.path)
if not backlinks and page.path != "index.md":
orphans.add(page.path)
return orphans
async def find_missing_links(self) -> List[Tuple[str, str]]:
"""查找缺失的交叉引用"""
# 获取所有页面内容
all_pages = await self.db.get_all_pages()
page_paths = {page.path for page in all_pages}
# 获取所有链接关系
links_rows = await self.db.fetch_all("SELECT source, target FROM wiki_links")
missing = []
for source, target in links_rows:
if target not in page_paths:
missing.append((source, target))
return missing
async def get_link_graph(self) -> dict:
"""获取完整的链接图"""
pages = await self.db.get_all_pages()
graph = {}
for page in pages:
links = await self.db.get_links(page.path)
backlinks = await self.db.get_backlinks(page.path)
graph[page.path] = {
"title": page.title,
"category": page.category,
"links": sorted(links),
"backlinks": sorted(backlinks),
"links_count": len(links),
"backlinks_count": len(backlinks)
}
return graph
async def get_highly_connected_pages(self, threshold: int = 5) -> List[dict]:
"""获取高度连接的页面(入链 + 出链 >= threshold"""
pages = await self.db.get_all_pages()
highly_connected = []
for page in pages:
links = await self.db.get_links(page.path)
backlinks = await self.db.get_backlinks(page.path)
total = len(links) + len(backlinks)
if total >= threshold:
highly_connected.append({
"path": page.path,
"title": page.title,
"links_count": len(links),
"backlinks_count": len(backlinks),
"total_connections": total
})
return sorted(highly_connected, key=lambda x: x["total_connections"], reverse=True)
async def get_disconnected_components(self) -> List[Set[str]]:
"""获取不连通的图分量(使用 BFS"""
pages = await self.db.get_all_pages()
page_paths = {page.path for page in pages}
if not page_paths:
return []
visited = set()
components = []
for start_path in page_paths:
if start_path in visited:
continue
# BFS 构建连通分量
component = set()
queue = [start_path]
while queue:
current = queue.pop(0)
if current in visited:
continue
visited.add(current)
component.add(current)
# 添加出链和入链
for link in await self.db.get_links(current):
if link in page_paths and link not in visited:
queue.append(link)
for backlink in await self.db.get_backlinks(current):
if backlink in page_paths and backlink not in visited:
queue.append(backlink)
components.append(component)
# 返回最大的分量以外的所有分量(即孤立的子图)
main_component = max(components, key=len)
return [c for c in components if c != main_component]
+226
View File
@@ -0,0 +1,226 @@
"""
Service Layer - IndexerService(索引服务)
管理 wiki 页面索引,支持增量更新和全量重建。
参考设计文档:第 2.3 节
"""
import logging
import asyncio
from pathlib import Path
from typing import List, Set, Optional
from datetime import datetime
from ..storage import Database, WikiPage, compute_content_hash
from .parser import ParserService
logger = logging.getLogger(__name__)
class IndexerService:
"""索引服务 - 管理页面索引"""
def __init__(self, db: Database, parser: ParserService, wiki_vault_path: str):
self.db = db
self.parser = parser
self.wiki_vault_path = Path(wiki_vault_path)
self._lock = asyncio.Lock()
async def index_page(self, page_path: str) -> Optional[WikiPage]:
"""索引单个页面"""
try:
full_path = self.wiki_vault_path / page_path
# 读取文件内容
with open(full_path, 'r', encoding='utf-8') as f:
content = f.read()
# 解析页面
parsed = self.parser.parse_wiki_page(page_path, content)
# 计算内容哈希
content_hash = compute_content_hash(content)
# 检查是否需要更新
stored_hash = await self.db.get_page_hash(page_path)
if stored_hash == content_hash:
logger.debug(f"Page unchanged, skipping: {page_path}")
return None
# 创建 WikiPage 对象
page = WikiPage(
path=parsed["path"],
title=parsed["title"],
category=parsed["category"],
tags=parsed["tags"],
summary=parsed["summary"],
content_hash=content_hash,
lifecycle="draft", # 默认为 draft,后续可通过 wiki_lint 提升
source_tool=parsed["source_tool"],
created_at=parsed["created_at"],
updated_at=parsed["updated_at"],
indexed_at=parsed["indexed_at"]
)
# 更新数据库
await self.db.upsert_page(page)
# 更新 FTS5 索引
await self.db.update_fts_content(
page_path,
page.title,
content,
page.summary
)
# 更新链接关系
await self.db.delete_links(page_path)
for link in parsed.get("links", []):
await self.db.upsert_link(page_path, link)
# 更新标签
for tag in page.tags:
await self.db.upsert_tag(tag)
logger.info(f"Indexed page: {page_path}")
return page
except Exception as e:
logger.error(f"Failed to index page {page_path}: {e}")
return None
async def index_batch(self, page_paths: List[str]) -> dict:
"""批量索引页面"""
results = {
"success": 0,
"failed": 0,
"skipped": 0,
"pages": []
}
for page_path in page_paths:
try:
page = await self.index_page(page_path)
if page:
results["success"] += 1
results["pages"].append(page_path)
else:
results["skipped"] += 1
except Exception as e:
logger.error(f"Failed to index {page_path}: {e}")
results["failed"] += 1
return results
async def get_dirty_pages(self) -> List[str]:
"""获取需要重新索引的页面(增量更新)"""
all_pages = self.scan_wiki_vault()
dirty_pages = []
for page_path in all_pages:
full_path = self.wiki_vault_path / page_path
try:
with open(full_path, 'r', encoding='utf-8') as f:
content = f.read()
current_hash = compute_content_hash(content)
stored_hash = await self.db.get_page_hash(page_path)
if stored_hash != current_hash:
dirty_pages.append(page_path)
except Exception as e:
logger.warning(f"Failed to check {page_path}: {e}")
return dirty_pages
async def incremental_update(self) -> dict:
"""增量更新索引 - 只处理变化的页面"""
logger.info("Starting incremental update...")
async with self._lock:
# 1. 获取所有 wiki 页面
all_pages = self.scan_wiki_vault()
# 2. 检查每个页面的哈希
for page_path in all_pages:
try:
full_path = self.wiki_vault_path / page_path
with open(full_path, 'r', encoding='utf-8') as f:
content = f.read()
current_hash = compute_content_hash(content)
stored_hash = await self.db.get_page_hash(page_path)
if stored_hash != current_hash:
await self.index_page(page_path)
except Exception as e:
logger.error(f"Failed to update {page_path}: {e}")
# 3. 处理删除的页面
indexed_paths = await self.db.get_all_indexed_paths()
for path in indexed_paths:
if path not in all_pages:
await self.db.delete_page(path)
await self.db.delete_fts_content(path)
logger.info(f"Deleted from index: {path}")
# 更新元数据
await self.db.set_meta("last_incremental_update", datetime.now().isoformat())
stats = await self.db.get_stats()
logger.info(f"Incremental update completed: {stats}")
return stats
async def rebuild_index(self) -> dict:
"""全量重建索引"""
logger.info("Starting full index rebuild...")
async with self._lock:
# 清空现有索引
await self.db.execute("DELETE FROM wiki_pages")
await self.db.execute("DELETE FROM wiki_content")
await self.db.execute("DELETE FROM wiki_fts")
await self.db.execute("DELETE FROM wiki_links")
await self.db.execute("DELETE FROM wiki_tags")
await self.db.execute("DELETE FROM wiki_page_tags")
logger.info("Cleared existing index")
# 扫描并索引所有页面
all_pages = self.scan_wiki_vault()
results = await self.index_batch(all_pages)
# 更新元数据
await self.db.set_meta("last_full_reindex", datetime.now().isoformat())
stats = await self.db.get_stats()
logger.info(f"Full rebuild completed: {stats}")
return {"results": results, "stats": stats}
def scan_wiki_vault(self) -> List[str]:
"""扫描 wiki vault,返回所有 .md 文件的相对路径"""
if not self.wiki_vault_path.exists():
logger.error(f"Wiki vault path does not exist: {self.wiki_vault_path}")
return []
markdown_files = list(self.wiki_vault_path.rglob("*.md"))
relative_paths = []
for md_file in markdown_files:
try:
relative_path = md_file.relative_to(self.wiki_vault_path).as_posix()
relative_paths.append(relative_path)
except ValueError:
logger.warning(f"Failed to get relative path for: {md_file}")
return sorted(relative_paths)
async def get_index_status(self) -> dict:
"""获取索引状态"""
stats = await self.db.get_stats()
dirty_pages = await self.get_dirty_pages()
return {
"total_pages": stats.get("total_pages", 0),
"total_links": stats.get("total_links", 0),
"total_tags": stats.get("total_tags", 0),
"last_indexed": stats.get("last_indexed"),
"dirty_pages": len(dirty_pages),
"dirty_page_list": dirty_pages[:10] # 只返回前 10 个
}
+239
View File
@@ -0,0 +1,239 @@
"""
Service Layer - ParserService(解析服务)
解析 Markdown 文件,提取 frontmatter、链接、标签等。
参考设计文档:第 2.3 节
"""
import re
import logging
import json
from typing import List, Dict, Optional, Tuple
from datetime import datetime
from pathlib import Path
logger = logging.getLogger(__name__)
class ParserService:
"""解析服务 - 解析 Markdown 文件"""
# Frontmatter 正则
FRONTMATTER_PATTERN = re.compile(r'^---\s*\n(.*?)\n---\s*\n', re.DOTALL)
# Wikilink 正则
WIKILINK_PATTERN = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]')
# Tag 正则(frontmatter 中的 tags 字段)
TAG_PATTERN = re.compile(r'tags:\s*\[(.*?)\]')
def parse_frontmatter(self, content: str) -> Dict:
"""
解析 frontmatter
返回格式:
{
"name": "...",
"description": "...",
"metadata": {...},
"custom": {...} # 自定义字段
}
"""
frontmatter = {}
match = self.FRONTMATTER_PATTERN.match(content)
if match:
yaml_content = match.group(1)
# 简单解析(对于复杂情况,应该使用 PyYAML)
for line in yaml_content.split('\n'):
if ':' in line:
key, value = line.split(':', 1)
key = key.strip()
value = value.strip()
frontmatter[key] = value
return frontmatter
def extract_links(self, content: str) -> List[str]:
"""
提取所有 [[wikilinks]]
返回链接目标列表(去重)
"""
links = self.WIKILINK_PATTERN.findall(content)
return list(set(links))
def extract_tags(self, content: str) -> List[str]:
"""
从 frontmatter 中提取标签
支持:
- tags: ["tag1", "tag2"]
- tags: [tag1, tag2]
"""
tags = []
match = self.TAG_PATTERN.search(content)
if match:
tag_content = match.group(1)
# 尝试 JSON 解析
try:
tags = json.loads(f"[{tag_content}]")
except:
# 简单逗号分隔
tags = [t.strip().strip('"\'') for t in tag_content.split(',')]
return tags
def extract_title(self, content: str) -> str:
"""
提取标题(优先级:frontmatter > 第一个 # 标题)
返回标题或空字符串
"""
# 先检查 frontmatter
frontmatter = self.parse_frontmatter(content)
if "title" in frontmatter:
return frontmatter["title"]
# 检查第一个 # 标题
lines = content.split('\n')
for line in lines:
line = line.strip()
if line.startswith('#'):
# 移除 # 符号和空格
title = re.sub(r'^#+\s*', '', line)
return title
return ""
def extract_summary(self, content: str, max_length: int = 200) -> str:
"""
提取摘要(优先级:frontmatter description > 第一段文字)
限制在 max_length 字符内
"""
# 先检查 frontmatter
frontmatter = self.parse_frontmatter(content)
if "description" in frontmatter:
return frontmatter["description"][:max_length]
# 移除 frontmatter 和代码块
content_without_frontmatter = self.FRONTMATTER_PATTERN.sub('', content)
content_without_code = re.sub(r'```.*?```', '', content_without_frontmatter, flags=re.DOTALL)
# 获取第一段
lines = content_without_code.strip().split('\n\n')
for paragraph in lines:
paragraph = paragraph.strip()
if paragraph and not paragraph.startswith('#'):
# 移除 Markdown 格式
clean_paragraph = re.sub(r'[*_`#\[\]]', '', paragraph)
if clean_paragraph:
return clean_paragraph[:max_length]
return ""
def infer_category(self, path: str) -> str:
"""
从路径推断分类
例如:practices/xxx.md -> practices
"""
path_obj = Path(path)
if path_obj.parent.name:
return path_obj.parent.name
return "uncategorized"
def infer_source_tool(self, content: str) -> str:
"""
从内容推断来源工具
检查 frontmatter 中的特定字段:
- source_tool: claude/web_reader/gitea/other
"""
frontmatter = self.parse_frontmatter(content)
return frontmatter.get("source_tool", "other")
def validate_page(self, path: str, content: str) -> List[str]:
"""
验证页面,返回问题列表
检查:
- 是否有标题
- 是否有摘要
- 是否有分类
- frontmatter 是否有效
"""
issues = []
# 检查标题
title = self.extract_title(content)
if not title:
issues.append("Missing title")
# 检查摘要
summary = self.extract_summary(content)
if not summary:
issues.append("Missing summary")
# 检查分类
category = self.infer_category(path)
if category == "uncategorized":
issues.append("Uncategorized page")
# 检查 frontmatter
frontmatter = self.parse_frontmatter(content)
if not frontmatter:
issues.append("Missing or invalid frontmatter")
return issues
def parse_wiki_page(self, path: str, content: str, created_at: Optional[str] = None, updated_at: Optional[str] = None) -> Dict:
"""
完整解析 wiki 页面,返回结构化数据
返回格式与 WikiPage 兼容
"""
title = self.extract_title(content)
summary = self.extract_summary(content)
category = self.infer_category(path)
tags = self.extract_tags(content)
source_tool = self.infer_source_tool(content)
links = self.extract_links(content)
now = datetime.now().isoformat()
return {
"path": path,
"title": title,
"category": category,
"tags": tags,
"summary": summary,
"links": links,
"source_tool": source_tool,
"created_at": created_at or now,
"updated_at": updated_at or now,
"indexed_at": now
}
def format_wikilink(self, path: str, title: Optional[str] = None) -> str:
"""
格式化 wikilink
如果提供 title,使用 [[path|title]] 格式
否则使用 [[path]] 格式
"""
if title:
return f"[[{path}|{title}]]"
return f"[[{path}]]"
def resolve_wikilink_path(self, link: str, current_path: str) -> str:
"""
解析相对 wikilink 路径
例如:在 practices/a.md 中的 [[../concepts/x]] 解析为 concepts/x
"""
# TODO: 实现相对路径解析
return link
+120
View File
@@ -0,0 +1,120 @@
"""
Service Layer - QueryService查询服务
负责所有查询逻辑包括 FTS5 全文搜索标签搜索页面查询等
参考设计文档 2.3
"""
import logging
from typing import List, Set, Optional
from ..storage import Database, WikiPage
from .cache import CacheService
logger = logging.getLogger(__name__)
class QueryService:
"""查询服务 - 负责所有查询逻辑"""
def __init__(self, db: Database, cache: CacheService):
self.db = db
self.cache = cache
async def search(self, query: str, limit: int = 10) -> List[WikiPage]:
"""FTS5 全文搜索"""
# 1. 检查缓存
cache_key = f"search:{query}:{limit}"
cached = await self.cache.get(cache_key)
if cached:
logger.debug(f"Cache hit for search: {query}")
return cached
# 2. FTS5 搜索
results = await self.db.fts_search(query, limit)
# 3. 缓存结果
await self.cache.set(cache_key, results, ttl=3600)
return results
async def search_by_tags(self, tags: List[str]) -> List[WikiPage]:
"""按标签搜索"""
cache_key = f"tags:{','.join(tags)}"
cached = await self.cache.get(cache_key)
if cached:
return cached
results = await self.db.search_by_tags(tags)
await self.cache.set(cache_key, results, ttl=3600)
return results
async def search_by_source_tool(self, tool_name: str, limit: int = 50) -> List[WikiPage]:
"""按来源工具搜索(memory_bridge 使用)"""
cache_key = f"tool:{tool_name}:{limit}"
cached = await self.cache.get(cache_key)
if cached:
return cached
results = await self.db.search_by_source_tool(tool_name, limit)
await self.cache.set(cache_key, results, ttl=1800) # 30 分钟 TTL
return results
async def get_page(self, path: str) -> Optional[WikiPage]:
"""获取单个页面"""
cache_key = f"page:{path}"
cached = await self.cache.get(cache_key)
if cached:
return cached
page = await self.db.get_page(path)
if page:
await self.cache.set(cache_key, page, ttl=3600)
return page
async def get_links(self, path: str) -> Set[str]:
"""获取页面的出链"""
cache_key = f"links:{path}"
cached = await self.cache.get(cache_key)
if cached:
return cached
links = await self.db.get_links(path)
await self.cache.set(cache_key, links, ttl=3600)
return links
async def get_backlinks(self, path: str) -> Set[str]:
"""获取页面的反向链接"""
cache_key = f"backlinks:{path}"
cached = await self.cache.get(cache_key)
if cached:
return cached
backlinks = await self.db.get_backlinks(path)
await self.cache.set(cache_key, backlinks, ttl=3600)
return backlinks
async def find_orphans(self) -> Set[str]:
"""查找孤立页面(无反向链接)"""
cache_key = "orphans"
cached = await self.cache.get(cache_key)
if cached:
return cached
all_pages = await self.db.get_all_pages()
orphans = set()
for page in all_pages:
backlinks = await self.db.get_backlinks(page.path)
if not backlinks and page.path != "index.md":
orphans.add(page.path)
await self.cache.set(cache_key, orphans, ttl=1800)
return orphans
async def get_stats(self) -> dict:
"""获取索引统计信息"""
return await self.db.get_stats()
async def invalidate_cache(self, pattern: str = "*") -> int:
"""使缓存失效"""
return await self.cache.invalidate(pattern)
+14
View File
@@ -0,0 +1,14 @@
"""
Storage Layer - 数据存储层
提供数据库访问和缓存功能
"""
from .database import Database, WikiPage, fix_dirty_states, compute_content_hash
__all__ = [
"Database",
"WikiPage",
"fix_dirty_states",
"compute_content_hash"
]
+531
View File
@@ -0,0 +1,531 @@
"""
Storage Layer - Database
使用 aiosqlite 实现真正的异步支持配合 WAL 模式和并发保护
参考设计文档 2.4
"""
import aiosqlite
import asyncio
import logging
import time
from typing import List, Tuple, Dict, Set, Optional, Any
from datetime import datetime
from dataclasses import dataclass, asdict
import hashlib
import json
logger = logging.getLogger(__name__)
@dataclass
class WikiPage:
"""Wiki 页面模型"""
path: str
title: str
category: str
tags: List[str]
summary: str
content_hash: str
lifecycle: str
source_tool: str
created_at: str
updated_at: str
indexed_at: str
def is_stale(self, days: int = 90) -> bool:
"""检查页面是否过期"""
updated = datetime.fromisoformat(self.updated_at)
return (datetime.now() - updated).days > days
def to_dict(self) -> dict:
"""转换为字典"""
return asdict(self)
class Database:
"""SQLite 数据库封装,使用 aiosqlite 实现真正的异步支持"""
def __init__(self, path: str):
self.path = path
self._conn: Optional[aiosqlite.Connection] = None
self._lock = asyncio.Lock()
async def connect(self) -> None:
"""建立连接,启用 WAL 模式和并发保护"""
logger.info(f"Connecting to database: {self.path}")
self._conn = await aiosqlite.connect(self.path)
# 启用 WAL 模式提升并发性能
await self._conn.execute("PRAGMA journal_mode=WAL")
await self._conn.execute("PRAGMA busy_timeout=10000") # 10s
await self._conn.execute("PRAGMA synchronous=NORMAL")
await self._conn.commit()
# 创建表结构
await self._create_tables()
logger.info("Database connected and initialized")
async def _create_tables(self) -> None:
"""创建所有表结构"""
# 页面索引表
await self._conn.execute("""
CREATE TABLE IF NOT EXISTS wiki_pages (
path TEXT PRIMARY KEY,
title TEXT NOT NULL,
category TEXT,
tags TEXT,
summary TEXT,
content_hash TEXT NOT NULL,
lifecycle TEXT DEFAULT 'draft',
source_tool TEXT DEFAULT 'other',
created_at TIMESTAMP,
updated_at TIMESTAMP,
indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# 索引
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_category ON wiki_pages(category)")
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_lifecycle ON wiki_pages(lifecycle)")
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_updated ON wiki_pages(updated_at)")
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_source_tool ON wiki_pages(source_tool)")
# 内容表(FTS5 外部内容表)- 必须先创建
await self._conn.execute("""
CREATE TABLE IF NOT EXISTS wiki_content (
path TEXT PRIMARY KEY,
content TEXT NOT NULL
)
""")
# FTS5 全文搜索表
await self._conn.execute("""
CREATE VIRTUAL TABLE IF NOT EXISTS wiki_fts USING fts5(
path UNINDEXED,
title,
content,
summary,
content=wiki_content,
content_rowid=rowid,
tokenize = 'porter unicode61'
)
""")
# 链接关系表
await self._conn.execute("""
CREATE TABLE IF NOT EXISTS wiki_links (
source TEXT NOT NULL,
target TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (source, target)
)
""")
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_links_target ON wiki_links(target)")
# 标签索引表
await self._conn.execute("""
CREATE TABLE IF NOT EXISTS wiki_tags (
tag TEXT PRIMARY KEY,
count INTEGER DEFAULT 0,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# 页面-标签关联表
await self._conn.execute("""
CREATE TABLE IF NOT EXISTS wiki_page_tags (
path TEXT NOT NULL,
tag TEXT NOT NULL,
PRIMARY KEY (path, tag),
FOREIGN KEY (path) REFERENCES wiki_pages(path) ON DELETE CASCADE,
FOREIGN KEY (tag) REFERENCES wiki_tags(tag) ON DELETE CASCADE
)
""")
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_tag ON wiki_page_tags(tag)")
# 索引元数据表
await self._conn.execute("""
CREATE TABLE IF NOT EXISTS wiki_meta (
key TEXT PRIMARY KEY,
value TEXT,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
await self._conn.commit()
async def execute(self, sql: str, params: tuple = ()) -> aiosqlite.Cursor:
"""执行 SQL(带写入锁)"""
async with self._lock:
cursor = await self._conn.execute(sql, params)
await self._conn.commit()
return cursor
async def execute_with_retry(self, sql: str, params: tuple = (), max_retries: int = 2) -> aiosqlite.Cursor:
"""带重试的数据库操作(指数退避)"""
for attempt in range(max_retries):
try:
return await self.execute(sql, params)
except aiosqlite.OperationalError as e:
if "database is locked" in str(e) and attempt < max_retries - 1:
wait_time = 0.1 * (2 ** attempt)
logger.warning(f"Database locked, retrying in {wait_time}s (attempt {attempt + 1}/{max_retries})")
await asyncio.sleep(wait_time)
continue
raise
async def fetch_all(self, sql: str, params: tuple = ()) -> List[Tuple]:
"""查询所有结果(读操作无需锁,WAL 自动处理)"""
cursor = await self._conn.execute(sql, params)
return await cursor.fetchall()
async def fetch_one(self, sql: str, params: tuple = ()) -> Optional[Tuple]:
"""查询单个结果"""
rows = await self.fetch_all(sql, params)
return rows[0] if rows else None
# === 页面操作 ===
async def get_page(self, path: str) -> Optional[WikiPage]:
"""获取单个页面"""
row = await self.fetch_one(
"SELECT path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at FROM wiki_pages WHERE path = ?",
(path,)
)
if not row:
return None
return WikiPage(
path=row[0],
title=row[1],
category=row[2],
tags=json.loads(row[3]) if row[3] else [],
summary=row[4],
content_hash=row[5],
lifecycle=row[6],
source_tool=row[7],
created_at=row[8],
updated_at=row[9],
indexed_at=row[10]
)
async def get_all_pages(self) -> List[WikiPage]:
"""获取所有页面"""
rows = await self.fetch_all(
"SELECT path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at FROM wiki_pages"
)
return [
WikiPage(
path=row[0],
title=row[1],
category=row[2],
tags=json.loads(row[3]) if row[3] else [],
summary=row[4],
content_hash=row[5],
lifecycle=row[6],
source_tool=row[7],
created_at=row[8],
updated_at=row[9],
indexed_at=row[10]
)
for row in rows
]
async def get_page_hash(self, path: str) -> Optional[str]:
"""获取页面内容哈希"""
row = await self.fetch_one("SELECT content_hash FROM wiki_pages WHERE path = ?", (path,))
return row[0] if row else None
async def upsert_page(self, page: WikiPage) -> None:
"""插入或更新页面"""
await self.execute_with_retry("""
INSERT INTO wiki_pages (path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
title = excluded.title,
category = excluded.category,
tags = excluded.tags,
summary = excluded.summary,
content_hash = excluded.content_hash,
lifecycle = excluded.lifecycle,
source_tool = excluded.source_tool,
updated_at = excluded.updated_at,
indexed_at = excluded.indexed_at
""", (
page.path, page.title, page.category, json.dumps(page.tags),
page.summary, page.content_hash, page.lifecycle, page.source_tool,
page.created_at, page.updated_at, page.indexed_at
))
async def delete_page(self, path: str) -> None:
"""删除页面(级联删除相关数据)"""
await self.execute_with_retry("DELETE FROM wiki_pages WHERE path = ?", (path,))
async def get_all_indexed_paths(self) -> Set[str]:
"""获取所有已索引的页面路径"""
rows = await self.fetch_all("SELECT path FROM wiki_pages")
return {row[0] for row in rows}
# === FTS5 搜索 ===
async def fts_search(self, query: str, limit: int = 10) -> List[WikiPage]:
"""FTS5 全文搜索"""
# 先从 FTS5 获取匹配的路径
fts_rows = await self.fetch_all(
"SELECT path FROM wiki_fts WHERE wiki_fts MATCH ? ORDER BY rank LIMIT ?",
(query, limit)
)
paths = [row[0] for row in fts_rows]
if not paths:
return []
# 从 wiki_pages 获取完整数据
placeholders = ','.join('?' * len(paths))
rows = await self.fetch_all(
f"SELECT path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at FROM wiki_pages WHERE path IN ({placeholders})",
paths
)
return [
WikiPage(
path=row[0],
title=row[1],
category=row[2],
tags=json.loads(row[3]) if row[3] else [],
summary=row[4],
content_hash=row[5],
lifecycle=row[6],
source_tool=row[7],
created_at=row[8],
updated_at=row[9],
indexed_at=row[10]
)
for row in rows
]
async def search_by_tags(self, tags: List[str]) -> List[WikiPage]:
"""按标签搜索"""
if not tags:
return []
placeholders = ','.join('?' * len(tags))
rows = await self.fetch_all(
f"SELECT DISTINCT p.path, p.title, p.category, p.tags, p.summary, p.content_hash, p.lifecycle, p.source_tool, p.created_at, p.updated_at, p.indexed_at FROM wiki_pages p JOIN wiki_page_tags pt ON p.path = pt.path WHERE pt.tag IN ({placeholders})",
tags
)
return [
WikiPage(
path=row[0],
title=row[1],
category=row[2],
tags=json.loads(row[3]) if row[3] else [],
summary=row[4],
content_hash=row[5],
lifecycle=row[6],
source_tool=row[7],
created_at=row[8],
updated_at=row[9],
indexed_at=row[10]
)
for row in rows
]
async def search_by_source_tool(self, tool_name: str, limit: int = 50) -> List[WikiPage]:
"""按来源工具搜索(memory_bridge 使用)"""
rows = await self.fetch_all(
"SELECT path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at FROM wiki_pages WHERE source_tool = ? ORDER BY updated_at DESC LIMIT ?",
(tool_name, limit)
)
return [
WikiPage(
path=row[0],
title=row[1],
category=row[2],
tags=json.loads(row[3]) if row[3] else [],
summary=row[4],
content_hash=row[5],
lifecycle=row[6],
source_tool=row[7],
created_at=row[8],
updated_at=row[9],
indexed_at=row[10]
)
for row in rows
]
# === 链接操作 ===
async def get_links(self, path: str) -> Set[str]:
"""获取页面的出链"""
rows = await self.fetch_all("SELECT target FROM wiki_links WHERE source = ?", (path,))
return {row[0] for row in rows}
async def get_backlinks(self, path: str) -> Set[str]:
"""获取页面的反向链接"""
rows = await self.fetch_all("SELECT source FROM wiki_links WHERE target = ?", (path,))
return {row[0] for row in rows}
async def upsert_link(self, source: str, target: str) -> None:
"""插入或更新链接"""
await self.execute_with_retry(
"INSERT INTO wiki_links (source, target) VALUES (?, ?) ON CONFLICT(source, target) DO NOTHING",
(source, target)
)
async def delete_links(self, path: str) -> None:
"""删除页面的所有链接"""
await self.execute_with_retry("DELETE FROM wiki_links WHERE source = ?", (path,))
# === 标签操作 ===
async def upsert_tag(self, tag: str, count: int = 1) -> None:
"""插入或更新标签"""
await self.execute_with_retry(
"INSERT INTO wiki_tags (tag, count) VALUES (?, ?) ON CONFLICT(tag) DO UPDATE SET count = count + excluded.count",
(tag, count)
)
async def get_all_tags(self) -> Dict[str, int]:
"""获取所有标签"""
rows = await self.fetch_all("SELECT tag, count FROM wiki_tags")
return {row[0]: row[1] for row in rows}
# === 元数据操作 ===
async def get_meta(self, key: str, default: str = "") -> str:
"""获取元数据"""
row = await self.fetch_one("SELECT value FROM wiki_meta WHERE key = ?", (key,))
return row[0] if row else default
async def set_meta(self, key: str, value: str) -> None:
"""设置元数据"""
await self.execute_with_retry(
"INSERT INTO wiki_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value)
)
# === FTS5 内容更新 ===
async def update_fts_content(self, path: str, title: str, content: str, summary: str) -> None:
"""更新 FTS5 内容"""
# 先更新内容表
await self.execute_with_retry(
"INSERT INTO wiki_content (path, content) VALUES (?, ?) ON CONFLICT(path) DO UPDATE SET content = excluded.content",
(path, content)
)
# 再更新 FTS5 表(会自动从内容表同步)
await self.execute_with_retry(
"INSERT INTO wiki_fts (path, title, summary) VALUES (?, ?, ?) ON CONFLICT(path) DO UPDATE SET title = excluded.title, summary = excluded.summary",
(path, title, summary)
)
async def delete_fts_content(self, path: str) -> None:
"""删除 FTS5 内容"""
await self.execute_with_retry("DELETE FROM wiki_content WHERE path = ?", (path,))
await self.execute_with_retry("DELETE FROM wiki_fts WHERE path = ?", (path,))
# === 状态查询 ===
async def get_stats(self) -> Dict[str, Any]:
"""获取索引统计信息"""
page_count = await self.fetch_one("SELECT COUNT(*) FROM wiki_pages")
link_count = await self.fetch_one("SELECT COUNT(*) FROM wiki_links")
tag_count = await self.fetch_one("SELECT COUNT(*) FROM wiki_tags")
last_indexed = await self.fetch_one("SELECT MAX(indexed_at) FROM wiki_pages")
return {
"total_pages": page_count[0] if page_count else 0,
"total_links": link_count[0] if link_count else 0,
"total_tags": tag_count[0] if tag_count else 0,
"last_indexed": last_indexed[0] if last_indexed and last_indexed[0] else None
}
async def get_recent_pages(self, days: int = 7, limit: int = 50) -> List[WikiPage]:
"""获取最近更新的页面"""
rows = await self.fetch_all(
"SELECT path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at FROM wiki_pages WHERE updated_at >= datetime('now', '-' || ? || ' days') ORDER BY updated_at DESC LIMIT ?",
(str(days), limit)
)
return [
WikiPage(
path=row[0],
title=row[1],
category=row[2],
tags=json.loads(row[3]) if row[3] else [],
summary=row[4],
content_hash=row[5],
lifecycle=row[6],
source_tool=row[7],
created_at=row[8],
updated_at=row[9],
indexed_at=row[10]
)
for row in rows
]
async def get_new_tags(self, days: int = 7) -> List[Tuple[str, str]]:
"""获取新增标签"""
rows = await self.fetch_all(
"SELECT tag, updated_at FROM wiki_tags WHERE updated_at >= datetime('now', '-' || ? || ' days') ORDER BY updated_at DESC",
(str(days),)
)
return list(rows)
# === 完整性检查 ===
async def check_integrity(self) -> bool:
"""检查数据库完整性"""
try:
result = await self.fetch_one("PRAGMA integrity_check")
if result and result[0] == "ok":
return True
logger.warning(f"Database integrity check failed: {result}")
return False
except Exception as e:
logger.error(f"Integrity check error: {e}")
return False
# === 关闭连接 ===
async def close(self) -> None:
"""关闭连接"""
if self._conn:
await self._conn.close()
self._conn = None
logger.info("Database connection closed")
async def fix_dirty_states(db: Database) -> None:
"""启动时清理可能的脏状态"""
logger.info("Checking for dirty states...")
try:
# 1. 检查 WAL 文件是否损坏
if db._conn:
await db._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
await db._conn.commit()
# 2. 检查数据库完整性
if not await db.check_integrity():
raise Exception("Database integrity check failed")
logger.info("No dirty states found, database is healthy")
except Exception as e:
logger.warning(f"Dirty states detected, attempting recovery: {e}")
# TODO: 实现自动重建逻辑
# await rebuild_index(db)
def compute_content_hash(content: str) -> str:
"""计算内容 MD5 哈希"""
return hashlib.md5(content.encode('utf-8')).hexdigest()
+25
View File
@@ -0,0 +1,25 @@
"""
Tool Layer - 工具层
提供所有 MCP Tools
"""
from .wiki_query import WikiQueryTool
from .memory_bridge import MemoryBridgeTool
from .wiki_status import WikiStatusTool
from .wiki_lint import WikiLintTool
from .cross_linker import CrossLinkerTool
from .tag_taxonomy import TagTaxonomyTool
from .wiki_synthesize import WikiSynthesizeTool
from .daily_update import DailyUpdateTool
__all__ = [
"WikiQueryTool",
"MemoryBridgeTool",
"WikiStatusTool",
"WikiLintTool",
"CrossLinkerTool",
"TagTaxonomyTool",
"WikiSynthesizeTool",
"DailyUpdateTool"
]
+87
View File
@@ -0,0 +1,87 @@
"""
Tool Layer - cross_linker 工具
扫描 wiki自动发现缺失的交叉引用
参考设计文档 2.2
"""
import logging
from typing import Dict, Any, List
from ..services import GraphService
logger = logging.getLogger(__name__)
class CrossLinkerTool:
"""cross_linker 工具实现"""
def __init__(self, graph_service: GraphService):
self.graph_service = graph_service
async def handle(self, path: str = "", dry_run: bool = True) -> Dict[str, Any]:
"""
扫描 wiki发现缺失的交叉引用
Args:
path: 指定路径空字符串表示全部
dry_run: 是否为模拟运行不实际修改
Returns:
{
"missing_links": [
{
"source": "practices/a.md",
"target": "concepts/x",
"suggestion": "Consider creating [[concepts/x]]"
}
],
"total": 12,
"dry_run": true
}
"""
# 获取缺失的链接
all_missing = await self.graph_service.find_missing_links()
# 过滤指定路径
if path:
missing_links = [(s, t) for s, t in all_missing if s == path or t == path]
else:
missing_links = all_missing
# 格式化结果
missing_links_formatted = []
for source, target in missing_links:
missing_links_formatted.append({
"source": source,
"target": target,
"suggestion": f"Consider creating [[{target}]] or updating the link in [[{source}]]"
})
return {
"missing_links": missing_links_formatted[:50], # 最多返回 50 条
"total": len(missing_links),
"dry_run": dry_run
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "cross_linker",
"description": "扫描 wiki,自动发现缺失的交叉引用",
"inputSchema": {
"type": "object",
"properties": {
"path": {
"type": "string",
"default": "",
"description": "指定路径(空字符串表示全部)"
},
"dry_run": {
"type": "boolean",
"default": True,
"description": "是否为模拟运行(不实际修改)"
}
}
}
}
+105
View File
@@ -0,0 +1,105 @@
"""
Tool Layer - daily_update 工具
日常维护检查源新鲜度更新 index重新生成 hot.md
参考设计文档 2.2
"""
import logging
from typing import Dict, Any
from datetime import datetime
from pathlib import Path
from ..services import QueryService, IndexerService
logger = logging.getLogger(__name__)
class DailyUpdateTool:
"""daily_update 工具实现"""
def __init__(self, query_service: QueryService, indexer_service: IndexerService):
self.query_service = query_service
self.indexer_service = indexer_service
async def handle(self) -> Dict[str, Any]:
"""
日常维护
Returns:
{
"updated": 5,
"new": 2,
"hot_md_generated": true,
"index_status": {...}
}
"""
# 1. 增量更新索引
index_status = await self.indexer_service.incremental_update()
# 2. 获取最近更新的页面
recent_pages = await self.query_service.db.get_recent_pages(days=7, limit=20)
# 3. 获取新增标签
new_tags = await self.query_service.db.get_new_tags(days=7)
# 4. 查找孤立页面
from ..services import GraphService
graph_service = GraphService(self.query_service.db)
orphans = await graph_service.find_orphans()
# 5. 生成 hot.md
hot_md_generated = await self._generate_hot_md(recent_pages, new_tags, orphans)
return {
"updated": len(recent_pages),
"new": len(new_tags),
"hot_md_generated": hot_md_generated,
"index_status": index_status
}
async def _generate_hot_md(self, recent_pages, new_tags, orphans) -> bool:
"""生成热点文件"""
try:
# 确定 hot.md 保存路径
vault_path = Path(self.query_service.db.path).parent.parent / "wiki-vault"
hot_path = vault_path / "hot.md"
# 格式化内容
content = f"""# Wiki Hot - {datetime.now().strftime('%Y-%m-%d')}
## 最近更新(7 天内)
"""
for page in recent_pages[:10]:
content += f"- [[{page.path}|{page.title}]] - {page.updated_at}\n"
content += "\n## 新增标签(7 天内)\n\n"
for tag, updated_at in new_tags[:10]:
content += f"- `{tag}` - {updated_at}\n"
content += "\n## 待链接页面(孤立页面)\n\n"
for orphan in sorted(list(orphans))[:10]:
content += f"- [[{orphan}]]\n"
# 写入文件
with open(hot_path, 'w', encoding='utf-8') as f:
f.write(content)
logger.info(f"Generated hot.md: {hot_path}")
return True
except Exception as e:
logger.error(f"Failed to generate hot.md: {e}")
return False
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "daily_update",
"description": "日常维护(检查源新鲜度、更新 index、重新生成 hot.md",
"inputSchema": {
"type": "object",
"properties": {}
}
}
+112
View File
@@ -0,0 +1,112 @@
"""
Tool Layer - memory_bridge 工具
AI 工具来源浏览和对比 wiki 知识
参考设计文档 2.2
"""
import logging
from typing import Dict, Any, Optional
from datetime import datetime
from ..services import QueryService
logger = logging.getLogger(__name__)
class MemoryBridgeTool:
"""memory_bridge 工具实现"""
def __init__(self, query_service: QueryService):
self.query_service = query_service
async def handle(self, tool_name: str = "claude", date_range: str = "") -> Dict[str, Any]:
"""
AI 工具来源浏览和对比 wiki 知识
Args:
tool_name: AI 工具名称claude/web_reader/gitea/other
date_range: 日期范围 "2024-01-01:2024-12-31" 或空字符串表示不限
Returns:
{
"entries": [
{
"path": "practices/moziplus-orchestration.md",
"title": "moziplus 编排实践",
"summary": "...",
"updated_at": "2024-06-15T10:30:00",
"relevance_score": 0.85
}
],
"total": 12,
"tool_name": "claude",
"date_range": "2024-01-01:2024-12-31"
}
"""
# 按工具来源搜索
pages = await self.query_service.search_by_source_tool(tool_name, limit=100)
# 按日期范围过滤
if date_range:
try:
start_str, end_str = date_range.split(":")
start_date = datetime.fromisoformat(start_str)
end_date = datetime.fromisoformat(end_str)
filtered = []
for page in pages:
updated = datetime.fromisoformat(page.updated_at)
if start_date <= updated <= end_date:
filtered.append(page)
pages = filtered
except Exception as e:
logger.warning(f"Invalid date_range format '{date_range}': {e}")
# 格式化结果
entries = []
for page in pages:
# 计算相关性分数(基于摘要长度和更新时间)
recency_days = (datetime.now() - datetime.fromisoformat(page.updated_at)).days
relevance_score = max(0.1, 1.0 - recency_days / 365) # 简单衰减
entries.append({
"path": page.path,
"title": page.title,
"summary": page.summary,
"updated_at": page.updated_at,
"relevance_score": round(relevance_score, 2)
})
# 按相关性排序
entries.sort(key=lambda x: x["relevance_score"], reverse=True)
return {
"entries": entries[:50], # 最多返回 50 条
"total": len(entries),
"tool_name": tool_name,
"date_range": date_range
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "memory_bridge",
"description": "按 AI 工具来源浏览和对比 wiki 知识",
"inputSchema": {
"type": "object",
"properties": {
"tool_name": {
"type": "string",
"default": "claude",
"description": "AI 工具名称(claude/web_reader/gitea/other",
"enum": ["claude", "web_reader", "gitea", "other"]
},
"date_range": {
"type": "string",
"default": "",
"description": "日期范围,格式:YYYY-MM-DD:YYYY-MM-DD"
}
}
}
}
+117
View File
@@ -0,0 +1,117 @@
"""
Tool Layer - tag_taxonomy 工具
用受控词表强制标签一致性
参考设计文档 2.2
"""
import logging
from typing import Dict, Any, List, Set
from ..services import QueryService
logger = logging.getLogger(__name__)
class TagTaxonomyTool:
"""tag_taxonomy 工具实现"""
# 受控词表(可配置)
CONTROLLED_VOCABULARY: Set[str] = {
# 分类
"practices", "concepts", "entities", "projects", "skills", "references",
# 技术栈
"python", "typescript", "rust", "go", "swift",
# 领域
"ai", "ml", "database", "web", "mobile", "devops",
# 团队
"sanguo", "moziplus", "bitnet",
# 通用
"tutorial", "reference", "guide", "example", "draft"
}
def __init__(self, query_service: QueryService):
self.query_service = query_service
async def handle(self, path: str = "", enforce: bool = False) -> Dict[str, Any]:
"""
用受控词表强制标签一致性
Args:
path: 指定路径空字符串表示全部
enforce: 是否强制修正否则仅报告
Returns:
{
"conflicts": [
{
"path": "practices/example.md",
"invalid_tags": ["CustomTag"],
"suggested_tags": ["practices"]
}
],
"total_conflicts": 5,
"enforced": false
}
"""
conflicts = []
# 获取所有标签
all_tags = await self.query_service.db.get_all_tags()
# 检查非受控标签
invalid_tags = set(all_tags.keys()) - self.CONTROLLED_VOCABULARY
if invalid_tags:
# 获取使用这些标签的页面
for tag in invalid_tags:
tag_pages = await self.query_service.search_by_tags([tag])
for page in tag_pages:
page_tags = set(page.tags) & invalid_tags
if page_tags:
# 推断修正建议(基于页面分类)
suggested_tags = [page.category] if page.category else []
conflicts.append({
"path": page.path,
"invalid_tags": list(page_tags),
"suggested_tags": suggested_tags
})
if enforce and path and page.path == path:
# TODO: 实际修正标签
pass
return {
"conflicts": conflicts[:50],
"total_conflicts": len(conflicts),
"enforced": enforce
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "tag_taxonomy",
"description": "用受控词表强制标签一致性",
"inputSchema": {
"type": "object",
"properties": {
"path": {
"type": "string",
"default": "",
"description": "指定路径(空字符串表示全部)"
},
"enforce": {
"type": "boolean",
"default": False,
"description": "是否强制修正(否则仅报告)"
}
}
}
}
def get_controlled_vocabulary(self) -> List[str]:
"""获取当前受控词表"""
return sorted(list(self.CONTROLLED_VOCABULARY))
+164
View File
@@ -0,0 +1,164 @@
"""
Tool Layer - wiki_lint 工具
审计 wiki 健康格式链接frontmatter 规范
参考设计文档 2.2
"""
import logging
from typing import Dict, Any, List
from ..services import ParserService, QueryService, GraphService
logger = logging.getLogger(__name__)
class WikiLintTool:
"""wiki_lint 工具实现"""
def __init__(self, parser: ParserService, query_service: QueryService, graph_service: GraphService):
self.parser = parser
self.query_service = query_service
self.graph_service = graph_service
async def handle(self, path: str = "", level: str = "basic") -> Dict[str, Any]:
"""
审计 wiki 健康
Args:
path: 指定路径空字符串表示全部
level: 检查级别basic/strict
Returns:
{
"issues": [
{
"path": "practices/example.md",
"type": "missing_title",
"message": "Missing title",
"severity": "warning"
}
],
"fixes": [
{
"path": "practices/example.md",
"action": "add_title",
"suggestion": "# Example Page"
}
],
"summary": {
"total_issues": 5,
"critical": 0,
"warning": 5,
"info": 0
}
}
"""
issues = []
fixes = []
# 获取所有页面或指定页面
if path:
pages = [await self.query_service.get_page(path)]
else:
pages = await self.query_service.db.get_all_pages()
summary = {"critical": 0, "warning": 0, "info": 0}
for page in pages:
# 读取内容
try:
from pathlib import Path
vault_path = Path(self.query_service.db.path).parent.parent / "wiki-vault"
full_path = vault_path / page.path
with open(full_path, 'r', encoding='utf-8') as f:
content = f.read()
except Exception as e:
issues.append({
"path": page.path,
"type": "file_error",
"message": f"Failed to read file: {e}",
"severity": "critical"
})
summary["critical"] += 1
continue
# 验证页面
validation_issues = self.parser.validate_page(page.path, content)
for issue in validation_issues:
severity = "warning" if "Missing" in issue else "info"
issues.append({
"path": page.path,
"type": issue.lower().replace(" ", "_"),
"message": issue,
"severity": severity
})
summary[severity] += 1
# 生成修复建议
if "Missing title" in issue:
title = self.parser.extract_title(content)
if not title:
title = page.path.replace(".md", "").replace("-", " ").replace("_", " ").title()
fixes.append({
"path": page.path,
"action": "add_title",
"suggestion": f"# {title}"
})
# strict 模式额外检查
if level == "strict":
# 检查孤立页面
orphans = await self.graph_service.find_orphans()
for orphan in orphans:
issues.append({
"path": orphan,
"type": "orphan",
"message": "No backlinks found",
"severity": "info"
})
summary["info"] += 1
# 检查缺失链接
missing_links = await self.graph_service.find_missing_links()
for source, target in missing_links[:10]:
issues.append({
"path": source,
"type": "broken_link",
"message": f"Link to non-existent page: {target}",
"severity": "warning"
})
summary["warning"] += 1
return {
"issues": issues,
"fixes": fixes,
"summary": {
"total_issues": len(issues),
**summary
}
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "wiki_lint",
"description": "审计 wiki 健康(格式、链接、frontmatter 规范)",
"inputSchema": {
"type": "object",
"properties": {
"path": {
"type": "string",
"default": "",
"description": "指定路径(空字符串表示全部)"
},
"level": {
"type": "string",
"default": "basic",
"enum": ["basic", "strict"],
"description": "检查级别"
}
}
}
}
+116
View File
@@ -0,0 +1,116 @@
"""
Tool Layer - wiki_query 工具
基于 SQLite 索引查询 wiki支持关键词FTS5和标签搜索
参考设计文档 2.2
"""
import logging
from typing import List, Dict, Any
from ..services import QueryService
logger = logging.getLogger(__name__)
class WikiQueryTool:
"""wiki_query 工具实现"""
def __init__(self, query_service: QueryService):
self.query_service = query_service
async def handle(self, query: str = "", tags: List[str] = None, limit: int = 10) -> Dict[str, Any]:
"""
查询 wiki 页面
Args:
query: 查询关键词FTS5 全文搜索
tags: 标签过滤
limit: 返回数量限制
Returns:
{
"results": [
{
"path": "practices/moziplus-orchestration.md",
"title": "moziplus 编排实践",
"category": "practices",
"tags": ["moziplus", "orchestration"],
"summary": "...",
"lifecycle": "verified",
"updated_at": "2024-06-15T10:30:00",
"wikilink": "[[practices/moziplus-orchestration.md|moziplus 编排实践]]"
}
],
"total": 5,
"query": "...",
"tags": [...]
}
"""
if tags is None:
tags = []
results = []
# 组合查询
if query and tags:
# 先 FTS5 搜索,再标签过滤
fts_results = await self.query_service.search(query, limit * 2)
filtered = [r for r in fts_results if any(t in r.tags for t in tags)]
pages = filtered[:limit]
elif query:
# 仅 FTS5 搜索
pages = await self.query_service.search(query, limit)
elif tags:
# 仅标签搜索
pages = await self.query_service.search_by_tags(tags[:5]) # 限制标签数量
pages = pages[:limit]
else:
# 无查询条件,返回最近更新的页面
pages = []
# 格式化结果
for page in pages:
results.append({
"path": page.path,
"title": page.title,
"category": page.category,
"tags": page.tags,
"summary": page.summary,
"lifecycle": page.lifecycle,
"updated_at": page.updated_at,
"wikilink": f"[[{page.path}|{page.title}]]"
})
return {
"results": results,
"total": len(results),
"query": query,
"tags": tags
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "wiki_query",
"description": "查询 wiki 页面,支持关键词(FTS5)和标签搜索",
"inputSchema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "查询关键词(FTS5 全文搜索)"
},
"tags": {
"type": "array",
"items": {"type": "string"},
"description": "标签过滤"
},
"limit": {
"type": "integer",
"default": 10,
"description": "返回数量限制"
}
}
}
}
+75
View File
@@ -0,0 +1,75 @@
"""
Tool Layer - wiki_status 工具
显示 wiki 当前状态页面数待处理项增量差异
参考设计文档 2.2
"""
import logging
from typing import Dict, Any
from ..services import QueryService, IndexerService, GraphService
logger = logging.getLogger(__name__)
class WikiStatusTool:
"""wiki_status 工具实现"""
def __init__(self, query_service: QueryService, indexer_service: IndexerService, graph_service: GraphService):
self.query_service = query_service
self.indexer_service = indexer_service
self.graph_service = graph_service
async def handle(self) -> Dict[str, Any]:
"""
显示 wiki 当前状态
Returns:
{
"stats": {
"total_pages": 1353,
"total_links": 4200,
"total_tags": 156,
"last_indexed": "2024-06-26T10:30:00"
},
"orphans": ["path1", "path2"],
"orphans_count": 5,
"dirty_pages": 12,
"cache_stats": {
"size": 100,
"max_size": 1000,
"utilization": 0.1
}
}
"""
# 获取统计信息
stats = await self.query_service.get_stats()
# 获取孤立页面
orphans = await self.graph_service.find_orphans()
# 获取索引状态
index_status = await self.indexer_service.get_index_status()
# 获取缓存统计
cache_stats = await self.query_service.cache.get_stats()
return {
"stats": stats,
"orphans": sorted(list(orphans)),
"orphans_count": len(orphans),
"dirty_pages": index_status.get("dirty_pages", 0),
"cache_stats": cache_stats
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "wiki_status",
"description": "显示 wiki 当前状态(页面数、待处理项、增量差异)",
"inputSchema": {
"type": "object",
"properties": {}
}
}
+105
View File
@@ -0,0 +1,105 @@
"""
Tool Layer - wiki_synthesize 工具
发现跨概念的综合分析机会生成 synthesis 页面
参考设计文档 2.2
"""
import logging
from typing import Dict, Any, List
from ..services import QueryService, GraphService
logger = logging.getLogger(__name__)
class WikiSynthesizeTool:
"""wiki_synthesize 工具实现"""
def __init__(self, query_service: QueryService, graph_service: GraphService):
self.query_service = query_service
self.graph_service = graph_service
async def handle(self, concepts: List[str] = None, threshold: float = 0.3) -> Dict[str, Any]:
"""
发现跨概念的综合分析机会
Args:
concepts: 概念列表空表示自动发现
threshold: 相关性阈值
Returns:
{
"synthesis": [
{
"concept_a": "SQLite",
"concept_b": "Concurrency",
"common_references": ["page1", "page2"],
"strength": 0.8,
"suggestion": "Consider creating synthesis page: SQLite & Concurrency"
}
],
"total": 5
}
"""
synthesis = []
# 如果未提供概念,自动发现高相关性概念
if not concepts:
# 获取高度连接的页面作为候选
highly_connected = await self.graph_service.get_highly_connected_pages(threshold=10)
concepts = [item["title"] for item in highly_connected[:10]]
# 分析概念间的关联
for i, concept_a in enumerate(concepts):
for concept_b in concepts[i + 1:]:
# 查找同时引用两个概念的页面
pages_a = await self.query_service.search(concept_a, limit=50)
pages_b = await self.query_service.search(concept_b, limit=50)
paths_a = {p.path for p in pages_a}
paths_b = {p.path for p in pages_b}
common = paths_a & paths_b
if len(common) >= 2: # 至少 2 个共同引用
strength = len(common) / max(len(paths_a), len(paths_b))
if strength >= threshold:
synthesis.append({
"concept_a": concept_a,
"concept_b": concept_b,
"common_references": sorted(list(common)),
"strength": round(strength, 2),
"suggestion": f"Consider creating synthesis page: {concept_a} & {concept_b}"
})
# 按关联强度排序
synthesis.sort(key=lambda x: x["strength"], reverse=True)
return {
"synthesis": synthesis[:20],
"total": len(synthesis)
}
def get_schema(self) -> dict:
"""返回 MCP Tool schema"""
return {
"name": "wiki_synthesize",
"description": "发现跨概念的综合分析机会,生成 synthesis 页面",
"inputSchema": {
"type": "object",
"properties": {
"concepts": {
"type": "array",
"items": {"type": "string"},
"description": "概念列表(空表示自动发现)"
},
"threshold": {
"type": "number",
"default": 0.3,
"description": "相关性阈值"
}
}
}
}