feat: 搜索改进与代码修复
- 连字符扩展：multi-agent → multi agent 提升搜索召回率 - 中文同义词支持：添加中英文混合查询扩展 - Snippet 高亮预留：为 FTS5 snippet 功能预留接口 - YAML 解析增强：支持 HTML 实体解码 - 标签关联维护：自动维护 wiki_page_tags 关联表 - 缓存失效优化：支持智能前缀匹配 - 设计文档更新：汇总近期改动 (v1.3) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -94,23 +94,13 @@ class Database:
|
||||
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_updated ON wiki_pages(updated_at)")
|
||||
await self._conn.execute("CREATE INDEX IF NOT EXISTS idx_pages_source_tool ON wiki_pages(source_tool)")
|
||||
|
||||
# 内容表（FTS5 外部内容表）- 必须先创建
|
||||
await self._conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS wiki_content (
|
||||
path TEXT PRIMARY KEY,
|
||||
content TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
# FTS5 全文搜索表
|
||||
# FTS5 全文搜索表（不使用外部内容表，简化结构）
|
||||
await self._conn.execute("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS wiki_fts USING fts5(
|
||||
path UNINDEXED,
|
||||
title,
|
||||
content,
|
||||
summary,
|
||||
content=wiki_content,
|
||||
content_rowid=rowid,
|
||||
tokenize = 'porter unicode61'
|
||||
)
|
||||
""")
|
||||
@@ -241,7 +231,8 @@ class Database:
|
||||
return row[0] if row else None
|
||||
|
||||
async def upsert_page(self, page: WikiPage) -> None:
|
||||
"""插入或更新页面"""
|
||||
"""插入或更新页面（包括标签关联）"""
|
||||
# 先更新页面基本信息
|
||||
await self.execute_with_retry("""
|
||||
INSERT INTO wiki_pages (path, title, category, tags, summary, content_hash, lifecycle, source_tool, created_at, updated_at, indexed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
@@ -261,6 +252,9 @@ class Database:
|
||||
page.created_at, page.updated_at, page.indexed_at
|
||||
))
|
||||
|
||||
# 更新标签关联
|
||||
await self.update_page_tags(page.path, page.tags)
|
||||
|
||||
async def delete_page(self, path: str) -> None:
|
||||
"""删除页面（级联删除相关数据）"""
|
||||
await self.execute_with_retry("DELETE FROM wiki_pages WHERE path = ?", (path,))
|
||||
@@ -314,8 +308,14 @@ class Database:
|
||||
|
||||
return query.strip()
|
||||
|
||||
async def fts_search(self, query: str, limit: int = 10) -> List[WikiPage]:
|
||||
"""FTS5 全文搜索"""
|
||||
async def fts_search(self, query: str, limit: int = 10, with_snippets: bool = False) -> List[WikiPage]:
|
||||
"""FTS5 全文搜索
|
||||
|
||||
Args:
|
||||
query: 搜索查询
|
||||
limit: 返回结果数量限制
|
||||
with_snippets: 是否返回匹配片段（暂未实现，预留接口）
|
||||
"""
|
||||
# 验证和清理查询
|
||||
try:
|
||||
safe_query = self._validate_fts_query(query)
|
||||
@@ -447,6 +447,24 @@ class Database:
|
||||
(tag, count)
|
||||
)
|
||||
|
||||
async def update_page_tags(self, path: str, tags: List[str]) -> None:
|
||||
"""更新页面的标签关联
|
||||
|
||||
先删除旧的标签关联，再插入新的。
|
||||
同时更新 wiki_tags 表中的计数。
|
||||
"""
|
||||
# 删除旧的标签关联
|
||||
await self.execute_with_retry("DELETE FROM wiki_page_tags WHERE path = ?", (path,))
|
||||
|
||||
# 插入新的标签关联
|
||||
for tag in tags:
|
||||
await self.execute_with_retry(
|
||||
"INSERT INTO wiki_page_tags (path, tag) VALUES (?, ?)",
|
||||
(path, tag)
|
||||
)
|
||||
# 更新标签计数
|
||||
await self.upsert_tag(tag)
|
||||
|
||||
async def get_all_tags(self) -> Dict[str, int]:
|
||||
"""获取所有标签"""
|
||||
rows = await self.fetch_all("SELECT tag, count FROM wiki_tags")
|
||||
@@ -469,22 +487,45 @@ class Database:
|
||||
# === FTS5 内容更新 ===
|
||||
|
||||
async def update_fts_content(self, path: str, title: str, content: str, summary: str) -> None:
|
||||
"""更新 FTS5 内容"""
|
||||
# 先更新内容表
|
||||
"""更新 FTS5 内容
|
||||
|
||||
FTS5 虚拟表不支持 UPSERT，使用 DELETE + INSERT 替代。
|
||||
所有内容（包括 content 字段）直接存储在 FTS 表中。
|
||||
|
||||
优化：连字符扩展（multi-agent → multi agent）以提升搜索召回率
|
||||
"""
|
||||
# 先删除旧记录（如果存在）
|
||||
await self.execute_with_retry("DELETE FROM wiki_fts WHERE path = ?", (path,))
|
||||
|
||||
# 扩展连字符词汇（multi-agent → multi agent）
|
||||
# 这样搜索 "multi" 或 "agent" 都能匹配到
|
||||
expanded_title = self._expand_hyphenated_text(title)
|
||||
expanded_summary = self._expand_hyphenated_text(summary)
|
||||
|
||||
# 插入扩展后的 title 和 summary（原版保留在 content 中）
|
||||
await self.execute_with_retry(
|
||||
"INSERT INTO wiki_content (path, content) VALUES (?, ?) ON CONFLICT(path) DO UPDATE SET content = excluded.content",
|
||||
(path, content)
|
||||
"INSERT INTO wiki_fts (path, title, summary, content) VALUES (?, ?, ?, ?)",
|
||||
(path, f"{title} {expanded_title}", f"{summary} {expanded_summary}", content)
|
||||
)
|
||||
|
||||
# 再更新 FTS5 表（会自动从内容表同步）
|
||||
await self.execute_with_retry(
|
||||
"INSERT INTO wiki_fts (path, title, summary) VALUES (?, ?, ?) ON CONFLICT(path) DO UPDATE SET title = excluded.title, summary = excluded.summary",
|
||||
(path, title, summary)
|
||||
)
|
||||
def _expand_hyphenated_text(self, text: str) -> str:
|
||||
"""扩展连字符词汇
|
||||
|
||||
multi-agent system → multi agent system
|
||||
这样搜索 "multi" 或 "agent" 都能匹配
|
||||
|
||||
Args:
|
||||
text: 原始文本
|
||||
|
||||
Returns:
|
||||
扩展后的文本（连字符替换为空格）
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
return text.replace('-', ' ')
|
||||
|
||||
async def delete_fts_content(self, path: str) -> None:
|
||||
"""删除 FTS5 内容"""
|
||||
await self.execute_with_retry("DELETE FROM wiki_content WHERE path = ?", (path,))
|
||||
await self.execute_with_retry("DELETE FROM wiki_fts WHERE path = ?", (path,))
|
||||
|
||||
# === 状态查询 ===
|
||||
|
||||
Reference in New Issue
Block a user