知识库搜索优化:列表项二次切分,提升细粒度匹配精度

This commit is contained in:
youpaishidifu
2026-08-03 17:31:45 +08:00
parent 24036ecab9
commit c3a210efef
+86 -1
View File
@@ -114,6 +114,7 @@ class KnowledgeBase:
# 按行首的 ## 标题切分(前面可能有空白)
parts = re.split(r'(?=^#{1,6}\s)', content, flags=re.MULTILINE)
section_id = 0
for i, part in enumerate(parts):
part_stripped = part.strip()
if not part_stripped or len(part_stripped) < 5:
@@ -122,12 +123,30 @@ class KnowledgeBase:
lines = part_stripped.split('\n')
heading = lines[0].strip()
body = '\n'.join(lines[1:]).strip() if len(lines) > 1 else ""
# 检测body中的列表项,进行二次切分
# 列表项以 "- " 开头,每个列表项作为独立子section
if body:
# 按列表项切分
list_sections = KnowledgeBase._split_body_by_list_items(body, heading)
if list_sections:
for ls in list_sections:
sections.append({
"heading": ls["heading"],
"content": ls["content"],
"body": ls["body"],
"section_id": section_id,
})
section_id += 1
continue
sections.append({
"heading": heading,
"content": part_stripped,
"body": body,
"section_id": i,
"section_id": section_id,
})
section_id += 1
# 如果没有切分出多个 section,整个文档作为一段
if len(sections) <= 1:
@@ -140,6 +159,72 @@ class KnowledgeBase:
return sections
@staticmethod
def _split_body_by_list_items(body: str, parent_heading: str) -> List[dict]:
"""将body中的列表项切分为独立子section。
列表项以 "- " 开头,每个列表项及其后续内容作为独立子section。
返回子section列表,如果列表项少于2个则返回空列表(不切分)。
"""
lines = body.split('\n')
list_items = []
current_item = []
current_item_start = -1
for i, line in enumerate(lines):
stripped = line.strip()
if stripped.startswith('- ') and len(stripped) > 2:
# 新的列表项开始
if current_item:
list_items.append({
'start': current_item_start,
'end': i - 1,
'text': '\n'.join(current_item).strip(),
'title_line': current_item[0][2:].strip() # 去掉 "- " 前缀
})
current_item = [line]
current_item_start = i
elif current_item:
current_item.append(line)
# 处理最后一个列表项
if current_item:
list_items.append({
'start': current_item_start,
'end': len(lines) - 1,
'text': '\n'.join(current_item).strip(),
'title_line': current_item[0][2:].strip()
})
# 如果列表项少于2个,不切分
if len(list_items) < 2:
return []
sections = []
for item in list_items:
# 提取列表项的标题(前20个字符)
title_line = item['title_line']
# 去掉可能的链接和图片标记
title_clean = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', title_line)
title_clean = re.sub(r'!\[[^\]]*\]\([^\)]+\)', '', title_clean)
title_clean = title_clean.strip()
# 截取标题(最多30字符)
if len(title_clean) > 30:
title_clean = title_clean[:30] + '...'
# 构建子section内容
sub_content = item['text']
sub_heading = f"{parent_heading} - {title_clean}"
sections.append({
"heading": sub_heading,
"content": sub_content,
"body": sub_content,
})
return sections
def set_path(self, path: str):
self.root_path = path
self.reload()