知识库搜索优化:列表项二次切分,提升细粒度匹配精度
This commit is contained in:
+86
-1
@@ -114,6 +114,7 @@ class KnowledgeBase:
|
||||
# 按行首的 ## 标题切分(前面可能有空白)
|
||||
parts = re.split(r'(?=^#{1,6}\s)', content, flags=re.MULTILINE)
|
||||
|
||||
section_id = 0
|
||||
for i, part in enumerate(parts):
|
||||
part_stripped = part.strip()
|
||||
if not part_stripped or len(part_stripped) < 5:
|
||||
@@ -122,12 +123,30 @@ class KnowledgeBase:
|
||||
lines = part_stripped.split('\n')
|
||||
heading = lines[0].strip()
|
||||
body = '\n'.join(lines[1:]).strip() if len(lines) > 1 else ""
|
||||
|
||||
# 检测body中的列表项,进行二次切分
|
||||
# 列表项以 "- " 开头,每个列表项作为独立子section
|
||||
if body:
|
||||
# 按列表项切分
|
||||
list_sections = KnowledgeBase._split_body_by_list_items(body, heading)
|
||||
if list_sections:
|
||||
for ls in list_sections:
|
||||
sections.append({
|
||||
"heading": ls["heading"],
|
||||
"content": ls["content"],
|
||||
"body": ls["body"],
|
||||
"section_id": section_id,
|
||||
})
|
||||
section_id += 1
|
||||
continue
|
||||
|
||||
sections.append({
|
||||
"heading": heading,
|
||||
"content": part_stripped,
|
||||
"body": body,
|
||||
"section_id": i,
|
||||
"section_id": section_id,
|
||||
})
|
||||
section_id += 1
|
||||
|
||||
# 如果没有切分出多个 section,整个文档作为一段
|
||||
if len(sections) <= 1:
|
||||
@@ -140,6 +159,72 @@ class KnowledgeBase:
|
||||
|
||||
return sections
|
||||
|
||||
@staticmethod
|
||||
def _split_body_by_list_items(body: str, parent_heading: str) -> List[dict]:
|
||||
"""将body中的列表项切分为独立子section。
|
||||
|
||||
列表项以 "- " 开头,每个列表项及其后续内容作为独立子section。
|
||||
返回子section列表,如果列表项少于2个则返回空列表(不切分)。
|
||||
"""
|
||||
lines = body.split('\n')
|
||||
list_items = []
|
||||
current_item = []
|
||||
current_item_start = -1
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith('- ') and len(stripped) > 2:
|
||||
# 新的列表项开始
|
||||
if current_item:
|
||||
list_items.append({
|
||||
'start': current_item_start,
|
||||
'end': i - 1,
|
||||
'text': '\n'.join(current_item).strip(),
|
||||
'title_line': current_item[0][2:].strip() # 去掉 "- " 前缀
|
||||
})
|
||||
current_item = [line]
|
||||
current_item_start = i
|
||||
elif current_item:
|
||||
current_item.append(line)
|
||||
|
||||
# 处理最后一个列表项
|
||||
if current_item:
|
||||
list_items.append({
|
||||
'start': current_item_start,
|
||||
'end': len(lines) - 1,
|
||||
'text': '\n'.join(current_item).strip(),
|
||||
'title_line': current_item[0][2:].strip()
|
||||
})
|
||||
|
||||
# 如果列表项少于2个,不切分
|
||||
if len(list_items) < 2:
|
||||
return []
|
||||
|
||||
sections = []
|
||||
for item in list_items:
|
||||
# 提取列表项的标题(前20个字符)
|
||||
title_line = item['title_line']
|
||||
# 去掉可能的链接和图片标记
|
||||
title_clean = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', title_line)
|
||||
title_clean = re.sub(r'!\[[^\]]*\]\([^\)]+\)', '', title_clean)
|
||||
title_clean = title_clean.strip()
|
||||
|
||||
# 截取标题(最多30字符)
|
||||
if len(title_clean) > 30:
|
||||
title_clean = title_clean[:30] + '...'
|
||||
|
||||
# 构建子section内容
|
||||
sub_content = item['text']
|
||||
sub_heading = f"{parent_heading} - {title_clean}"
|
||||
|
||||
sections.append({
|
||||
"heading": sub_heading,
|
||||
"content": sub_content,
|
||||
"body": sub_content,
|
||||
})
|
||||
|
||||
return sections
|
||||
|
||||
def set_path(self, path: str):
|
||||
self.root_path = path
|
||||
self.reload()
|
||||
|
||||
Reference in New Issue
Block a user