fix(kb): 混合评分策略修复短查询匹配失败
TF-IDF 余弦相似度单独使用对短中文查询过于严格(3-5 个词 vs 数百个词的长文档)。 改为混合评分:0.4×TF-IDF + 0.6×关键词命中率,让短查询只要命中关键词就能返回结果。 同时降低 auto_reply_min_score 从 0.35→0.30,适配混合评分的数值范围。
This commit is contained in:
+30
-8
@@ -126,8 +126,17 @@ class KnowledgeBase:
|
||||
def is_ready(self) -> bool:
|
||||
return self._ready and len(self.docs) > 0
|
||||
|
||||
def search(self, query: str, top_k: int = 3, min_score: float = 0.30) -> List[Tuple[dict, float]]:
|
||||
"""搜索知识库,返回 (doc, score) 列表。"""
|
||||
def search(self, query: str, top_k: int = 3, min_score: float = 0.05) -> List[Tuple[dict, float]]:
|
||||
"""搜索知识库,返回 (doc, score) 列表。
|
||||
|
||||
混合评分策略:
|
||||
1) TF-IDF 余弦相似度(权重 0.4)— 适合长查询,对短查询偏严
|
||||
2) 关键词命中率(权重 0.6)— 适合短查询,对关键词命中敏感
|
||||
最终得分 = 0.4 * tfidf_score + 0.6 * keyword_hit_ratio
|
||||
|
||||
这种混合方式让短查询(3-5 个词)只要命中关键词就能返回结果,
|
||||
同时长查询仍能利用 TF-IDF 的语义区分能力。
|
||||
"""
|
||||
if not self.is_ready or not query.strip():
|
||||
return []
|
||||
|
||||
@@ -136,13 +145,27 @@ class KnowledgeBase:
|
||||
if len(query.strip()) < 2 or len(tokens) == 0:
|
||||
return []
|
||||
|
||||
query_vec = self._vectorizer.transform([query])
|
||||
scores = cosine_similarity(query_vec, self._tfidf_matrix)[0]
|
||||
top_indices = np.argsort(scores)[::-1][:top_k]
|
||||
query_stripped = query.strip()
|
||||
query_vec = self._vectorizer.transform([query_stripped])
|
||||
tfidf_scores = cosine_similarity(query_vec, self._tfidf_matrix)[0]
|
||||
|
||||
# 关键词命中率:查询 tokens 中有多少个出现在文档内容中
|
||||
query_tokens_set = set(tokens)
|
||||
keyword_ratios = []
|
||||
for doc in self.docs:
|
||||
content = doc["content"]
|
||||
hit = sum(1 for t in query_tokens_set if t in content)
|
||||
keyword_ratios.append(hit / max(len(query_tokens_set), 1))
|
||||
keyword_ratios = np.array(keyword_ratios)
|
||||
|
||||
# 混合评分
|
||||
combined_scores = 0.4 * tfidf_scores + 0.6 * keyword_ratios
|
||||
|
||||
top_indices = np.argsort(combined_scores)[::-1][:top_k]
|
||||
|
||||
results = []
|
||||
for idx in top_indices:
|
||||
score = float(scores[idx])
|
||||
score = float(combined_scores[idx])
|
||||
if score < min_score:
|
||||
continue
|
||||
results.append((self.docs[idx], score))
|
||||
@@ -150,8 +173,7 @@ class KnowledgeBase:
|
||||
return results
|
||||
|
||||
# Fallback: 子串匹配(查询包含在文档标题或内容中)
|
||||
q = query.strip()
|
||||
# 短文本(<4 字符)不做子串匹配,避免"19:15""14:30"等时间戳误匹配 KB 文档
|
||||
q = query_stripped
|
||||
if len(q) >= 4:
|
||||
for doc in self.docs:
|
||||
if q in doc["content"] or q in doc["title"]:
|
||||
|
||||
Reference in New Issue
Block a user