fix: 修复时间戳提问豁免无上限+知识库链接文字误匹配
Bug1: 提问豁免(_is_question_message)让继承极老时间戳的消息绕过过滤,
如22:07的消息在次日10:43仍通过stage splitting放行。
修复:给提问豁免加24h硬上限,继承时间戳>24h不再豁免。
Bug2: protect.md heading含[sdf1_login登录插件](...)链接,
heading_no_link_text保留链接文字,导致phrase_in_text匹配触发5x加分。
修复:新增full_text_phrase(移除整个链接含文字),
所有短语匹配改为用full_text_phrase,链接文字不再参与评分。
This commit is contained in:
+10
-7
@@ -522,16 +522,19 @@ class KnowledgeBase:
|
||||
body = doc.get("body", "")
|
||||
body_len = len(body.strip())
|
||||
heading = doc.get("heading", "")
|
||||
# 清理heading中的链接语法,保留链接文本
|
||||
# 清理heading中的链接语法,保留链接文本(用于TF-IDF/关键词匹配)
|
||||
heading_clean = _link_pattern.sub(r'\1', heading)
|
||||
# 进一步清理:去掉链接文本本身(如 "[sdf1_login登录插件]" → "")
|
||||
# 完整移除链接(含链接文本)(用于精确短语匹配,防止链接文字误匹配)
|
||||
# 如"[sdf1_login登录插件](sdf1_login.md)" → " ",防止"登录插件"链接文字触发5x加分
|
||||
heading_no_link_text = _link_pattern.sub('', heading)
|
||||
full_text_check = heading_clean + " " + body
|
||||
full_text_clean = heading_no_link_text + " " + body
|
||||
# ⭐ 短语匹配用:移除整个链接(含文字),防止链接文字误匹配
|
||||
full_text_phrase = heading_no_link_text + " " + body
|
||||
# 先计算正文含查询词情况
|
||||
body_hits = sum(1 for t in query_tokens_set if t in self._body_token_sets[i])
|
||||
# ⭐ 精确短语匹配检查(使用清理后的文本,排除链接文本)
|
||||
phrase_in_text = query_stripped in full_text_clean
|
||||
phrase_in_text = query_stripped in full_text_phrase
|
||||
# 单层 # 标题且无实质正文 → 大幅降权
|
||||
if heading.startswith("# ") and body_len < 50:
|
||||
combined_scores[i] *= 0.2
|
||||
@@ -545,8 +548,8 @@ class KnowledgeBase:
|
||||
combined_scores[i] *= 1.1
|
||||
# ⭐ 精确短语匹配:文档包含完整查询短语 → 巨幅奖励
|
||||
# 防止"php端的管理员密码"匹配到不含"php"的云背包文档
|
||||
# ⭐ 使用清理后的full_text(排除链接文本)进行短语匹配
|
||||
if query_stripped in full_text_clean:
|
||||
# ⭐ 使用full_text_phrase(排除链接文字),防止链接文字如"sdf1_login登录插件"误匹配
|
||||
if query_stripped in full_text_phrase:
|
||||
combined_scores[i] *= 5.0
|
||||
elif len(query_stripped) >= 4:
|
||||
# 2字以上子串匹配:如"php端"、"管理员密码"
|
||||
@@ -554,7 +557,7 @@ class KnowledgeBase:
|
||||
for qlen in range(min(len(query_stripped), 6), 3, -1):
|
||||
for start in range(len(query_stripped) - qlen + 1):
|
||||
phrase = query_stripped[start:start+qlen]
|
||||
if phrase in full_text_clean:
|
||||
if phrase in full_text_phrase:
|
||||
sub_hits += 1
|
||||
if sub_hits >= 2:
|
||||
combined_scores[i] *= 2.5
|
||||
@@ -564,7 +567,7 @@ class KnowledgeBase:
|
||||
# 防止"登录插件"被拆分为"登录"+"插件"分别匹配,导致错误文档得分更高
|
||||
core_phrases = ['登录插件', '资源包', '下载地址', '配置文件']
|
||||
for phrase in core_phrases:
|
||||
if phrase in query_stripped and phrase in full_text_clean:
|
||||
if phrase in query_stripped and phrase in full_text_phrase:
|
||||
combined_scores[i] *= 2.0
|
||||
break
|
||||
# ⭐ 否定式查询标记:查询问"是否X"时,在reranking阶段惩罚不含X的section
|
||||
|
||||
+4
-2
@@ -2872,7 +2872,8 @@ class MessageMonitor:
|
||||
# ⭐ 提问豁免:消息本身是明确提问(以?/?结尾、含中文、非md/噪声),
|
||||
# 即使继承的时间戳极老(来自截图内其他内容如历史/文档时间戳),
|
||||
# 也应放行交由下游去重,避免把用户刚发的提问误判为远古历史永久丢弃。
|
||||
if self._is_question_message(buf.strip()):
|
||||
# ⚠ 限制:继承时间戳>24h的老消息不再豁免(截图内远古历史不应触发回复)
|
||||
if self._is_question_message(buf.strip()) and not self._is_timestamp_old(last_ts, 86400):
|
||||
logger.debug(f"[ocr] stage[{i}] 继承时间戳{last_ts}极老但消息为明确提问,放行交由下游去重: {buf.strip()[:60]!r}")
|
||||
else:
|
||||
logger.debug(f"[ocr] stage[{i}] 继承时间戳{last_ts}极老(>2h)且非新鲜内容,视为远古历史跳过: {buf.strip()[:60]!r}")
|
||||
@@ -2912,7 +2913,8 @@ class MessageMonitor:
|
||||
if last_ts and self._is_timestamp_old(last_ts, 300):
|
||||
if self._is_timestamp_old(last_ts, 7200) and not _fresh_tail:
|
||||
# ⭐ 提问豁免(同中段逻辑):明确提问即使是尾部、继承极老时间戳也放行
|
||||
if self._is_question_message(buf.strip()):
|
||||
# ⚠ 限制:继承时间戳>24h的老消息不再豁免
|
||||
if self._is_question_message(buf.strip()) and not self._is_timestamp_old(last_ts, 86400):
|
||||
logger.debug(f"[ocr] stage[{i}] 尾部时间戳{last_ts}极老但消息为明确提问,放行交由下游去重: {buf.strip()[:60]!r}")
|
||||
else:
|
||||
logger.debug(f"[ocr] stage[{i}] 尾部时间戳{last_ts}极老(>2h)且非新鲜内容,视为远古历史跳过尾部: {buf.strip()[:60]!r}")
|
||||
|
||||
Reference in New Issue
Block a user