fix: OCR口语词/碎片词污染核心词+下载全文回退+动作表格兜底
1. non_core_kw新增: 有人(@OCR误读)/一嘴/那家/那价 2. core_kw过滤纯ASCII<=2碎片词(ij等OCR噪声) 3. 下载意图新增全文回退: section无'登录'但整文档是登录插件(全文17次)则放行 4. 动作查询表格兜底: 查询'操作模式'vs表'操作方式'词级匹配失败时, 表格+action词命中放行
This commit is contained in:
+55
-1
@@ -4,6 +4,8 @@ import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import os
|
||||
import functools
|
||||
import jieba
|
||||
|
||||
logger = logging.getLogger("msgbot.autoreply")
|
||||
@@ -110,6 +112,16 @@ class AutoReply:
|
||||
toks = set(t.lower() for t in jieba.cut(text) if len(t) >= 2)
|
||||
return toks - AutoReply._STOPWORDS
|
||||
|
||||
@staticmethod
|
||||
@functools.lru_cache(maxsize=512)
|
||||
def _full_doc_text(path: str) -> str:
|
||||
"""读取知识库文档全文(缓存)。下载意图回退检查用——section粒度丢失跨节主题词。"""
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _has_content_relevance(query: str, doc: dict, score: float = 0.0) -> bool:
|
||||
"""查询词与匹配文档必须共享至少一个有区分度的核心词,防偶然/牛头不对马嘴匹配。
|
||||
@@ -178,6 +190,19 @@ class AutoReply:
|
||||
non_action_body = body_tokens - action_kw
|
||||
has_body_nonaction = bool(non_action_q & non_action_body)
|
||||
if not has_body_nonaction:
|
||||
# ⭐ 表格兜底(2026-08-17):词级匹配可能因近义词/OCR差异失败
|
||||
# 例:查询"操作模式" vs 表格"操作方式"——body含'操作'但已被action_kw剔除,
|
||||
# non_action交集为空 → 误杀 protect.md 的操作方式表格
|
||||
# 规则:heading主体命中(has_ht) + body是表格(|分隔) + 查询动作词在表格中出现 → 放行
|
||||
_is_table_body = ('|' in (body or '')) or ('-|-' in (body or ''))
|
||||
# 注意:不能用 set(body_clean)——字符集不包含多字符词,须用子串判断
|
||||
_action_hits_body = any(kw in body_clean[:800] for kw in (q_mean & action_kw))
|
||||
if has_ht and _is_table_body and _action_hits_body:
|
||||
logger.debug(
|
||||
f"动作查询表格兜底放行: query={query[:40]!r} "
|
||||
f"doc_heading={heading[:30]!r}"
|
||||
)
|
||||
else:
|
||||
# 正文(不含动作词)完全没有共同词 → 直接拒绝,无论 heading 是否命中
|
||||
logger.debug(
|
||||
f"动作查询正文无共同词拒绝: query={query[:40]!r} "
|
||||
@@ -280,8 +305,18 @@ class AutoReply:
|
||||
# 2026-08-17 新增:口语填充词"一句"不应作为核心词
|
||||
# 用户常说"我再问一句","一句"是填充词,不影响查询意图
|
||||
'一句', '问一句', '再问',
|
||||
# 2026-08-17 新增:OCR将@提及读成"有人"("有人@拜"实为@某人)
|
||||
# "有人"是口语占位/OCR噪声,无查询意图,类似"大家"
|
||||
'有人',
|
||||
# 2026-08-17 新增:"再问一嘴"的口语填充("问一嘴"已有,补"一嘴")
|
||||
'一嘴',
|
||||
# 2026-08-17 新增:口语"咱那家"的"那家"(OCR常写成"那价")
|
||||
'那家', '那价',
|
||||
}
|
||||
core_kw = q_mean - non_core_kw
|
||||
# ⭐ OCR碎片过滤:纯ASCII且≤2字符的词(如"丿IJ"切成'ij')是OCR噪声
|
||||
# 不参与核心词覆盖判定——否则垃圾词抬高分母,真核心词反而达不到阈值
|
||||
core_kw = {kw for kw in core_kw if not (kw.isascii() and len(kw) <= 2)}
|
||||
doc_text = heading_clean + " " + title.lower() + " " + body_clean[:800]
|
||||
core_hits = sum(1 for kw in core_kw if kw in doc_text)
|
||||
|
||||
@@ -307,10 +342,21 @@ class AutoReply:
|
||||
other_hits = sum(1 for kw in other_core_kw if kw in doc_text)
|
||||
if other_hits >= 1:
|
||||
_download_match = True
|
||||
else:
|
||||
# ⭐ 2026-08-17 全文回退:section 粒度丢失跨节主题词
|
||||
# sdf1_login.md "### 2.1 下载" 正文只有仓库链接(GitHub/gitee...)不含"登录",
|
||||
# 但整篇文档主题就是登录插件(全文"登录"×17)
|
||||
# beibao.md "## 下载地址" 全文"登录"×0 → 全文回退仍拒绝,防误匹配
|
||||
full_hits = sum(
|
||||
1 for kw in other_core_kw
|
||||
if kw in AutoReply._full_doc_text(doc.get('path', ''))
|
||||
)
|
||||
if full_hits >= 1:
|
||||
_download_match = True
|
||||
else:
|
||||
logger.debug(
|
||||
f"下载意图拒绝(核心词未命中): query={query[:40]!r} "
|
||||
f"doc_title={title!r} other_kw={other_core_kw!r} hits={other_hits}"
|
||||
f"doc_title={title!r} other_kw={other_core_kw!r} hits={other_hits} full_hits={full_hits}"
|
||||
)
|
||||
else:
|
||||
# 复合标题:检查标题主体部分(<br>或列表之前)是否含"下载"
|
||||
@@ -323,6 +369,14 @@ class AutoReply:
|
||||
other_hits = sum(1 for kw in other_core_kw if kw in doc_text)
|
||||
if other_hits >= 1:
|
||||
_download_match = True
|
||||
else:
|
||||
# 复合标题分支同样支持全文回退(见上方注释)
|
||||
full_hits = sum(
|
||||
1 for kw in other_core_kw
|
||||
if kw in AutoReply._full_doc_text(doc.get('path', ''))
|
||||
)
|
||||
if full_hits >= 1:
|
||||
_download_match = True
|
||||
if _download_match:
|
||||
logger.debug(f"下载意图放行: query={query[:30]!r} heading={heading[:30]!r}")
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user