fix: 修复OCR消息被误判为bot回复碎片的问题 (Task #1654)
问题:'关于我们'、'服务器基本信息'等短查询被误判为bot回复碎片而丢弃 根因:bot回复碎片检测逻辑过于宽松,字符重叠率≥50%且cjk≥2就判定为碎片 修复: 1. 添加问句豁免:包含疑问词或以问号结尾的消息跳过碎片检测 2. 添加前缀豁免:短消息(<10字符)如果是已回复文本的前缀,视为用户查询 3. 调整策略1阈值:最长公共子串占比≥70%且≥5字符(原≥60%且≥3字符) 4. 调整策略2阈值:字符重叠率≥60%且cjk≥3(原≥50%且cjk≥2) 5. 策略3要求文本长度≥5(防止短消息误判)
This commit is contained in:
+25
-5
@@ -10182,9 +10182,14 @@ class MessageMonitor:
|
|||||||
# ⭐ 新增:bot回复碎片检测(在基线轮判断之前过滤,防止OCR残片入库)
|
# ⭐ 新增:bot回复碎片检测(在基线轮判断之前过滤,防止OCR残片入库)
|
||||||
# 如果文本与已回复内容高度重叠(字符重叠率≥50%),直接丢弃
|
# 如果文本与已回复内容高度重叠(字符重叠率≥50%),直接丢弃
|
||||||
# 例:"端0(基岩版)" vs 已回复表格含"端口(基岩版) | 30679" → "基岩版"3字重叠 → 拦截
|
# 例:"端0(基岩版)" vs 已回复表格含"端口(基岩版) | 30679" → "基岩版"3字重叠 → 拦截
|
||||||
|
# ⭐ Task #1654 修复:避免误判用户查询(如"服务器基本信息"、"关于我们")为回复碎片
|
||||||
|
# 增加最小长度要求、提高重叠比例阈值、添加问句豁免
|
||||||
if self._replied_texts:
|
if self._replied_texts:
|
||||||
import difflib
|
import difflib
|
||||||
filtered_new_items = []
|
filtered_new_items = []
|
||||||
|
# ⭐ 问句豁免词列表(与_is_reply_fragment保持一致)
|
||||||
|
_ask_words = ('怎么办', '为什么', '能不能', '怎么', '如何', '什么', '可以',
|
||||||
|
'帮忙', '是啥', '是啥啊', '在哪', '哪个', '哪里', '多少', '几', '吗', '呢')
|
||||||
for item in new_items:
|
for item in new_items:
|
||||||
ts = item.strip()
|
ts = item.strip()
|
||||||
if len(ts) < 3:
|
if len(ts) < 3:
|
||||||
@@ -10192,30 +10197,45 @@ class MessageMonitor:
|
|||||||
continue
|
continue
|
||||||
is_fragment = False
|
is_fragment = False
|
||||||
t_norm = self._norm_frag(ts)
|
t_norm = self._norm_frag(ts)
|
||||||
|
# ⭐ 问句豁免:如果文本包含疑问词或以问号结尾,视为用户查询,跳过碎片检测
|
||||||
|
_is_ask = any(w in ts for w in _ask_words) or ts.rstrip().endswith(('?', '?'))
|
||||||
|
if _is_ask:
|
||||||
|
logger.debug(f"[{platform}] OCR过滤: 问句豁免,跳过碎片检测 ts={ts!r}")
|
||||||
|
filtered_new_items.append(item)
|
||||||
|
continue
|
||||||
for r in self._replied_texts:
|
for r in self._replied_texts:
|
||||||
if len(r) < 5:
|
if len(r) < 5:
|
||||||
continue
|
continue
|
||||||
r_norm = self._norm_frag(r)
|
r_norm = self._norm_frag(r)
|
||||||
if not t_norm or not r_norm:
|
if not t_norm or not r_norm:
|
||||||
continue
|
continue
|
||||||
# 策略1:最长公共子串占比≥60%且≥3字符(严格匹配)
|
# ⭐ 前缀豁免:如果文本是已回复文本的前缀(长度<10),可能是用户查询
|
||||||
|
if len(ts) < 10 and r.startswith(ts):
|
||||||
|
logger.debug(f"[{platform}] OCR过滤: 前缀豁免,跳过碎片检测 ts={ts!r} vs r={r[:60]!r}")
|
||||||
|
continue
|
||||||
|
# ⭐ 策略1:最长公共子串占比≥70%且≥5字符(更严格,避免短消息误判)
|
||||||
sm = difflib.SequenceMatcher(None, t_norm, r_norm)
|
sm = difflib.SequenceMatcher(None, t_norm, r_norm)
|
||||||
lm = sm.find_longest_match(0, len(t_norm), 0, len(r_norm)).size
|
lm = sm.find_longest_match(0, len(t_norm), 0, len(r_norm)).size
|
||||||
if lm >= 0.6 * len(t_norm) and lm >= 3:
|
if lm >= 0.7 * len(t_norm) and lm >= 5:
|
||||||
logger.debug(f"[{platform}] OCR过滤: bot回复碎片(lm={lm},ratio={lm/len(t_norm):.0%}) ts={ts!r}")
|
logger.debug(f"[{platform}] OCR过滤: bot回复碎片(lm={lm},ratio={lm/len(t_norm):.0%}) ts={ts!r}")
|
||||||
is_fragment = True
|
is_fragment = True
|
||||||
break
|
break
|
||||||
# 策略2:字符级重叠率≥50%且核心词命中(宽松匹配,防OCR噪声)
|
# ⭐ 策略2:字符级重叠率≥60%且至少有3个中文字符重叠(调整阈值)
|
||||||
# 计算t_norm中有多少字符出现在r_norm中
|
# 计算t_norm中有多少字符出现在r_norm中
|
||||||
r_char_set = set(r_norm)
|
r_char_set = set(r_norm)
|
||||||
overlap = sum(1 for c in t_norm if c in r_char_set)
|
overlap = sum(1 for c in t_norm if c in r_char_set)
|
||||||
overlap_ratio = overlap / len(t_norm) if t_norm else 0
|
overlap_ratio = overlap / len(t_norm) if t_norm else 0
|
||||||
# 如果重叠率≥50%且至少有2个中文字符重叠 → 可能是碎片
|
# 如果重叠率≥60%且至少有3个中文字符重叠 → 可能是碎片
|
||||||
cjk_overlap = sum(1 for c in t_norm if '\u4e00' <= c <= '\u9fff' and c in r_char_set)
|
cjk_overlap = sum(1 for c in t_norm if '\u4e00' <= c <= '\u9fff' and c in r_char_set)
|
||||||
if overlap_ratio >= 0.5 and cjk_overlap >= 2:
|
if overlap_ratio >= 0.6 and cjk_overlap >= 3:
|
||||||
logger.debug(f"[{platform}] OCR过滤: bot回复碎片(char_overlap={overlap}/{len(t_norm)},cjk={cjk_overlap}) ts={ts!r}")
|
logger.debug(f"[{platform}] OCR过滤: bot回复碎片(char_overlap={overlap}/{len(t_norm)},cjk={cjk_overlap}) ts={ts!r}")
|
||||||
is_fragment = True
|
is_fragment = True
|
||||||
break
|
break
|
||||||
|
# ⭐ 策略3:完全子串匹配(文本完全包含在已回复文本中)
|
||||||
|
if len(ts) >= 5 and any(ts in r for r in self._replied_texts):
|
||||||
|
logger.debug(f"[{platform}] OCR过滤: bot回复碎片(完全子串) ts={ts!r}")
|
||||||
|
is_fragment = True
|
||||||
|
break
|
||||||
if not is_fragment:
|
if not is_fragment:
|
||||||
filtered_new_items.append(item)
|
filtered_new_items.append(item)
|
||||||
new_items = filtered_new_items
|
new_items = filtered_new_items
|
||||||
|
|||||||
Reference in New Issue
Block a user