fix: 优化同轮语义去重——降低关键词重叠阈值至25%+动作词检测

- 骨架完全相同 → 跳过
- SequenceMatcher相似度>=70% → 跳过
- 关键词重叠>=25% 或 含核心动作词且重叠>=3 → 跳过

修复场景:"我想知道那个怎么撤销导入的兑换码" vs "那啥怎么去撤销那个导入过的激活码"
两消息共享动作词(撤销/导入)和4个关键词,判定为重复
This commit is contained in:
youpaishidifu
2026-08-30 22:12:01 +08:00
parent 4f378ee5b8
commit 59ae875acc
+25 -9
View File
@@ -10184,22 +10184,38 @@ class MessageMonitor:
# 使用骨架匹配快速判断
sk_cur = _cn_skeleton(_cleaned)
sk_prev = _cn_skeleton(_prev_cleaned)
if len(sk_cur) >= 6 and sk_prev and sk_cur == sk_prev:
logger.info(
f"[{platform}] 同轮语义去重: 骨架相同,跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
)
_is_same_round_dup = True
break
# Jaccard相似度兜底
if len(sk_cur) >= 4 and len(sk_prev) >= 4:
if len(sk_cur) >= 6 and sk_prev:
# 精确匹配
if sk_cur == sk_prev:
logger.info(
f"[{platform}] 同轮语义去重: 骨架相同,跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
)
_is_same_round_dup = True
break
# 高相似度匹配
from difflib import SequenceMatcher
sim = SequenceMatcher(None, sk_cur, sk_prev).ratio()
if sim >= 0.85:
if sim >= 0.7:
logger.info(
f"[{platform}] 同轮语义去重: 相似度{sim:.0%},跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
)
_is_same_round_dup = True
break
# 关键词重叠检测(针对长文本)
if len(sk_cur) >= 8 and len(sk_prev) >= 8:
kw_cur = {sk_cur[i:i+2] for i in range(len(sk_cur)-1)}
kw_prev = {sk_prev[i:i+2] for i in range(len(sk_prev)-1)}
overlap = kw_cur & kw_prev
total = kw_cur | kw_prev
# 重叠度>=25%或包含核心动作词则判定为重复
_action_kw = {'撤销', '导入', '兑换', '激活', '查询', '打印', '操作'}
has_action = bool(kw_cur & _action_kw) and bool(kw_prev & _action_kw)
if total and (len(overlap) / len(total) >= 0.25 or (has_action and len(overlap) >= 3)):
logger.info(
f"[{platform}] 同轮语义去重: 关键词重叠{len(overlap)}/{len(total)},跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
)
_is_same_round_dup = True
break
if _is_same_round_dup:
continue