fix: 优化同轮语义去重——降低关键词重叠阈值至25%+动作词检测
- 骨架完全相同 → 跳过 - SequenceMatcher相似度>=70% → 跳过 - 关键词重叠>=25% 或 含核心动作词且重叠>=3 → 跳过 修复场景:"我想知道那个怎么撤销导入的兑换码" vs "那啥怎么去撤销那个导入过的激活码" 两消息共享动作词(撤销/导入)和4个关键词,判定为重复
This commit is contained in:
+25
-9
@@ -10184,22 +10184,38 @@ class MessageMonitor:
|
||||
# 使用骨架匹配快速判断
|
||||
sk_cur = _cn_skeleton(_cleaned)
|
||||
sk_prev = _cn_skeleton(_prev_cleaned)
|
||||
if len(sk_cur) >= 6 and sk_prev and sk_cur == sk_prev:
|
||||
logger.info(
|
||||
f"[{platform}] 同轮语义去重: 骨架相同,跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
|
||||
)
|
||||
_is_same_round_dup = True
|
||||
break
|
||||
# Jaccard相似度兜底
|
||||
if len(sk_cur) >= 4 and len(sk_prev) >= 4:
|
||||
if len(sk_cur) >= 6 and sk_prev:
|
||||
# 精确匹配
|
||||
if sk_cur == sk_prev:
|
||||
logger.info(
|
||||
f"[{platform}] 同轮语义去重: 骨架相同,跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
|
||||
)
|
||||
_is_same_round_dup = True
|
||||
break
|
||||
# 高相似度匹配
|
||||
from difflib import SequenceMatcher
|
||||
sim = SequenceMatcher(None, sk_cur, sk_prev).ratio()
|
||||
if sim >= 0.85:
|
||||
if sim >= 0.7:
|
||||
logger.info(
|
||||
f"[{platform}] 同轮语义去重: 相似度{sim:.0%},跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
|
||||
)
|
||||
_is_same_round_dup = True
|
||||
break
|
||||
# 关键词重叠检测(针对长文本)
|
||||
if len(sk_cur) >= 8 and len(sk_prev) >= 8:
|
||||
kw_cur = {sk_cur[i:i+2] for i in range(len(sk_cur)-1)}
|
||||
kw_prev = {sk_prev[i:i+2] for i in range(len(sk_prev)-1)}
|
||||
overlap = kw_cur & kw_prev
|
||||
total = kw_cur | kw_prev
|
||||
# 重叠度>=25%或包含核心动作词则判定为重复
|
||||
_action_kw = {'撤销', '导入', '兑换', '激活', '查询', '打印', '操作'}
|
||||
has_action = bool(kw_cur & _action_kw) and bool(kw_prev & _action_kw)
|
||||
if total and (len(overlap) / len(total) >= 0.25 or (has_action and len(overlap) >= 3)):
|
||||
logger.info(
|
||||
f"[{platform}] 同轮语义去重: 关键词重叠{len(overlap)}/{len(total)},跳过: {_cleaned[:40]!r} ~ {_prev_cleaned[:40]!r}"
|
||||
)
|
||||
_is_same_round_dup = True
|
||||
break
|
||||
if _is_same_round_dup:
|
||||
continue
|
||||
|
||||
|
||||
Reference in New Issue
Block a user