fix: 反向包含剥离误伤问句+语义去重阈值过低
问题1: 反向包含剥离把用户问句中的共享词(如'操作')误判为bot回复片段剥掉 - 原逻辑: 逗号分段后,part in r → 剥离 - 根因: '那领地系统有哪两种操作模式啊?'中'操作'匹配旧回复表头'操作方式' - 修复: 检测问号/疑问词时跳过策略2(问句豁免) 问题2: 语义去重阈值0.3太低,轻微OCR变体就触发重复 - 原阈值: 0.3(Jaccard on CJK bigrams) - 实测: '操作模式?' vs '操作模式啊?游戏专' Jaccard=0.47 > 0.3 → 误杀 - 修复: 提高到0.75,仅真正重复查询(OCR变体/加@前缀)才拦截
This commit is contained in:
+29
-22
@@ -4748,26 +4748,33 @@ class MessageMonitor:
|
||||
# 策略2:反向包含剥离——逗号分段后逐段判断是否属于 bot 回复
|
||||
_has_comma = ',' in ts or (',' in ts and re.search(r'[\u4e00-\u9fff],[\u4e00-\u9fff]', ts))
|
||||
if not stripped_bot and _has_comma:
|
||||
parts = re.split(r'[,,]', ts)
|
||||
parts = [p.strip() for p in parts if p.strip()]
|
||||
if len(parts) >= 2:
|
||||
user_parts = []
|
||||
for part in parts:
|
||||
if len(part) < 2:
|
||||
continue
|
||||
is_bot_frag = False
|
||||
npart = self._norm_frag(part)
|
||||
if npart:
|
||||
for r in replied:
|
||||
nr = self._norm_frag(r)
|
||||
if len(nr) >= 2 and npart in nr:
|
||||
is_bot_frag = True
|
||||
break
|
||||
if not is_bot_frag:
|
||||
user_parts.append(part)
|
||||
if len(user_parts) < len(parts):
|
||||
ts = ','.join(user_parts).strip(',').strip()
|
||||
stripped_bot = True
|
||||
# ⭐ 2026-08-17 问句豁免:用户提问即使含与旧回复共享的片段,也不应被剥离
|
||||
# 例:"那领地系统有哪两种操作模式啊?游戏专" — "操作"与旧回复表头匹配
|
||||
# 但这是用户新问句,不是旧回复的回声
|
||||
is_question_mark = bool(re.search(r'[??]', ts))
|
||||
is_question_word = any(qw in ts for qw in ('吗', '呢', '多少', '怎么', '如何', '哪', '什么', '几'))
|
||||
is_question = is_question_mark or is_question_word
|
||||
if not is_question:
|
||||
parts = re.split(r'[,,]', ts)
|
||||
parts = [p.strip() for p in parts if p.strip()]
|
||||
if len(parts) >= 2:
|
||||
user_parts = []
|
||||
for part in parts:
|
||||
if len(part) < 2:
|
||||
continue
|
||||
is_bot_frag = False
|
||||
npart = self._norm_frag(part)
|
||||
if npart:
|
||||
for r in replied:
|
||||
nr = self._norm_frag(r)
|
||||
if len(nr) >= 2 and npart in nr:
|
||||
is_bot_frag = True
|
||||
break
|
||||
if not is_bot_frag:
|
||||
user_parts.append(part)
|
||||
if len(user_parts) < len(parts):
|
||||
ts = ','.join(user_parts).strip(',').strip()
|
||||
stripped_bot = True
|
||||
# 策略3:尾部匹配——OCR把bot回复直接拼在用户消息末尾
|
||||
# 使用纯中文相似度匹配,抵抗OCR噪声和英文/数字干扰
|
||||
if not stripped_bot:
|
||||
@@ -5265,7 +5272,7 @@ class MessageMonitor:
|
||||
self._semantic_fingerprint_cache = kept
|
||||
|
||||
result = []
|
||||
SIM_THRESHOLD = 0.3 # Jaccard相似度阈值:>=0.3视为语义重复
|
||||
SIM_THRESHOLD = 0.75 # Jaccard相似度阈值:>=0.75视为语义重复(提高防误杀)
|
||||
for text in texts:
|
||||
is_dup = False
|
||||
for kept_text in result:
|
||||
@@ -5305,7 +5312,7 @@ class MessageMonitor:
|
||||
]
|
||||
|
||||
# 与最近查询比较
|
||||
SIM_THRESHOLD = 0.3
|
||||
SIM_THRESHOLD = 0.75 # Jaccard相似度阈值:>=0.75视为语义重复(提高防误杀)
|
||||
for recent_text, recent_ts in self._recent_user_queries:
|
||||
sim = self._semantic_similarity(text, recent_text)
|
||||
if sim >= SIM_THRESHOLD:
|
||||
|
||||
Reference in New Issue
Block a user