fix: 修复消息入库后被查重跳过的问题
monitor.py: - 修改 _is_similar_to_history 函数:添加 exclude_self 参数 - exclude_self=True 时排除自身刚入库的记录,防止'入库→查重→跳过'死循环 - 修改 _poll 中的调用:使用 exclude_self=True 排除自身 - record_user_message 中的调用保持 exclude_self=False(默认值) 问题分析: 消息在 _process_ocr_output 中被入库(record_user_message), 然后在 _poll 中被查重(_is_similar_to_history), 发现与刚入库的消息相似(100%),所以被跳过。 解决方案: 在 _poll 中使用 exclude_self=True, 排除自身刚入库的记录,防止死循环。
This commit is contained in:
+21
-14
@@ -435,17 +435,18 @@ class MessageHistory:
|
||||
cleaned = ''.join(c for c in text if '\u4e00' <= c <= '\u9fff')
|
||||
return cleaned
|
||||
|
||||
def _is_similar_to_history(self, platform: str, content: str, threshold: float = 0.5) -> bool:
|
||||
def _is_similar_to_history(self, platform: str, content: str, threshold: float = 0.5, exclude_self: bool = False) -> bool:
|
||||
"""检查消息是否与数据库历史消息相似(清理后计算相似度)。
|
||||
|
||||
|
||||
参数:
|
||||
platform: 平台名称
|
||||
content: 消息内容
|
||||
threshold: 相似度阈值,默认0.5(50%)
|
||||
|
||||
exclude_self: 是否排除自身(content完全匹配时跳过),用于轮询阶段查重
|
||||
|
||||
返回:
|
||||
True = 重复消息,应跳过;False = 新消息,可继续处理。
|
||||
|
||||
|
||||
逻辑:
|
||||
1. 清理当前消息:去掉英文字母、数字、标点等,只保留中文字符
|
||||
2. 查询该平台最近10条用户消息
|
||||
@@ -455,11 +456,11 @@ class MessageHistory:
|
||||
"""
|
||||
if not content or not content.strip():
|
||||
return False
|
||||
|
||||
|
||||
cleaned_current = self._clean_text_for_dedup(content)
|
||||
if len(cleaned_current) < 2: # 太短无法比对
|
||||
return False
|
||||
|
||||
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
# 查询该平台最近10条用户消息
|
||||
@@ -468,36 +469,41 @@ class MessageHistory:
|
||||
(platform,)
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
|
||||
for row in rows:
|
||||
historical_content = row[0]
|
||||
|
||||
# ⭐ exclude_self 模式:content完全相同 → 是自身刚入库的记录 → 跳过比对
|
||||
if exclude_self and historical_content.strip() == content.strip():
|
||||
continue
|
||||
|
||||
cleaned_historical = self._clean_text_for_dedup(historical_content)
|
||||
if len(cleaned_historical) < 2:
|
||||
continue
|
||||
|
||||
|
||||
# 计算相似度:字符重叠率
|
||||
# 方法:计算两个字符串的公共字符比例(使用集合交集)
|
||||
set_current = set(cleaned_current)
|
||||
set_historical = set(cleaned_historical)
|
||||
|
||||
|
||||
if not set_current or not set_historical:
|
||||
continue
|
||||
|
||||
|
||||
# 交集大小除以较小集合的大小(Szymkiewicz-Simpson系数)
|
||||
intersection = len(set_current & set_historical)
|
||||
min_size = min(len(set_current), len(set_historical))
|
||||
similarity = intersection / min_size if min_size > 0 else 0
|
||||
|
||||
|
||||
if similarity >= threshold:
|
||||
logger.debug(
|
||||
f"[{platform}] 数据库查重: 相似度{similarity:.0%} >= {threshold:.0%}, "
|
||||
f"跳过重复消息: '{content[:40]!r}' ~ '{historical_content[:40]!r}'"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
return False
|
||||
|
||||
def record_user_message(self, platform: str, content: str):
|
||||
@@ -10055,7 +10061,8 @@ class MessageMonitor:
|
||||
self._last_new_msg_time[platform] = time.time()
|
||||
continue
|
||||
# ⭐ 数据库查重:清理后计算相似度,重复率≥50%就跳过(用户要求)
|
||||
if self._msg_history and self._msg_history._is_similar_to_history(platform, cleaned_item, threshold=0.5):
|
||||
# ⭐ exclude_self=True:排除自身刚入库的记录,防止"入库→查重→跳过"死循环
|
||||
if self._msg_history and self._msg_history._is_similar_to_history(platform, cleaned_item, threshold=0.5, exclude_self=True):
|
||||
logger.info(f"[{platform}] 数据库查重: 与历史消息相似,跳过: {cleaned_item[:60]}")
|
||||
self._last_new_msg_time[platform] = time.time()
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user