fix: 修复消息入库后被查重跳过的问题

monitor.py:
- 修改 _is_similar_to_history 函数:添加 exclude_self 参数
- exclude_self=True 时排除自身刚入库的记录,防止'入库→查重→跳过'死循环
- 修改 _poll 中的调用:使用 exclude_self=True 排除自身
- record_user_message 中的调用保持 exclude_self=False(默认值)

问题分析:
消息在 _process_ocr_output 中被入库(record_user_message),
然后在 _poll 中被查重(_is_similar_to_history),
发现与刚入库的消息相似(100%),所以被跳过。

解决方案:
在 _poll 中使用 exclude_self=True,
排除自身刚入库的记录,防止死循环。
This commit is contained in:
youpaishidifu
2026-08-30 16:32:35 +08:00
parent 778d269e50
commit d52c4ef2f5
+21 -14
View File
@@ -435,17 +435,18 @@ class MessageHistory:
cleaned = ''.join(c for c in text if '\u4e00' <= c <= '\u9fff') cleaned = ''.join(c for c in text if '\u4e00' <= c <= '\u9fff')
return cleaned return cleaned
def _is_similar_to_history(self, platform: str, content: str, threshold: float = 0.5) -> bool: def _is_similar_to_history(self, platform: str, content: str, threshold: float = 0.5, exclude_self: bool = False) -> bool:
"""检查消息是否与数据库历史消息相似(清理后计算相似度)。 """检查消息是否与数据库历史消息相似(清理后计算相似度)。
参数: 参数:
platform: 平台名称 platform: 平台名称
content: 消息内容 content: 消息内容
threshold: 相似度阈值,默认0.5(50%) threshold: 相似度阈值,默认0.5(50%)
exclude_self: 是否排除自身(content完全匹配时跳过),用于轮询阶段查重
返回: 返回:
True = 重复消息,应跳过;False = 新消息,可继续处理。 True = 重复消息,应跳过;False = 新消息,可继续处理。
逻辑: 逻辑:
1. 清理当前消息:去掉英文字母、数字、标点等,只保留中文字符 1. 清理当前消息:去掉英文字母、数字、标点等,只保留中文字符
2. 查询该平台最近10条用户消息 2. 查询该平台最近10条用户消息
@@ -455,11 +456,11 @@ class MessageHistory:
""" """
if not content or not content.strip(): if not content or not content.strip():
return False return False
cleaned_current = self._clean_text_for_dedup(content) cleaned_current = self._clean_text_for_dedup(content)
if len(cleaned_current) < 2: # 太短无法比对 if len(cleaned_current) < 2: # 太短无法比对
return False return False
conn = self._get_conn() conn = self._get_conn()
try: try:
# 查询该平台最近10条用户消息 # 查询该平台最近10条用户消息
@@ -468,36 +469,41 @@ class MessageHistory:
(platform,) (platform,)
) )
rows = cursor.fetchall() rows = cursor.fetchall()
for row in rows: for row in rows:
historical_content = row[0] historical_content = row[0]
# ⭐ exclude_self 模式:content完全相同 → 是自身刚入库的记录 → 跳过比对
if exclude_self and historical_content.strip() == content.strip():
continue
cleaned_historical = self._clean_text_for_dedup(historical_content) cleaned_historical = self._clean_text_for_dedup(historical_content)
if len(cleaned_historical) < 2: if len(cleaned_historical) < 2:
continue continue
# 计算相似度:字符重叠率 # 计算相似度:字符重叠率
# 方法:计算两个字符串的公共字符比例(使用集合交集) # 方法:计算两个字符串的公共字符比例(使用集合交集)
set_current = set(cleaned_current) set_current = set(cleaned_current)
set_historical = set(cleaned_historical) set_historical = set(cleaned_historical)
if not set_current or not set_historical: if not set_current or not set_historical:
continue continue
# 交集大小除以较小集合的大小(Szymkiewicz-Simpson系数) # 交集大小除以较小集合的大小(Szymkiewicz-Simpson系数)
intersection = len(set_current & set_historical) intersection = len(set_current & set_historical)
min_size = min(len(set_current), len(set_historical)) min_size = min(len(set_current), len(set_historical))
similarity = intersection / min_size if min_size > 0 else 0 similarity = intersection / min_size if min_size > 0 else 0
if similarity >= threshold: if similarity >= threshold:
logger.debug( logger.debug(
f"[{platform}] 数据库查重: 相似度{similarity:.0%} >= {threshold:.0%}, " f"[{platform}] 数据库查重: 相似度{similarity:.0%} >= {threshold:.0%}, "
f"跳过重复消息: '{content[:40]!r}' ~ '{historical_content[:40]!r}'" f"跳过重复消息: '{content[:40]!r}' ~ '{historical_content[:40]!r}'"
) )
return True return True
finally: finally:
conn.close() conn.close()
return False return False
def record_user_message(self, platform: str, content: str): def record_user_message(self, platform: str, content: str):
@@ -10055,7 +10061,8 @@ class MessageMonitor:
self._last_new_msg_time[platform] = time.time() self._last_new_msg_time[platform] = time.time()
continue continue
# ⭐ 数据库查重:清理后计算相似度,重复率≥50%就跳过(用户要求) # ⭐ 数据库查重:清理后计算相似度,重复率≥50%就跳过(用户要求)
if self._msg_history and self._msg_history._is_similar_to_history(platform, cleaned_item, threshold=0.5): # ⭐ exclude_self=True:排除自身刚入库的记录,防止"入库→查重→跳过"死循环
if self._msg_history and self._msg_history._is_similar_to_history(platform, cleaned_item, threshold=0.5, exclude_self=True):
logger.info(f"[{platform}] 数据库查重: 与历史消息相似,跳过: {cleaned_item[:60]}") logger.info(f"[{platform}] 数据库查重: 与历史消息相似,跳过: {cleaned_item[:60]}")
self._last_new_msg_time[platform] = time.time() self._last_new_msg_time[platform] = time.time()
continue continue