fix: 修复消息入库后被查重跳过的问题
monitor.py: - 修改 _is_similar_to_history 函数:添加 exclude_self 参数 - exclude_self=True 时排除自身刚入库的记录,防止'入库→查重→跳过'死循环 - 修改 _poll 中的调用:使用 exclude_self=True 排除自身 - record_user_message 中的调用保持 exclude_self=False(默认值) 问题分析: 消息在 _process_ocr_output 中被入库(record_user_message), 然后在 _poll 中被查重(_is_similar_to_history), 发现与刚入库的消息相似(100%),所以被跳过。 解决方案: 在 _poll 中使用 exclude_self=True, 排除自身刚入库的记录,防止死循环。
This commit is contained in:
+9
-2
@@ -435,13 +435,14 @@ class MessageHistory:
|
|||||||
cleaned = ''.join(c for c in text if '\u4e00' <= c <= '\u9fff')
|
cleaned = ''.join(c for c in text if '\u4e00' <= c <= '\u9fff')
|
||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
def _is_similar_to_history(self, platform: str, content: str, threshold: float = 0.5) -> bool:
|
def _is_similar_to_history(self, platform: str, content: str, threshold: float = 0.5, exclude_self: bool = False) -> bool:
|
||||||
"""检查消息是否与数据库历史消息相似(清理后计算相似度)。
|
"""检查消息是否与数据库历史消息相似(清理后计算相似度)。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
platform: 平台名称
|
platform: 平台名称
|
||||||
content: 消息内容
|
content: 消息内容
|
||||||
threshold: 相似度阈值,默认0.5(50%)
|
threshold: 相似度阈值,默认0.5(50%)
|
||||||
|
exclude_self: 是否排除自身(content完全匹配时跳过),用于轮询阶段查重
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
True = 重复消息,应跳过;False = 新消息,可继续处理。
|
True = 重复消息,应跳过;False = 新消息,可继续处理。
|
||||||
@@ -471,6 +472,11 @@ class MessageHistory:
|
|||||||
|
|
||||||
for row in rows:
|
for row in rows:
|
||||||
historical_content = row[0]
|
historical_content = row[0]
|
||||||
|
|
||||||
|
# ⭐ exclude_self 模式:content完全相同 → 是自身刚入库的记录 → 跳过比对
|
||||||
|
if exclude_self and historical_content.strip() == content.strip():
|
||||||
|
continue
|
||||||
|
|
||||||
cleaned_historical = self._clean_text_for_dedup(historical_content)
|
cleaned_historical = self._clean_text_for_dedup(historical_content)
|
||||||
if len(cleaned_historical) < 2:
|
if len(cleaned_historical) < 2:
|
||||||
continue
|
continue
|
||||||
@@ -10055,7 +10061,8 @@ class MessageMonitor:
|
|||||||
self._last_new_msg_time[platform] = time.time()
|
self._last_new_msg_time[platform] = time.time()
|
||||||
continue
|
continue
|
||||||
# ⭐ 数据库查重:清理后计算相似度,重复率≥50%就跳过(用户要求)
|
# ⭐ 数据库查重:清理后计算相似度,重复率≥50%就跳过(用户要求)
|
||||||
if self._msg_history and self._msg_history._is_similar_to_history(platform, cleaned_item, threshold=0.5):
|
# ⭐ exclude_self=True:排除自身刚入库的记录,防止"入库→查重→跳过"死循环
|
||||||
|
if self._msg_history and self._msg_history._is_similar_to_history(platform, cleaned_item, threshold=0.5, exclude_self=True):
|
||||||
logger.info(f"[{platform}] 数据库查重: 与历史消息相似,跳过: {cleaned_item[:60]}")
|
logger.info(f"[{platform}] 数据库查重: 与历史消息相似,跳过: {cleaned_item[:60]}")
|
||||||
self._last_new_msg_time[platform] = time.time()
|
self._last_new_msg_time[platform] = time.time()
|
||||||
continue
|
continue
|
||||||
|
|||||||
Reference in New Issue
Block a user