修复重复处理:加模糊匹配去重兜底
OCR输出每次微小差异(勖彐碎片/行首数字/尾部截断)导致精确去重永远命中不了。 新增 _fuzzy_duplicate 用 difflib.SequenceMatcher.quick_ratio 做相似度去重(阈值0.72), 长度差距>40%直接跳过(quick_ratio上界<阈值)。精确匹配+模糊匹配双重兜底。
This commit is contained in:
+31
-4
@@ -13,6 +13,7 @@ import win32api
|
||||
import win32process
|
||||
import re
|
||||
import ctypes
|
||||
import difflib
|
||||
from PIL import ImageGrab
|
||||
from winrt.windows.media.ocr import OcrEngine
|
||||
from winrt.windows.globalization import Language
|
||||
@@ -514,6 +515,24 @@ class MessageMonitor:
|
||||
pass
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _fuzzy_duplicate(new_text: str, seen_set: set, threshold: float = 0.72) -> bool:
|
||||
"""检查 new_text 是否与 seen_set 中已有文本相似度超阈值。
|
||||
|
||||
OCR 输出有噪声(每次勖彐碎片、行首数字、尾部截断不同),精确匹配无法去重。
|
||||
用 difflib.SequenceMatcher 的 quick_ratio() 做快速上界剪枝。
|
||||
"""
|
||||
if not new_text or len(new_text) < 4:
|
||||
return False
|
||||
for existing in seen_set:
|
||||
if abs(len(existing) - len(new_text)) > max(len(existing), len(new_text)) * 0.4:
|
||||
# 长度差距超 40%,不可能相似(quick_ratio 上界 = 2*min/max < 阈值)
|
||||
continue
|
||||
ratio = difflib.SequenceMatcher(None, new_text, existing).quick_ratio()
|
||||
if ratio >= threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _ocr_read_qq(self) -> list:
|
||||
"""OCR 截屏 QQ 消息区域 -> Windows OCR 提取文本。
|
||||
包含窗口验证 + 垃圾过滤 + 连续垃圾清缓存。
|
||||
@@ -1353,6 +1372,7 @@ class MessageMonitor:
|
||||
self._empty_reads[platform] = 0
|
||||
|
||||
# 4. 用全局 _seen_messages 去重(跨平台、跨轮次防回环)
|
||||
# 精确匹配 + 模糊匹配(OCR 噪声导致精确匹配失效)
|
||||
seen = self._seen_messages
|
||||
replied = self._replied_texts
|
||||
new_items = []
|
||||
@@ -1363,10 +1383,17 @@ class MessageMonitor:
|
||||
# _replied_texts 优先检查(永不裁切,100% 拦住自己的回复)
|
||||
if ts in replied:
|
||||
continue
|
||||
if (ts not in seen
|
||||
and not self._is_system_message(ts)):
|
||||
seen.add(ts)
|
||||
new_items.append(ts)
|
||||
if ts in seen:
|
||||
continue
|
||||
# 模糊匹配:OCR 输出每次微小差异(勖彐碎片、行首数字、尾部截断)
|
||||
# 精确匹配命中不了 → 用 difflib 相似度去重
|
||||
if self._fuzzy_duplicate(ts, seen, threshold=0.72):
|
||||
logger.debug(f"[{platform}] 模糊重复跳过: {ts[:50]}")
|
||||
continue
|
||||
if self._is_system_message(ts):
|
||||
continue
|
||||
seen.add(ts)
|
||||
new_items.append(ts)
|
||||
# 不截断 _seen_messages — _replied_texts 已拦截回复回环,截断会误放历史消息
|
||||
# 极端情况(>50万行)才手动清,几乎不会出现
|
||||
|
||||
|
||||
Reference in New Issue
Block a user