fix(Task #1558): 重构消息去重:用关键词打分替代OCR时间戳判断
核心变化:
1. 完全移除OCR时间戳丢弃逻辑(_is_timestamp_old不再用于丢弃消息)
2. 改为关键词打分+时间感知去重:
- 得分≤1 → 直接放行(新消息)
- 得分≥1 → 检查最近一次打分时间:
* 最近300秒内有同类记录 → 丢弃(同一用户重复)
* 超过300秒或无记录 → 放行(不同用户的新提问)
3. 修复问题:
- 旧消息被陈旧时间戳继承导致误杀(如13:32:36继承给14:42:57的消息)
- 不同用户相同问题的误过滤(如"php管理员密码"被统一丢弃)
4. OCR时间戳仅作为参考,±10秒容差,超时用本地时间
修改文件:
- _keyword_scorer.py: 新增时间感知去重逻辑
- monitor.py: 移除时间戳丢弃分支
This commit is contained in:
+56
-8
@@ -188,12 +188,19 @@ class KeywordScorer:
|
||||
|
||||
def should_skip_message(self, text: str, platform: str,
|
||||
threshold: int = 2) -> Tuple[bool, int, List[str]]:
|
||||
"""判断是否应该跳过该消息(基于关键词得分)
|
||||
"""判断是否应该跳过该消息(基于关键词得分 + 时间感知)
|
||||
|
||||
⭐ Task #1558: 新逻辑
|
||||
- 得分≤1 → 直接放行(新消息)
|
||||
- 得分≥2 → 检查最近一次打分时间:
|
||||
* 最近打分在5分钟内 → 丢弃(同一用户重复)
|
||||
* 最近打分超过5分钟 → 放行(不同用户的新提问)
|
||||
这样避免一刀切≥2分就丢弃,因为不同用户可能提出相同问题。
|
||||
|
||||
Args:
|
||||
text: 消息文本
|
||||
platform: 平台标识
|
||||
threshold: 得分阈值,≥threshold 则跳过
|
||||
threshold: 得分阈值,≥threshold 则检查时间
|
||||
|
||||
Returns:
|
||||
(是否跳过, 得分, 匹配关键词)
|
||||
@@ -204,13 +211,54 @@ class KeywordScorer:
|
||||
|
||||
score, matched = self.score_against_history(keywords, platform)
|
||||
|
||||
# 记录本次得分
|
||||
is_new = score < threshold
|
||||
self.record_message_score(platform, text, keywords, score, matched, is_new)
|
||||
# 记录本次得分(先记录用于后续检查)
|
||||
self.record_message_score(platform, text, keywords, score, matched, is_new=True)
|
||||
|
||||
# 得分≤1:直接放行(新消息)
|
||||
if score <= 1:
|
||||
return False, score, matched
|
||||
|
||||
# 得分≥2:检查最近一次同类消息的打分时间
|
||||
should_skip = self._should_skip_by_recent_score(platform, score, matched)
|
||||
|
||||
should_skip = score >= threshold
|
||||
if should_skip:
|
||||
logger.info(f"[keyword-scorer] 跳过消息(得分{score}≥{threshold}): {text[:40]}... "
|
||||
f"匹配关键词: {matched}")
|
||||
logger.info(f"[keyword-scorer] 跳过消息(得分{score}≥{threshold},5分钟内): {text[:40]}... "
|
||||
f"匹配关键词: {matched[:5]}")
|
||||
else:
|
||||
logger.info(f"[keyword-scorer] 放行消息(得分{score}≥{threshold},超过5分钟或无记录): "
|
||||
f"{text[:40]}... 匹配关键词: {matched[:5]}")
|
||||
|
||||
return should_skip, score, matched
|
||||
|
||||
def _should_skip_by_recent_score(self, platform: str, score: int, matched: List[str]) -> bool:
|
||||
"""检查最近一次同类消息的打分时间,超过5分钟则放行"""
|
||||
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
try:
|
||||
from datetime import timedelta
|
||||
cutoff_time = (datetime.now() - timedelta(seconds=300)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# 查询最近300秒内得分≥1的消息(不限score阈值,只看时间)
|
||||
cursor = conn.execute("""
|
||||
SELECT local_time, score
|
||||
FROM keyword_scores
|
||||
WHERE platform = ?
|
||||
AND score >= 1
|
||||
AND matched_keywords != ''
|
||||
AND local_time > ?
|
||||
ORDER BY id DESC
|
||||
LIMIT 3
|
||||
""", (platform, cutoff_time))
|
||||
|
||||
recent = cursor.fetchall()
|
||||
if not recent:
|
||||
# 无近期记录 → 放行(可能是新用户的相同提问)
|
||||
logger.debug(f"[keyword-scorer] 无近期记录,放行: score={score}, matched={matched[:3]}")
|
||||
return False
|
||||
|
||||
# 有近期记录 → 丢弃(同一用户重复提问)
|
||||
logger.debug(f"[keyword-scorer] 发现近期记录,跳过: count={len(recent)}, "
|
||||
f"最近时间={recent[0][0]}, score={recent[0][1]}")
|
||||
return True
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
Reference in New Issue
Block a user