修复三个OCR过滤问题：
1. Task #1440: 时间戳过滤器加强到5分钟窗口 - 继承时间戳超过5分钟(300s)的消息直接丢弃 - 修复：12:09的消息能通过13:49的过滤器（差100分钟） 2. Task #1441: bot自身消息查库过滤 - 新增get_recent_bot_replies方法查询数据库 - 检查OCR读到的内容与数据库中bot回复重合率≥90%则丢弃 - 修复：bot回复的消息被OCR读到后直接回复用户的问题 3. Task #1442: 数据库时间戳混乱修复 - 修改record_bot_reply和record_user_message使用本地时间 - SQLite的CURRENT_TIMESTAMP返回UTC时间导致时间戳混乱 - 修复：部分消息created_at在凌晨但实际测试不在凌晨 同时包含Task #1436的链接文本清理修复
This commit is contained in:
youpaishidifu
+17
-6
@@ -73,7 +73,7 @@ class AutoReply:
|
||||
else:
|
||||
sender = None
|
||||
|
||||
return {"platform": platform, "answer": answer, "sender": sender}
|
||||
return {"platform": platform, "answer": answer, "sender": sender, "msg": msg}
|
||||
|
||||
@staticmethod
|
||||
def _has_content_relevance(query: str, doc: dict, score: float = 0.0) -> bool:
|
||||
@@ -88,18 +88,28 @@ class AutoReply:
|
||||
if len(q_tokens) <= 2:
|
||||
return True # 短查询不拦截
|
||||
|
||||
# 1. 检查 heading + title
|
||||
doc_text = doc.get("heading", "") + " " + doc.get("title", "")
|
||||
# 1. 检查 heading + title（排除链接文本）
|
||||
heading = doc.get("heading", "")
|
||||
title = doc.get("title", "")
|
||||
# 清理 heading 中的 Markdown 链接：[text](url) → text
|
||||
_link_pattern = re.compile(r'\[([^\]]*)\]\([^\)]+\)')
|
||||
heading_clean = _link_pattern.sub(r'\1', heading)
|
||||
# 进一步清理：去掉链接文本本身
|
||||
heading_no_link = _link_pattern.sub('', heading)
|
||||
doc_text = heading_clean + " " + title
|
||||
d_tokens = set(t for t in jieba.cut(doc_text) if len(t) >= 2)
|
||||
has_heading_hit = bool(q_tokens & d_tokens)
|
||||
|
||||
# 2. 检查 content/body（TF-IDF匹配的段落正文）
|
||||
# 2. 检查 content/body（排除链接文本）
|
||||
body = doc.get("content", "") or doc.get("body", "")
|
||||
b_tokens = set()
|
||||
has_body_hit = False
|
||||
if body:
|
||||
# 清理 body 中的链接文本，防止 "[sdf1_login登录插件](sdf1_login.md)" 误匹配
|
||||
body_clean = _link_pattern.sub(r'\1', body)
|
||||
body_no_link = _link_pattern.sub('', body)
|
||||
# 截断前500字防超长文档
|
||||
b_tokens = set(t for t in jieba.cut(body[:500]) if len(t) >= 2)
|
||||
b_tokens = set(t for t in jieba.cut(body_no_link[:500]) if len(t) >= 2)
|
||||
has_body_hit = bool(q_tokens & b_tokens)
|
||||
|
||||
# 动作/配置类查询：不能只命中标题，正文必须也含核心词
|
||||
@@ -120,7 +130,8 @@ class AutoReply:
|
||||
# 3. 子串回退：jieba分词可能产生"包是""还是"等碎片token，
|
||||
# 导致token交集为空但文档确实包含查询核心词（如"资源包"）。
|
||||
# 检查查询中任意2~6字子串是否出现在文档文本中。
|
||||
full_text = doc_text + " " + body[:500]
|
||||
# ⭐ 使用清理后的文本（排除链接文本）
|
||||
full_text = doc_text + " " + body_no_link[:500] if body else doc_text
|
||||
stripped_query = re.sub(r'[？?！!。，,\s]', '', query)
|
||||
if len(stripped_query) >= 4:
|
||||
for qlen in range(min(len(stripped_query), 6), 1, -1):
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"pid": 9260, "mode": "GPU", "status": "idle", "last_img_len": 0, "ts": 1786081979.525074, "iso": "2026-08-07T13:52:59.525074", "error": ""}
|
||||
+59
-9
@@ -230,9 +230,14 @@ class MessageHistory:
|
||||
content_hash = hashlib.md5(content.strip().encode('utf-8')).hexdigest()
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
# ⭐ Task #1442: 使用本地时间而不是UTC时间
|
||||
# SQLite的CURRENT_TIMESTAMP返回UTC时间，导致时间戳混乱
|
||||
# 修复：使用Python本地时间
|
||||
from datetime import datetime
|
||||
local_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
conn.execute(
|
||||
"INSERT INTO message_history (platform, sender_type, content, content_hash) VALUES (?, 'bot', ?, ?)",
|
||||
(platform, content.strip(), content_hash)
|
||||
"INSERT INTO message_history (platform, sender_type, content, content_hash, created_at) VALUES (?, 'bot', ?, ?, ?)",
|
||||
(platform, content.strip(), content_hash, local_time)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
@@ -245,9 +250,13 @@ class MessageHistory:
|
||||
content_hash = hashlib.md5(content.strip().encode('utf-8')).hexdigest()
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
# ⭐ Task #1442: 使用本地时间而不是UTC时间
|
||||
# SQLite的CURRENT_TIMESTAMP返回UTC时间，导致时间戳混乱
|
||||
from datetime import datetime
|
||||
local_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
conn.execute(
|
||||
"INSERT INTO message_history (platform, sender_type, content, content_hash) VALUES (?, 'user', ?, ?)",
|
||||
(platform, content.strip(), content_hash)
|
||||
"INSERT INTO message_history (platform, sender_type, content, content_hash, created_at) VALUES (?, 'user', ?, ?, ?)",
|
||||
(platform, content.strip(), content_hash, local_time)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
@@ -278,11 +287,11 @@ class MessageHistory:
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row and row[0]:
|
||||
# 将数据库时间字符串转换为时间戳
|
||||
# ⭐ Task #1442: 正确解析本地时间
|
||||
# 现在created_at存储的是本地时间（如"2026-08-06 14:30:00"）
|
||||
try:
|
||||
from datetime import datetime
|
||||
bot_time_str = row[0]
|
||||
# 假设格式为 "2026-08-05 15:09:53"
|
||||
bot_time = datetime.strptime(bot_time_str, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
# 如果用户消息时间戳晚于最后bot回复时间，视为新消息
|
||||
if user_msg_time > bot_time:
|
||||
@@ -304,7 +313,9 @@ class MessageHistory:
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
# 将SQLite时间戳转换为Unix时间
|
||||
# ⭐ Task #1442: 正确解析本地时间
|
||||
# 现在created_at存储的是本地时间（如"2026-08-06 14:30:00"）
|
||||
# 直接解析即可，无需时区转换
|
||||
from datetime import datetime
|
||||
dt = datetime.strptime(row[0], '%Y-%m-%d %H:%M:%S')
|
||||
return dt.timestamp()
|
||||
@@ -312,6 +323,18 @@ class MessageHistory:
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_recent_bot_replies(self, platform: str, limit: int = 20) -> list:
|
||||
"""获取该平台最近的bot回复内容列表，用于查库比对"""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cursor = conn.execute(
|
||||
"SELECT content FROM message_history WHERE platform=? AND sender_type='bot' ORDER BY id DESC LIMIT ?",
|
||||
(platform, limit)
|
||||
)
|
||||
return [row[0] for row in cursor.fetchall() if row[0]]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class MessageMonitor:
|
||||
"""多线程消息监控器"""
|
||||
@@ -2885,7 +2908,11 @@ class MessageMonitor:
|
||||
buf = ''
|
||||
last_ts = part
|
||||
continue
|
||||
logger.debug(f"[ocr] stage[{i}] 继承时间戳{last_ts}较老但消息无独立时间戳，放行交由下游去重: {buf.strip()[:60]!r}")
|
||||
# ⭐ Task #1440: 继承时间戳超过5分钟(300s)的消息直接丢弃
|
||||
# 修复：12:09的消息能通过13:49的过滤器（差100分钟）
|
||||
# 根因：无独立时间戳的消息被放行交由下游去重，但下游去重不够严格
|
||||
logger.debug(f"[ocr] stage[{i}] 继承时间戳{last_ts}较老(>5min)且消息无独立时间戳，丢弃: {buf.strip()[:60]!r}")
|
||||
buf = ''
|
||||
if self._is_markdown_document(buf.strip()):
|
||||
logger.debug(f"[ocr] stage[{i}] md文档直接跳过: {buf.strip()[:80]!r}")
|
||||
elif self._is_noise_text(buf.strip()):
|
||||
@@ -2926,7 +2953,10 @@ class MessageMonitor:
|
||||
logger.debug(f"[ocr] stage[{i}] 尾部时间戳{last_ts}极老(>2h)且非新鲜内容，视为远古历史跳过尾部: {buf.strip()[:60]!r}")
|
||||
buf = ''
|
||||
else:
|
||||
logger.debug(f"[ocr] stage[{i}] 尾部时间戳{last_ts}较老但消息无独立时间戳，放行交由下游去重: {buf.strip()[:60]!r}")
|
||||
# ⭐ Task #1440: 尾部继承时间戳超过5分钟(300s)的消息直接丢弃
|
||||
# 修复：与中段逻辑一致，超时一律丢弃
|
||||
logger.debug(f"[ocr] stage[{i}] 尾部时间戳{last_ts}较老(>5min)且消息无独立时间戳，丢弃: {buf.strip()[:60]!r}")
|
||||
buf = ''
|
||||
if buf.strip():
|
||||
if self._is_markdown_document(buf.strip()):
|
||||
logger.debug(f"[ocr] stage[{i}] md文档直接跳过(尾部): {buf.strip()[:80]!r}")
|
||||
@@ -3298,6 +3328,26 @@ class MessageMonitor:
|
||||
logger.debug(f"[{platform}] OCR 过滤: 回复内容片段 line={line[:60]!r}")
|
||||
continue
|
||||
|
||||
# ⭐ Task #1441: bot自身消息查库过滤
|
||||
# 检查OCR读到的内容是否与数据库中bot回复的消息重合率≥90%
|
||||
# 如果是，说明这是bot自己回复的消息，不是用户消息，应丢弃
|
||||
if len(line) > 10 and self._msg_history:
|
||||
try:
|
||||
import difflib
|
||||
# 查询数据库中最近的bot回复（限制20条）
|
||||
recent_bot_replies = self._msg_history.get_recent_bot_replies(platform, limit=20)
|
||||
for bot_reply in recent_bot_replies:
|
||||
if len(bot_reply) < 10:
|
||||
continue
|
||||
# 计算重合率
|
||||
sm = difflib.SequenceMatcher(None, line, bot_reply)
|
||||
ratio = sm.ratio()
|
||||
if ratio >= 0.9:
|
||||
logger.debug(f"[{platform}] OCR 过滤: bot自身消息(查库重合率{ratio:.2f}) line={line[:60]!r}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug(f"[{platform}] OCR 查库过滤异常: {e}")
|
||||
|
||||
# === 全面 md 语法兜底：正常用户不写 md，只可能是 bot 回复 ===
|
||||
# ** 加粗标记（用户聊天从不打 **）
|
||||
if '**' in line:
|
||||
|
||||
Reference in New Issue
Block a user