youpaishidifu
256935d70c
问题：查询'那录插件的pHp后端那管理员默认密码是多少?'（OCR将'登'识别为'录'）， beibao.md的'使用方法-管理员'（score=79）压过phpadmin.md的'php管理员后端'（score=64）。 修复：增加语义意图验证规则： 1. 查询含'php'+'后端/管理'时，heading必须含'php'或'管理' 2. 查询含'管理员密码'时，heading必须含'密码/管理/php' 效果：php管理密码查询现在正确返回phpadmin.md的'## php管理员后端'
387 lines
19 KiB
Python
387 lines
19 KiB
Python
"""自动回复 — 消息匹配知识库后准备答案，由主线程发送"""
|
||
|
||
import logging
|
||
import re
|
||
import threading
|
||
import jieba
|
||
|
||
logger = logging.getLogger("msgbot.autoreply")
|
||
|
||
|
||
class AutoReply:
|
||
"""自动回复控制器"""
|
||
|
||
def __init__(self, monitor):
|
||
self.monitor = monitor
|
||
self._enabled = False
|
||
self._min_score = 0.30
|
||
|
||
# 已回复的消息指纹（防重复）
|
||
self._replied: dict = {}
|
||
self._lock = threading.Lock()
|
||
|
||
def set_enabled(self, enabled: bool):
|
||
self._enabled = enabled
|
||
logger.info(f"自动回复 {'已开启' if enabled else '已关闭'}")
|
||
|
||
def set_min_score(self, score: float):
|
||
self._min_score = max(0.0, min(10.0, score))
|
||
|
||
@property
|
||
def enabled(self) -> bool:
|
||
return self._enabled
|
||
|
||
@property
|
||
def min_score(self) -> float:
|
||
return self._min_score
|
||
|
||
def on_kb_match(self, platform: str, msg: str, top_doc: dict, score: float) -> str or None:
|
||
"""知识库匹配命中时调用。线程安全。
|
||
返回要发送的回答文本，返回 None 表示不发送。
|
||
"""
|
||
if not self._enabled:
|
||
return None
|
||
if score < self._min_score:
|
||
logger.debug(
|
||
f"[{platform}] 匹配度 {score:.2f} < 阈值 {self._min_score}，跳过回复"
|
||
)
|
||
return None
|
||
# 内容相关性检查：查询词与匹配文档至少有一个共同2字+词，防偶然匹配
|
||
if not self._has_content_relevance(msg, top_doc, score):
|
||
heading = top_doc.get("heading", "")[:40]
|
||
body_preview = (top_doc.get("content", "") or top_doc.get("body", ""))[:60]
|
||
logger.debug(
|
||
f"[{platform}] 内容相关性检查失败，跳过: "
|
||
f"msg={msg[:40]!r} score={score:.2f} doc_heading={heading!r} "
|
||
f"doc_body={body_preview!r}"
|
||
)
|
||
return None
|
||
with self._lock:
|
||
return self._get_answer(platform, msg, top_doc, score)
|
||
|
||
def get_reply_data(self, platform: str, msg: str, top_doc: dict, score: float, cleaned_msg: str = None) -> dict or None:
|
||
"""返回 {platform, answer, sender, msg} 供主线程发送。
|
||
cleaned_msg 为去噪后的查询（用于相关性校验与去重指纹），
|
||
msg 保留原始文本（含时间戳）供主线程做消息新鲜度检查。
|
||
"""
|
||
query = cleaned_msg if cleaned_msg else msg
|
||
answer = self.on_kb_match(platform, query, top_doc, score)
|
||
if not answer:
|
||
return None
|
||
|
||
# 获取发送者（在同一线程提取，不涉及窗口操作）
|
||
raw = self.monitor.get_raw_clipboard(platform)
|
||
sender = self.monitor._parse_last_sender(raw)
|
||
# 只有发送者是有效姓名且不在消息内容中时才切会话
|
||
if sender and sender not in msg and score >= self._min_score + 0.05:
|
||
pass
|
||
else:
|
||
sender = None
|
||
|
||
return {"platform": platform, "answer": answer, "sender": sender, "msg": msg}
|
||
|
||
# 相关性检查用的停用词（通用/无区分度的词），这些词命中不算"有内容相关性"
|
||
_STOPWORDS = {
|
||
"插件","使用","怎么","如何","吗","呢","吧","是","的","了","有","哪",
|
||
"两","种","这","那","和","与","或","在","我","你","他","它","我们","你们",
|
||
"一个","可以","需要","什么","为什么","是否","还是","以及","进行","通过",
|
||
"对于","关于","一些","这样","那样","这个","那个","请问","用","到","对",
|
||
"给","把","被","让","请","该","此","为","等","其","将","时","后","前",
|
||
"会","能","要","想","没","不","别","快","如","若","知","都","也","就",
|
||
"已","再","更","最","很","太","较","但","却","因","故","则","且",
|
||
# 通用名词，出现在太多文档中导致误匹配
|
||
"服务器","玩家","游戏","世界","地图","技能","装备","物品","命令","指令",
|
||
}
|
||
|
||
@staticmethod
|
||
def _meaningful_tokens(text: str) -> set:
|
||
"""提取有区分度的关键词（长度≥2 且非停用词），全部小写便于匹配。"""
|
||
toks = set(t.lower() for t in jieba.cut(text) if len(t) >= 2)
|
||
return toks - AutoReply._STOPWORDS
|
||
|
||
@staticmethod
|
||
def _has_content_relevance(query: str, doc: dict, score: float = 0.0) -> bool:
|
||
"""查询词与匹配文档必须共享至少一个有区分度的核心词，防偶然/牛头不对马嘴匹配。
|
||
|
||
判定顺序：
|
||
1. 提取 query 的有区分度关键词（_meaningful_tokens，去停用词、小写）。
|
||
2. 检查这些词是否出现在 doc 的 heading+title 或 body（前 800 字）中。
|
||
3. 动作/配置类查询（怎么/如何/使用/配置…）额外要求正文也含核心词，
|
||
不能只命中标题（否则 "# Sdf1系列登录插件使用说明" 会因标题含词被误放）。
|
||
4. 子串回退：连续 4~6 字子串命中文档（应对 jieba 切碎的短语，如"资源包"）。
|
||
5. 高匹配度(≥10)但无任何共同词：TF-IDF 误匹配，拒绝。
|
||
"""
|
||
import re
|
||
q_mean = AutoReply._meaningful_tokens(query)
|
||
if not q_mean:
|
||
# 极短/纯停用词查询：要求整个 query 子串命中文档
|
||
stripped = re.sub(r'[？?！!。，,\s]', '', query).lower()
|
||
if len(stripped) >= 2:
|
||
full = (doc.get("heading", "") + " " + doc.get("title", "")
|
||
+ " " + (doc.get("content", "") or doc.get("body", ""))).lower()
|
||
return stripped in full
|
||
return False
|
||
|
||
_link = re.compile(r'\[([^\]]*)\]\([^\)]+\)')
|
||
heading = doc.get("heading", "")
|
||
title = doc.get("title", "")
|
||
# 清理 Markdown 链接文本，避免 "[sdf1_login登录插件](sdf1_login.md)" 误匹配
|
||
heading_clean = _link.sub(r'\1', heading).lower()
|
||
body = doc.get("content", "") or doc.get("body", "")
|
||
body_clean = _link.sub(r'\1', body).lower()
|
||
|
||
ht_tokens = AutoReply._meaningful_tokens(heading_clean + " " + title)
|
||
body_tokens = AutoReply._meaningful_tokens(body_clean[:800])
|
||
has_ht = bool(q_mean & ht_tokens)
|
||
has_body = bool(q_mean & body_tokens)
|
||
|
||
# 动作/配置类查询：标题和正文都必须含核心词，防止"牛头不对马嘴"误匹配
|
||
# ⭐ 扩展：has_body=False（正文完全无共同词）时直接拒绝，无论 heading 是否命中
|
||
# 例："领地系统有哪些操作模式?" vs sdf1_login CLI段落 → 拒绝
|
||
# "领地系统有哪些操作模式?" vs protect 权限段落 → 正常（有body命中）
|
||
action_kw = {"怎么", "如何", "使用", "配置", "修改", "设置", "安装", "用法", "操作"}
|
||
if q_mean & action_kw:
|
||
# ⭐ 排除 action_kw 词参与 body 覆盖统计，避免"使用"(verb)误命中
|
||
non_action_q = q_mean - action_kw
|
||
non_action_body = body_tokens - action_kw
|
||
has_body_nonaction = bool(non_action_q & non_action_body)
|
||
if not has_body_nonaction:
|
||
# 正文（不含动作词）完全没有共同词 → 直接拒绝，无论 heading 是否命中
|
||
logger.debug(
|
||
f"动作查询正文无共同词拒绝: query={query[:40]!r} "
|
||
f"doc_heading={heading[:30]!r} body_hits=0 ht_hit={has_ht}"
|
||
)
|
||
return False
|
||
if not has_ht:
|
||
# 正文有非动作词命中但 heading 无命中 → 检查 body 覆盖量（见下方正文覆盖不足防御）
|
||
pass
|
||
# has_ht and has_body_nonaction → 通过
|
||
# 注：原 has_body 已包含 action_kw 词，此处用 has_body_nonaction 更严格
|
||
has_body = has_body_nonaction
|
||
|
||
# ⭐ 后台/前台关键词区分：查询含"后台"但文档不含"后台"→ 不相关
|
||
full = heading_clean + " " + title.lower() + " " + body_clean[:800]
|
||
if "后台" in query and "后台" not in full:
|
||
logger.debug(f"后台关键词缺失: query={query[:40]!r} doc={doc.get('heading', '')[:30]!r}")
|
||
return False
|
||
if "前台" in query and "前台" not in full:
|
||
logger.debug(f"前台关键词缺失: query={query[:40]!r} doc={doc.get('heading', '')[:30]!r}")
|
||
return False
|
||
|
||
# ⭐ 语义意图验证：OCR查询可能含错字（如"录"→"登"、"php"→"pHp"），
|
||
# 需要结合heading确认查询意图，防止"云背包管理员密码"压过"php管理员后端"
|
||
# 查询含"php"+"后端"时，heading必须含"php"或"管理"
|
||
if 'php' in query.lower() and ('后端' in query or '管理' in query):
|
||
if 'php' not in heading_clean and '管理' not in heading_clean:
|
||
logger.debug(f"php查询意图不匹配: query={query[:40]!r} heading={heading[:30]!r}")
|
||
return False
|
||
# 查询含"管理员密码"时，heading必须含"密码"或"管理"或"php"
|
||
if '管理员' in query and '密码' in query:
|
||
if '密码' not in heading_clean and '管理' not in heading_clean and 'php' not in heading_clean:
|
||
logger.debug(f"管理员密码查询意图不匹配: query={query[:40]!r} heading={heading[:30]!r}")
|
||
return False
|
||
|
||
if has_ht or has_body:
|
||
# ⭐ 正文覆盖不足防御（牛头不对马嘴修复）：
|
||
# 当 heading 无命中但有 body 命中时，检查 body 覆盖了多少查询词。
|
||
# 例："领地系统有哪些权限级别?" vs sdf1_login 的 feature list
|
||
# | 领地系统 | 保护自己领地不受损失 | — body 只命中2/5个词(领地,系统)→ 误选
|
||
# 策略：
|
||
# - has_ht=True 时 heading 已提供语义锚定，只需任意 body_hit 即可（正确结果往往 score 低但方向对）
|
||
# - has_ht=False 且 has_body=True 时，要求 body 覆盖足够比例，防止单点散命中通过
|
||
# min_body_hits = min(ceil(q_count*40%), 4) 且至少为 2
|
||
# 5词查询需≥2，6词需≥3，7词需≥3，8词需≥4
|
||
if not has_ht and has_body:
|
||
q_count = len(q_mean)
|
||
body_hits = len(q_mean & body_tokens)
|
||
# 无 heading 命中的查询需更多 body 覆盖，防散点命中漏过：
|
||
# min_body_hits = min(q_count, ceil(q_count*60%))，最低 2
|
||
# 例：2token→ceil(1.2)=2, min(2,2)=2(全中)；3token→ceil(1.8)=2,min(3,2)=2(缺1词过)
|
||
# 4token→ceil(2.4)=3,min(4,3)=3(缺1词不过)；5token→ceil(3)=3
|
||
min_body_hits = min(q_count, (q_count * 3 + 4) // 5) # ceil(q_count*0.6)
|
||
min_body_hits = max(2, min_body_hits)
|
||
if body_hits < min_body_hits:
|
||
logger.debug(
|
||
f"正文覆盖不足跳过(防牛头不对马嘴): query={query[:40]!r} "
|
||
f"doc_heading={heading[:30]!r} body_hits={body_hits}/{q_count} "
|
||
f"(阈值≥{min_body_hits}), ht_hit={has_ht}"
|
||
)
|
||
return False
|
||
return True
|
||
|
||
# ⭐ 下载意图放行：查询含"下载/下到/哪下"等口语，且section heading含"下载"
|
||
# 修复"在哪儿能下到那登录插件啊?"——原文无"下载"二字（口语"下到"），
|
||
# q_mean 不含"下载"，"## 4. 下载地址"heading 虽有"下载"仍被相关性拒绝，
|
||
# 导致误回"插件设置"（body 恰含"登录超时秒数"）
|
||
if any(w in query for w in ('下载', '下到', '哪下', '在哪下', '怎么下')) and '下载' in heading_clean:
|
||
logger.debug(f"下载意图放行: query={query[:30]!r} heading={heading[:30]!r}")
|
||
return True
|
||
|
||
# 子串回退：jieba 可能产生碎片 token，导致交集为空但文档确实含查询短语
|
||
full = heading_clean + " " + title.lower() + " " + body_clean[:800]
|
||
stripped = re.sub(r'[？?！!。，,\s]', '', query).lower()
|
||
if len(stripped) >= 4:
|
||
# ⭐ 修复：子串回退必须同时满足两个条件：
|
||
# 1. 有有意义的子串（≥4字）命中文档
|
||
# 2. 至少有一个 meaningful_token 命中文档（防止"服务器"等通用词泛匹配）
|
||
has_meaningful_hit = bool(q_mean & (set(heading_clean) | set(title.lower()) | set(body_clean[:800])))
|
||
if not has_meaningful_hit:
|
||
# 没有 meaningful_token 命中，即使有子串匹配也拒绝
|
||
logger.debug(f"无有意义token命中拒绝: query={query[:40]!r} doc_heading={heading[:30]!r}")
|
||
return False
|
||
for qlen in range(min(len(stripped), 6), 3, -1):
|
||
for start in range(len(stripped) - qlen + 1):
|
||
phrase = stripped[start:start + qlen]
|
||
if phrase in full:
|
||
logger.debug(
|
||
f"子串回退命中: phrase={phrase!r} in doc={doc.get('heading', '')[:30]!r}"
|
||
)
|
||
return True
|
||
|
||
# 高匹配度但无共同词：TF-IDF 误匹配，拒绝
|
||
if score >= 10.0 and not (has_ht or has_body):
|
||
return False
|
||
|
||
return False
|
||
|
||
def _get_answer(self, platform: str, msg: str, doc: dict, score: float) -> str or None:
|
||
"""提取答案 + 防重复检查。
|
||
优先返回 doc['answer']（Q&A 段），兜底使用 doc['body']（仅正文，不含 heading）。
|
||
KB 搜索已按 section 切分，doc['body'] 即最相关的片段正文。
|
||
只需截断到 300 字符避免消息过长。
|
||
"""
|
||
answer = doc.get("answer")
|
||
if answer:
|
||
answer = answer.strip()
|
||
else:
|
||
# 优先用 body（纯正文），兜底用 content（heading+正文）
|
||
body = doc.get("body", "").strip()
|
||
if body and len(body) > 20:
|
||
answer = self._extract_table_answer(body)
|
||
else:
|
||
content = doc.get("content", "").strip()
|
||
answer = self._extract_table_answer(content) if content else ""
|
||
|
||
if not answer:
|
||
return None
|
||
|
||
# 清理 HTML 换行标签（纯文本消息里 <br> 会原形显示）
|
||
import re
|
||
answer = re.sub(r'</?br\s*/?>', '\n', answer, flags=re.IGNORECASE)
|
||
answer = answer.strip()
|
||
|
||
# 清理 Markdown 格式，返回纯文本
|
||
answer = self._clean_markdown(answer)
|
||
if not answer:
|
||
return None
|
||
|
||
# 拒绝把纯标题/免责声明当作答案发送
|
||
if answer.startswith("# ") and len(answer) < 60:
|
||
logger.debug(f"[{platform}] 答案为纯标题，跳过: {answer[:60]!r}")
|
||
return None
|
||
if re.search(r'特别说明|免责声明|本插件.*自行修改|仅供参考|版权所有', answer):
|
||
logger.debug(f"[{platform}] 答案为免责声明/通用说明，跳过: {answer[:60]!r}")
|
||
return None
|
||
|
||
# 防重复指纹
|
||
fp = hash(f"{platform}:{msg}:{answer[:80]}")
|
||
replied_set = self._replied.setdefault(platform, set())
|
||
if fp in replied_set:
|
||
logger.debug(f"[{platform}] 重复命中，跳过: {msg[:40]}")
|
||
return None
|
||
replied_set.add(fp)
|
||
# 控制缓存上限
|
||
if len(replied_set) > 200:
|
||
self._replied[platform] = set(list(replied_set)[-100:])
|
||
|
||
logger.info(
|
||
f"[{platform}] 🎯 自动回复触发 | 匹配度 {score:.2f} | "
|
||
f"消息: {msg[:40]}"
|
||
)
|
||
return answer
|
||
|
||
@staticmethod
|
||
def _clean_markdown(text: str) -> str:
|
||
"""清理文本中的 Markdown 格式，返回纯文本。
|
||
|
||
处理：
|
||
- 标题语法：## 标题 → 标题
|
||
- 链接语法：[text](url) → text
|
||
- 列表语法：- item → item
|
||
- 加粗语法：**text** → text
|
||
- 引用语法：> text → text
|
||
- 编号列表：1. item → item
|
||
"""
|
||
import re
|
||
if not text:
|
||
return text
|
||
|
||
# 移除标题语法：## 标题 → 标题
|
||
text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
|
||
|
||
# 移除链接语法：[text](url) → text
|
||
text = re.sub(r'\[([^\]]*)\]\([^\)]+\)', r'\1', text)
|
||
|
||
# 移除列表语法：- item → item
|
||
text = re.sub(r'^[\-\*]\s+', '', text, flags=re.MULTILINE)
|
||
|
||
# 移除加粗语法：**text** → text
|
||
text = re.sub(r'\*\*(.+?)\*\*', r'\1', text)
|
||
|
||
# 移除引用语法：> text → text
|
||
text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE)
|
||
|
||
# 移除编号列表：1. item → item
|
||
text = re.sub(r'^\d{1,3}[\.\)、]\s+', '', text, flags=re.MULTILINE)
|
||
|
||
# 移除多余空行
|
||
text = re.sub(r'\n\s*\n', '\n', text)
|
||
|
||
return text.strip()
|
||
|
||
@staticmethod
|
||
def _extract_table_answer(text: str, max_chars: int = 800) -> str:
|
||
"""从含表格的文本中提取表格 + 紧邻上下文，避免返回整段 section。
|
||
|
||
问题背景：知识库含表格的文档（如 php.md "## php后台玩家端"）被匹配后，
|
||
原逻辑把整个 section（表格 + 其后大量无关图文）全部塞入回复。
|
||
实际答案只是表格附近的几句话。
|
||
|
||
提取规则：
|
||
- 表格块：连续以 `|` 开头且含 ≥2 个 `|` 的行（含表头/分隔/数据行）
|
||
- 上文：表格前最近 2 个非空行（通常是说明句），给答案提供语境
|
||
- 下文：不保留。表格之后的内容往往是下一个话题的开头（如"仪表盘：…"
|
||
"商城：…"），保留会重新引入整段无关图文，违背"只返回表格附近的几句话"
|
||
- 截断到 max_chars 字符
|
||
若文本中无完整表格，则退回前 max_chars 字符。
|
||
"""
|
||
if not text:
|
||
return ""
|
||
lines = text.split('\n')
|
||
table_start = None
|
||
table_end = None
|
||
for i, ln in enumerate(lines):
|
||
s = ln.strip()
|
||
if s.startswith('|') and s.endswith('|') and s.count('|') >= 2:
|
||
if table_start is None:
|
||
table_start = i
|
||
table_end = i
|
||
|
||
# 无表格或表格过短：退回前 max_chars 字符
|
||
if table_start is None:
|
||
return text[:max_chars]
|
||
|
||
# 上文：向前找最多 2 个非空行（说明句）
|
||
pre: list = []
|
||
j = table_start - 1
|
||
while j >= 0 and len(pre) < 2:
|
||
if lines[j].strip():
|
||
pre.insert(0, lines[j].strip())
|
||
j -= 1
|
||
|
||
block = pre + lines[table_start:table_end + 1]
|
||
out = '\n'.join(block)
|
||
if len(out) > max_chars:
|
||
out = out[:max_chars] + "..."
|
||
return out
|