feat(monitor): 添加偏旁合体还原功能
1. 新增 _load_radical_mapping() 函数 - 解析 unicode_radicals.md 构建偏旁组合→完整汉字映射表 - 例如: '讠吾'→'语', '亻尔'→'你', '彳亍'→'行' 2. 新增 _restore_radical_text() 函数 - 按长度降序匹配,避免短模式误匹配 - 将 OCR 拆分的偏旁组合还原为完整汉字 3. 改造偏旁过滤逻辑(OCR阶段 + Poll阶段) - 先尝试合体还原,再统计偏旁占比 - 还原后偏旁占比>10%才丢弃 - 修复 OCR 误识别(默讠/亻尔)导致有效消息被丢弃的问题 示例: 输入: '默讠人登录密码' 还原: '默认人登录密码' 结果: 放行
This commit is contained in:
+55
-10
@@ -96,6 +96,49 @@ def _is_radical_char(ch: str) -> bool:
|
||||
return cp in _RADICAL_CODEPOINTS
|
||||
return any(lo <= cp <= hi for lo, hi in _RADICAL_RANGES_FALLBACK)
|
||||
|
||||
|
||||
def _load_radical_mapping() -> dict:
|
||||
"""从 unicode_radicals.md 加载偏旁合体映射。
|
||||
|
||||
返回: {OCR识别的偏旁组合: 正确汉字}
|
||||
例如: {'讠吾': '语', '亻牛': '件', '彳亍': '行'}
|
||||
"""
|
||||
radical_map = {}
|
||||
try:
|
||||
with open(_RADICALS_MD, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if line.startswith("|") and "→" in line:
|
||||
parts = line.split("|")
|
||||
if len(parts) >= 5:
|
||||
example = parts[3].strip() # 示例列: 件→亻牛
|
||||
corrected = parts[4].strip() # 补充字符列: 件
|
||||
if "→" in example and corrected:
|
||||
ocr_form = example.split("→")[1].strip()
|
||||
radical_map[ocr_form] = corrected
|
||||
except Exception as e:
|
||||
logger.warning(f"[ocr] 加载偏旁合体映射失败({e})")
|
||||
logger.debug(f"[ocr] 偏旁合体映射: {len(radical_map)} 条")
|
||||
return radical_map
|
||||
|
||||
|
||||
_RADICAL_RESTORE_MAP = _load_radical_mapping()
|
||||
|
||||
|
||||
def _restore_radical_text(text: str) -> str:
|
||||
"""尝试合体还原偏旁碎片。
|
||||
|
||||
例如:
|
||||
- '讠吾' → '语'
|
||||
- '亻尔' → '你'
|
||||
- '默讠人' → '默认人' (部分还原)
|
||||
|
||||
策略:按长度降序匹配,避免短模式误匹配。
|
||||
"""
|
||||
restored = text
|
||||
for ocr_form, correct in sorted(_RADICAL_RESTORE_MAP.items(), key=lambda x: -len(x[0])):
|
||||
restored = restored.replace(ocr_form, correct)
|
||||
return restored
|
||||
|
||||
import subprocess
|
||||
import json
|
||||
import psutil
|
||||
@@ -3878,17 +3921,19 @@ class MessageMonitor:
|
||||
logger.debug(f"[{platform}] OCR 过滤: 结构化面板 line={line!r}")
|
||||
continue
|
||||
|
||||
# —— OCR 偏旁碎片过滤:包含偏旁字符就丢弃 ——
|
||||
# 偏旁字符(U+2E80-U+2FDF)在正常中文聊天中不会出现
|
||||
# OCR 拆字碎片如"亻牛""彡"只要出现就是碎片
|
||||
# ⭐ 2026-08-14 修复:放宽过滤,仅当偏旁字符占比过高时才丢弃
|
||||
# 原因:OCR 误识别会把"默认"识别成"默讠"、把"个"识别成"亻尔"
|
||||
# 这些是单个偏旁的 OCR 噪声,不应因此丢弃整行
|
||||
# 只有当偏旁字符占比 > 10% 时(真正乱码/碎片)才丢弃
|
||||
radical_count = sum(1 for ch in line if _is_radical_char(ch))
|
||||
if radical_count > 0 and radical_count / max(len(line), 1) > 0.1:
|
||||
logger.debug(f"[{platform}] OCR 过滤: 偏旁字符过多(占比{radical_count/max(len(line),1)*100:.0f}%) line={line!r}")
|
||||
# —— OCR 偏旁碎片过滤:尝试合体还原 + 阈值丢弃 ——
|
||||
# ⭐ 2026-08-14 修复:不再一有偏旁就丢弃,先尝试合体还原
|
||||
# 1. 将 OCR 拆分的偏旁组合还原为完整汉字(如 '讠吾'→'语')
|
||||
# 2. 还原后再统计偏旁占比,>10% 才丢弃
|
||||
# 原因:OCR 常把"默认"识别成"默讠人"、把"个"识别成"亻尔"
|
||||
# 这些是单个偏旁的 OCR 噪声,还原后是有效文本,不应丢弃
|
||||
restored_line = _restore_radical_text(line)
|
||||
radical_count = sum(1 for ch in restored_line if _is_radical_char(ch))
|
||||
if radical_count > 0 and radical_count / max(len(restored_line), 1) > 0.1:
|
||||
logger.debug(f"[{platform}] OCR 过滤: 偏旁字符过多(占比{radical_count/max(len(restored_line),1)*100:.0f}%) line={line!r} -> {restored_line!r}")
|
||||
continue
|
||||
# 使用还原后的文本继续后续处理
|
||||
line = restored_line
|
||||
# —— 系统配置内容过滤:SMTP/邮件地址/代码片段不是用户消息 ——
|
||||
if re.search(r'@example|smtp|\.py\b|\.bat\b|\.sh\b|//.*\.\w{2,4}\b', line, re.IGNORECASE):
|
||||
logger.debug(f"[{platform}] OCR 过滤: 系统配置内容 line={line!r}")
|
||||
|
||||
Reference in New Issue
Block a user