偏旁过滤改md库驱动: bot只查Unicode列码点, md缺失才回退硬编码范围, 维护只改md不改代码

This commit is contained in:
youpaishidifu
2026-08-01 13:22:24 +08:00
parent ba0e6e2450
commit 733f12c45b
2 changed files with 48 additions and 11 deletions
+40 -6
View File
@@ -41,18 +41,52 @@ import difflib
import os
# ⭐ 偏旁部首 Unicode 库:用于 OCR 碎片过滤
# CJK Radicals Supplement: U+2E80-U+2EFF
# CJK Radicals Extended: U+2F00-U+2FDF
# 这些字符在正常中文聊天中不会出现,OCR 拆字碎片才会有
_RADICAL_RANGES = (
# 单一数据源 = unicode_radicals.md(人和机器共用):
# - 人用肉眼看「偏旁」列维护
# - bot 启动时解析「Unicode」列(U+XXXX)构建码点集合,比对只看码点
# 维护方式:编辑 md 添加/删除行即可,无需改代码。
# 仅当 md 缺失/解析失败时才回退到硬编码码位范围(保底,正常不触发)。
_RADICALS_MD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "unicode_radicals.md")
# 保底范围:CJK Radicals Supplement U+2E80-U+2EFF + Extended-A U+2F00-U+2FDF
_RADICAL_RANGES_FALLBACK = (
(0x2E80, 0x2EFF), # CJK Radicals Supplement
(0x2F00, 0x2FDF), # CJK Radicals Extended-A
)
def _load_radical_codepoints() -> frozenset:
"""从 unicode_radicals.md 解析「Unicode」列,返回码点集合。
行格式: | 亻 | U+2E80 | 单人旁 | 件→亻牛 | 件 |
Unicode 列必须是 U+XXXX / U+XXXXX 十六进制;解析失败的行跳过。
"""
cps = set()
try:
with open(_RADICALS_MD, "r", encoding="utf-8") as f:
for line in f:
m = re.search(r"U\+([0-9A-Fa-f]{4,5})\b", line)
if m:
cps.add(int(m.group(1), 16))
except OSError as e:
logger.warning(f"[ocr] 偏旁库 {_RADICALS_MD} 读取失败({e}),回退码位范围")
return frozenset()
if not cps:
logger.warning("[ocr] 偏旁库为空,回退码位范围")
return frozenset()
logger.debug(f"[ocr] 偏旁库加载 {len(cps)} 个码点 <- {_RADICALS_MD}")
return frozenset(cps)
_RADICAL_CODEPOINTS = _load_radical_codepoints()
def _is_radical_char(ch: str) -> bool:
"""检查字符是否为 CJK 偏旁字符(U+2E80-U+2FDF)。"""
"""检查字符是否为 CJK 偏旁字符。
优先查 md 库的 Unicode 码点(维护只改 md);
库未加载成功时回退到硬编码码位范围(保底)。
"""
cp = ord(ch)
return any(lo <= cp <= hi for lo, hi in _RADICAL_RANGES)
if _RADICAL_CODEPOINTS:
return cp in _RADICAL_CODEPOINTS
return any(lo <= cp <= hi for lo, hi in _RADICAL_RANGES_FALLBACK)
import subprocess
import json