偏旁过滤改md库驱动: bot只查Unicode列码点, md缺失才回退硬编码范围, 维护只改md不改代码
This commit is contained in:
+40
-6
@@ -41,18 +41,52 @@ import difflib
|
||||
import os
|
||||
|
||||
# ⭐ 偏旁部首 Unicode 库:用于 OCR 碎片过滤
|
||||
# CJK Radicals Supplement: U+2E80-U+2EFF
|
||||
# CJK Radicals Extended: U+2F00-U+2FDF
|
||||
# 这些字符在正常中文聊天中不会出现,OCR 拆字碎片才会有
|
||||
_RADICAL_RANGES = (
|
||||
# 单一数据源 = unicode_radicals.md(人和机器共用):
|
||||
# - 人用肉眼看「偏旁」列维护
|
||||
# - bot 启动时解析「Unicode」列(U+XXXX)构建码点集合,比对只看码点
|
||||
# 维护方式:编辑 md 添加/删除行即可,无需改代码。
|
||||
# 仅当 md 缺失/解析失败时才回退到硬编码码位范围(保底,正常不触发)。
|
||||
_RADICALS_MD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "unicode_radicals.md")
|
||||
# 保底范围:CJK Radicals Supplement U+2E80-U+2EFF + Extended-A U+2F00-U+2FDF
|
||||
_RADICAL_RANGES_FALLBACK = (
|
||||
(0x2E80, 0x2EFF), # CJK Radicals Supplement
|
||||
(0x2F00, 0x2FDF), # CJK Radicals Extended-A
|
||||
)
|
||||
|
||||
def _load_radical_codepoints() -> frozenset:
|
||||
"""从 unicode_radicals.md 解析「Unicode」列,返回码点集合。
|
||||
|
||||
行格式: | 亻 | U+2E80 | 单人旁 | 件→亻牛 | 件 |
|
||||
Unicode 列必须是 U+XXXX / U+XXXXX 十六进制;解析失败的行跳过。
|
||||
"""
|
||||
cps = set()
|
||||
try:
|
||||
with open(_RADICALS_MD, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
m = re.search(r"U\+([0-9A-Fa-f]{4,5})\b", line)
|
||||
if m:
|
||||
cps.add(int(m.group(1), 16))
|
||||
except OSError as e:
|
||||
logger.warning(f"[ocr] 偏旁库 {_RADICALS_MD} 读取失败({e}),回退码位范围")
|
||||
return frozenset()
|
||||
if not cps:
|
||||
logger.warning("[ocr] 偏旁库为空,回退码位范围")
|
||||
return frozenset()
|
||||
logger.debug(f"[ocr] 偏旁库加载 {len(cps)} 个码点 <- {_RADICALS_MD}")
|
||||
return frozenset(cps)
|
||||
|
||||
_RADICAL_CODEPOINTS = _load_radical_codepoints()
|
||||
|
||||
def _is_radical_char(ch: str) -> bool:
|
||||
"""检查字符是否为 CJK 偏旁字符(U+2E80-U+2FDF)。"""
|
||||
"""检查字符是否为 CJK 偏旁字符。
|
||||
|
||||
优先查 md 库的 Unicode 码点(维护只改 md);
|
||||
库未加载成功时回退到硬编码码位范围(保底)。
|
||||
"""
|
||||
cp = ord(ch)
|
||||
return any(lo <= cp <= hi for lo, hi in _RADICAL_RANGES)
|
||||
if _RADICAL_CODEPOINTS:
|
||||
return cp in _RADICAL_CODEPOINTS
|
||||
return any(lo <= cp <= hi for lo, hi in _RADICAL_RANGES_FALLBACK)
|
||||
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
Reference in New Issue
Block a user