fix(ocr): EasyOCR CUDA崩溃防护——健康检查+冻结隔离+自动降级

问题:EasyOCR worker启动即崩溃(access violation in torch CUDA),
冻住整台电脑,用户需强制重启。

修复三层防护:
1. worker端:CUDA健康检查——启动时创建小张量测试CUDA可用性,
   失败则降级CPU,避免进入VGG模型加载的崩溃路径
2. 父进程端:CUDA_LAUNCH_BLOCKING=1——CUDA操作同步化,
   access violation时只杀子进程不冻整机(关键防冻结)
3. 父进程端:崩溃检测——读取崩溃日志,含access violation/torch
   则增加_easyocr_cuda_oom_count,第二次崩溃后强制CPU模式
   (_easyocr_cuda_oom_count>=2 → EASYOCR_FORCE_CPU=1)
This commit is contained in:
youpaishidifu
2026-08-02 21:22:59 +08:00
parent b21e8f8cd4
commit a858bbb9f6
2 changed files with 18 additions and 0 deletions
+12
View File
@@ -81,6 +81,18 @@ def _init_reader():
# 父进程 EASYOCR_FORCE_CPU=1 → 强制 CPU(显存反复 OOM 时降级,防冻结)
_force_cpu = os.environ.get("EASYOCR_FORCE_CPU") == "1"
use_gpu = (not _force_cpu) and torch.cuda.is_available()
if use_gpu:
# CUDA 健康检查:尝试创建一个小张量,避免驱动/显存问题导致后续崩溃
try:
_test_tensor = torch.zeros(1, device="cuda")
del _test_tensor
torch.cuda.empty_cache()
sys.stderr.write(f"[easyocr_worker] CUDA 健康检查通过 (设备: {torch.cuda.get_device_name(0)})\n")
sys.stderr.flush()
except Exception as cuda_err:
sys.stderr.write(f"[easyocr_worker] CUDA 健康检查失败({cuda_err}),降级 CPU\n")
sys.stderr.flush()
use_gpu = False
except Exception:
use_gpu = False
+6
View File
@@ -1240,6 +1240,8 @@ class MessageMonitor:
try:
_worker_env = dict(os.environ)
# ⭐ CUDA_LAUNCH_BLOCKING=1: CUDA 操作同步化,access violation 时只杀子进程不冻整机
_worker_env["CUDA_LAUNCH_BLOCKING"] = "1"
if self._easyocr_cuda_oom_count >= 2:
# ⭐ 连续 CUDA OOM/kernel error → 强制 CPU 模式启动(显存不足时防冻结)
_worker_env["EASYOCR_FORCE_CPU"] = "1"
@@ -1292,6 +1294,10 @@ class MessageMonitor:
if crash:
detail = (detail + "\n[ocr] worker 崩溃日志(easyocr_crash.log):\n" + crash).strip()
logger.error(f"[ocr] EasyOCR worker 启动即崩溃: {detail}")
# ⭐ 检测 CUDA 相关崩溃(access violation / torch 错误)→ 增加 OOM 计数,下次强制 CPU
if crash and ("access violation" in crash.lower() or "torch" in crash.lower()):
self._easyocr_cuda_oom_count += 1
logger.warning(f"[ocr] 检测到 CUDA 崩溃,累计{self._easyocr_cuda_oom_count}次,下次可能强制 CPU")
self._easyocr_proc = None
self._easyocr_retry_until = time.time() + 30
return False