fix(ocr): EasyOCR CUDA崩溃防护——健康检查+冻结隔离+自动降级
问题:EasyOCR worker启动即崩溃(access violation in torch CUDA), 冻住整台电脑,用户需强制重启。 修复三层防护: 1. worker端:CUDA健康检查——启动时创建小张量测试CUDA可用性, 失败则降级CPU,避免进入VGG模型加载的崩溃路径 2. 父进程端:CUDA_LAUNCH_BLOCKING=1——CUDA操作同步化, access violation时只杀子进程不冻整机(关键防冻结) 3. 父进程端:崩溃检测——读取崩溃日志,含access violation/torch 则增加_easyocr_cuda_oom_count,第二次崩溃后强制CPU模式 (_easyocr_cuda_oom_count>=2 → EASYOCR_FORCE_CPU=1)
This commit is contained in:
@@ -81,6 +81,18 @@ def _init_reader():
|
||||
# 父进程 EASYOCR_FORCE_CPU=1 → 强制 CPU(显存反复 OOM 时降级,防冻结)
|
||||
_force_cpu = os.environ.get("EASYOCR_FORCE_CPU") == "1"
|
||||
use_gpu = (not _force_cpu) and torch.cuda.is_available()
|
||||
if use_gpu:
|
||||
# CUDA 健康检查:尝试创建一个小张量,避免驱动/显存问题导致后续崩溃
|
||||
try:
|
||||
_test_tensor = torch.zeros(1, device="cuda")
|
||||
del _test_tensor
|
||||
torch.cuda.empty_cache()
|
||||
sys.stderr.write(f"[easyocr_worker] CUDA 健康检查通过 (设备: {torch.cuda.get_device_name(0)})\n")
|
||||
sys.stderr.flush()
|
||||
except Exception as cuda_err:
|
||||
sys.stderr.write(f"[easyocr_worker] CUDA 健康检查失败({cuda_err}),降级 CPU\n")
|
||||
sys.stderr.flush()
|
||||
use_gpu = False
|
||||
except Exception:
|
||||
use_gpu = False
|
||||
|
||||
|
||||
@@ -1240,6 +1240,8 @@ class MessageMonitor:
|
||||
|
||||
try:
|
||||
_worker_env = dict(os.environ)
|
||||
# ⭐ CUDA_LAUNCH_BLOCKING=1: CUDA 操作同步化,access violation 时只杀子进程不冻整机
|
||||
_worker_env["CUDA_LAUNCH_BLOCKING"] = "1"
|
||||
if self._easyocr_cuda_oom_count >= 2:
|
||||
# ⭐ 连续 CUDA OOM/kernel error → 强制 CPU 模式启动(显存不足时防冻结)
|
||||
_worker_env["EASYOCR_FORCE_CPU"] = "1"
|
||||
@@ -1292,6 +1294,10 @@ class MessageMonitor:
|
||||
if crash:
|
||||
detail = (detail + "\n[ocr] worker 崩溃日志(easyocr_crash.log):\n" + crash).strip()
|
||||
logger.error(f"[ocr] EasyOCR worker 启动即崩溃: {detail}")
|
||||
# ⭐ 检测 CUDA 相关崩溃(access violation / torch 错误)→ 增加 OOM 计数,下次强制 CPU
|
||||
if crash and ("access violation" in crash.lower() or "torch" in crash.lower()):
|
||||
self._easyocr_cuda_oom_count += 1
|
||||
logger.warning(f"[ocr] 检测到 CUDA 崩溃,累计{self._easyocr_cuda_oom_count}次,下次可能强制 CPU")
|
||||
self._easyocr_proc = None
|
||||
self._easyocr_retry_until = time.time() + 30
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user