修复内存泄漏和回复截断问题

1. 内存泄漏修复(Bug D):
   - 在 easyocr_worker.py 中添加 GPU 清理函数和信号处理器
   - 在 monitor.py 中创建 _kill_easyocr_proc() 辅助方法
   - 使用 terminate→wait→kill→gc.collect() 安全终止序列替换所有直接 kill()
   - 确保 CUDA 上下文在子进程退出时正确清理

2. 回复截断修复(Bug E):
   - 在 knowledge_base.py 的 search() 方法中检测指南类查询
   - 对于步骤类查询(如何/怎么/教程等),合并同一文档的所有步骤 section
   - 确保用户获得完整的操作指南而非单个步骤
This commit is contained in:
youpaishidifu
2026-08-05 17:56:07 +08:00
parent 5b20037052
commit fd5d2ee73a
3 changed files with 105 additions and 54 deletions
+35
View File
@@ -282,8 +282,43 @@ def _main():
sys.stderr.flush() sys.stderr.flush()
def _cleanup_gpu():
"""退出时清理 CUDA/GPU 内存,防止子进程被 kill 后 GPU 内存泄漏到系统。"""
try:
import gc
gc.collect()
except Exception:
pass
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
except Exception:
pass
def _signal_handler(signum, frame):
"""捕获 SIGTERM/SIGINT,先清理 GPU 再退出(proc.terminate() 触发此处理)。"""
_cleanup_gpu()
sys.exit(0)
# 注册清理钩子:atexit(正常退出)+ signal(SIGTERM/SIGINT)
import atexit
atexit.register(_cleanup_gpu)
try:
import signal
signal.signal(signal.SIGTERM, _signal_handler)
signal.signal(signal.SIGINT, _signal_handler)
except Exception:
pass
if __name__ == "__main__": if __name__ == "__main__":
try: try:
_main() _main()
except Exception as e: except Exception as e:
_flush_exit(f"_main 崩溃: {e}\n{traceback.format_exc()}") _flush_exit(f"_main 崩溃: {e}\n{traceback.format_exc()}")
finally:
_cleanup_gpu()
+27
View File
@@ -795,6 +795,33 @@ class KnowledgeBase:
doc_sections.insert(0, doc_sections.pop(best_idx)) doc_sections.insert(0, doc_sections.pop(best_idx))
# 从每个文档取前 1-2 个 section # 从每个文档取前 1-2 个 section
# ⭐ 修复:当查询是操作指南(如何/怎么/教程)时,合并同一文档的所有相关 section
# 避免"如何通过官网更新皮肤"只返回最后一个步骤
is_guide_query = any(w in query_stripped for w in ['如何', '怎么', '教程', '步骤', '操作', '指南'])
if is_guide_query and len(doc_sections) > 1:
# 检查是否为操作指南类文档(多个步骤 section)
step_headings = {'1.', '2.', '3.', '4.', '5.', '步骤', '第一', '第二', '第三', '第四', '第五'}
step_count = sum(1 for _, doc, _ in doc_sections if any(s in doc.get('heading', '') for s in step_headings))
if step_count >= 3:
# 合并所有步骤 section 的 body
merged_body = ""
for idx, doc, score in doc_sections:
heading = doc.get('heading', '')
body = doc.get('body', doc.get('content', ''))
if body and len(body) > 10:
merged_body += f"\n{heading}\n{body}\n"
if merged_body:
# 创建合并后的文档
merged_doc = doc_sections[0][1].copy()
merged_doc['body'] = merged_body.strip()
merged_doc['content'] = merged_body.strip()
# 删除 answer 字段,使用 body
if 'answer' in merged_doc:
del merged_doc['answer']
results.append((merged_doc, float(doc_sections[0][2])))
continue
# 默认:从每个文档取前 1-2 个 section
sub_k = 2 if len(results) < top_k - 1 else top_k - len(results) sub_k = 2 if len(results) < top_k - 1 else top_k - len(results)
for i, (idx, doc, score) in enumerate(doc_sections): for i, (idx, doc, score) in enumerate(doc_sections):
if i >= sub_k: if i >= sub_k:
+43 -54
View File
@@ -492,14 +492,7 @@ class MessageMonitor:
self._emergency_thread.join(timeout=2) self._emergency_thread.join(timeout=2)
self._emergency_thread = None self._emergency_thread = None
# 杀死 EasyOCR 子进程(防止 GPU 推理残留吃资源/冻电脑) # 杀死 EasyOCR 子进程(防止 GPU 推理残留吃资源/冻电脑)
try: self._kill_easyocr_proc("stop")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
self._easyocr_proc.wait(timeout=3)
logger.info("[stop] EasyOCR 子进程已杀死")
except Exception as e:
logger.debug(f"[stop] 杀 EasyOCR 异常: {e}")
self._easyocr_proc = None
# ⭐ 重置 broken 标记:stop 后 start 应能重新拉起 EasyOCR # ⭐ 重置 broken 标记:stop 后 start 应能重新拉起 EasyOCR
self._easyocr_broken = False self._easyocr_broken = False
self._easyocr_retry_until = 0.0 # 清除崩溃冷却 self._easyocr_retry_until = 0.0 # 清除崩溃冷却
@@ -524,6 +517,40 @@ class MessageMonitor:
def is_running(self) -> bool: def is_running(self) -> bool:
return self._running return self._running
def _kill_easyocr_proc(self, reason: str = ""):
"""安全杀死 EasyOCR 子进程:terminate→等待→kill→GC,防 CUDA 内存泄漏。
terminate() 发 SIGTERM 让 worker 有机会执行 atexit/signal 清理 CUDA 缓存;
3s 后仍存活则 kill() 强杀;最后 gc.collect() 释放主进程侧引用。
"""
import gc
if not self._easyocr_proc or self._easyocr_proc.poll() is not None:
self._easyocr_proc = None
return
_tag = f"[ocr] 杀 EasyOCR 子进程({reason})" if reason else "[ocr] 杀 EasyOCR 子进程"
try:
# 1) 先 terminate 让 worker 执行 GPU 清理(atexit/signal handler)
self._easyocr_proc.terminate()
try:
self._easyocr_proc.wait(timeout=3)
logger.debug(f"{_tag}: terminate 后正常退出")
except Exception:
# 2) 3s 内未退出 → 强杀
try:
self._easyocr_proc.kill()
self._easyocr_proc.wait(timeout=3)
logger.warning(f"{_tag}: terminate 无效,已 kill")
except Exception:
pass
except Exception as e:
logger.debug(f"{_tag} 异常: {e}")
self._easyocr_proc = None
# 3) 主进程 GC:释放 PIL Image / numpy array 等残留引用
try:
gc.collect()
except Exception:
pass
# ── 内存看门狗(诊断用,定位泄漏源)────────────────────────── # ── 内存看门狗(诊断用,定位泄漏源)──────────────────────────
def _start_memory_watchdog(self): def _start_memory_watchdog(self):
@@ -709,14 +736,7 @@ class MessageMonitor:
self._sending_event.clear() self._sending_event.clear()
# 杀死 EasyOCR 子进程(GPU 推理可能卡死不响应) # 杀死 EasyOCR 子进程(GPU 推理可能卡死不响应)
try: self._kill_easyocr_proc("急停")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
self._easyocr_proc.wait(timeout=2)
logger.warning("[急停] EasyOCR 子进程已杀死")
except Exception as e:
logger.debug(f"[急停] 杀 EasyOCR 异常: {e}")
self._easyocr_proc = None
# ⭐ 清零全部 OCR/EasyOCR 计数器:急停后不残留(下次启动从头评估 GPU/CPU) # ⭐ 清零全部 OCR/EasyOCR 计数器:急停后不残留(下次启动从头评估 GPU/CPU)
# CUDA 报错降级 CPU 的计数、连续超时计数、截图指纹等全部归零 # CUDA 报错降级 CPU 的计数、连续超时计数、截图指纹等全部归零
@@ -1644,9 +1664,7 @@ class MessageMonitor:
line = _q.get(timeout=60) line = _q.get(timeout=60)
except _QueueEmpty: except _QueueEmpty:
logger.error("[ocr] EasyOCR 子进程初始化超时(60s),杀死进程,30s 后重试") logger.error("[ocr] EasyOCR 子进程初始化超时(60s),杀死进程,30s 后重试")
self._easyocr_proc.kill() self._kill_easyocr_proc("初始化超时")
self._easyocr_proc.wait(timeout=3)
self._easyocr_proc = None
self._easyocr_retry_until = time.time() + 30 self._easyocr_retry_until = time.time() + 30
return False return False
if not line: if not line:
@@ -1685,19 +1703,12 @@ class MessageMonitor:
pass pass
# 非 ready 响应 → 协议异常 # 非 ready 响应 → 协议异常
logger.error(f"[ocr] EasyOCR worker 异常响应: {line[:100]!r}") logger.error(f"[ocr] EasyOCR worker 异常响应: {line[:100]!r}")
self._easyocr_proc.kill() self._kill_easyocr_proc("协议异常")
self._easyocr_proc.wait(timeout=3)
self._easyocr_proc = None
self._easyocr_retry_until = time.time() + 30 self._easyocr_retry_until = time.time() + 30
return False return False
except Exception as e: except Exception as e:
logger.error(f"[ocr] EasyOCR 握手失败: {e}") logger.error(f"[ocr] EasyOCR 握手失败: {e}")
try: self._kill_easyocr_proc("握手失败")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
except Exception:
pass
self._easyocr_proc = None
self._easyocr_retry_until = time.time() + 30 self._easyocr_retry_until = time.time() + 30
return False return False
@@ -1782,12 +1793,7 @@ class MessageMonitor:
self._easyocr_consecutive_failures += 1 self._easyocr_consecutive_failures += 1
if self._easyocr_consecutive_failures >= self._easyocr_max_failures: if self._easyocr_consecutive_failures >= self._easyocr_max_failures:
logger.debug(f"[ocr] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来") logger.debug(f"[ocr] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来")
try: self._kill_easyocr_proc("连续超时")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
except Exception:
pass
self._easyocr_proc = None
self._easyocr_consecutive_failures = 0 self._easyocr_consecutive_failures = 0
self._easyocr_consecutive_startup_failures = 0 # 重置启动失败计数 self._easyocr_consecutive_startup_failures = 0 # 重置启动失败计数
else: else:
@@ -1812,13 +1818,7 @@ class MessageMonitor:
self._easyocr_call_count += 1 self._easyocr_call_count += 1
if self._easyocr_call_count >= self._easyocr_max_calls: if self._easyocr_call_count >= self._easyocr_max_calls:
logger.debug(f"[ocr] EasyOCR 正常资源释放:{self._easyocr_max_calls}次调用完成,重启子进程清GPU缓存(非错误,不计入失败计数器)") logger.debug(f"[ocr] EasyOCR 正常资源释放:{self._easyocr_max_calls}次调用完成,重启子进程清GPU缓存(非错误,不计入失败计数器)")
try: self._kill_easyocr_proc("正常资源释放")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
self._easyocr_proc.wait(timeout=3)
except Exception:
pass
self._easyocr_proc = None
self._easyocr_call_count = 0 self._easyocr_call_count = 0
return result["text"] return result["text"]
else: else:
@@ -1831,13 +1831,7 @@ class MessageMonitor:
logger.warning( logger.warning(
f"[ocr] EasyOCR CUDA 错误(累计{self._easyocr_cuda_oom_count}次)," f"[ocr] EasyOCR CUDA 错误(累计{self._easyocr_cuda_oom_count}次),"
f"杀子进程+冷却,{('下次强制CPU' if self._easyocr_cuda_oom_count >= 2 else '保持GPU')}") f"杀子进程+冷却,{('下次强制CPU' if self._easyocr_cuda_oom_count >= 2 else '保持GPU')}")
try: self._kill_easyocr_proc("CUDA错误")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
self._easyocr_proc.wait(timeout=3)
except Exception:
pass
self._easyocr_proc = None
self._easyocr_retry_until = time.time() + 30 self._easyocr_retry_until = time.time() + 30
return None return None
except (BrokenPipeError, ConnectionError, OSError) as e: except (BrokenPipeError, ConnectionError, OSError) as e:
@@ -2111,12 +2105,7 @@ class MessageMonitor:
self._easyocr_consecutive_failures += 1 self._easyocr_consecutive_failures += 1
if self._easyocr_consecutive_failures >= self._easyocr_max_failures: if self._easyocr_consecutive_failures >= self._easyocr_max_failures:
logger.warning(f"[{platform}] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来") logger.warning(f"[{platform}] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来")
try: self._kill_easyocr_proc("双引擎连续超时")
if self._easyocr_proc and self._easyocr_proc.poll() is None:
self._easyocr_proc.kill()
except Exception:
pass
self._easyocr_proc = None
self._easyocr_consecutive_failures = 0 self._easyocr_consecutive_failures = 0
self._easyocr_consecutive_startup_failures = 0 self._easyocr_consecutive_startup_failures = 0