修复内存泄漏和回复截断问题
1. 内存泄漏修复(Bug D): - 在 easyocr_worker.py 中添加 GPU 清理函数和信号处理器 - 在 monitor.py 中创建 _kill_easyocr_proc() 辅助方法 - 使用 terminate→wait→kill→gc.collect() 安全终止序列替换所有直接 kill() - 确保 CUDA 上下文在子进程退出时正确清理 2. 回复截断修复(Bug E): - 在 knowledge_base.py 的 search() 方法中检测指南类查询 - 对于步骤类查询(如何/怎么/教程等),合并同一文档的所有步骤 section - 确保用户获得完整的操作指南而非单个步骤
This commit is contained in:
@@ -282,8 +282,43 @@ def _main():
|
|||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_gpu():
|
||||||
|
"""退出时清理 CUDA/GPU 内存,防止子进程被 kill 后 GPU 内存泄漏到系统。"""
|
||||||
|
try:
|
||||||
|
import gc
|
||||||
|
gc.collect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
import torch
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
torch.cuda.empty_cache()
|
||||||
|
torch.cuda.ipc_collect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _signal_handler(signum, frame):
|
||||||
|
"""捕获 SIGTERM/SIGINT,先清理 GPU 再退出(proc.terminate() 触发此处理)。"""
|
||||||
|
_cleanup_gpu()
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
# 注册清理钩子:atexit(正常退出)+ signal(SIGTERM/SIGINT)
|
||||||
|
import atexit
|
||||||
|
atexit.register(_cleanup_gpu)
|
||||||
|
try:
|
||||||
|
import signal
|
||||||
|
signal.signal(signal.SIGTERM, _signal_handler)
|
||||||
|
signal.signal(signal.SIGINT, _signal_handler)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
try:
|
try:
|
||||||
_main()
|
_main()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_flush_exit(f"_main 崩溃: {e}\n{traceback.format_exc()}")
|
_flush_exit(f"_main 崩溃: {e}\n{traceback.format_exc()}")
|
||||||
|
finally:
|
||||||
|
_cleanup_gpu()
|
||||||
|
|||||||
@@ -795,6 +795,33 @@ class KnowledgeBase:
|
|||||||
doc_sections.insert(0, doc_sections.pop(best_idx))
|
doc_sections.insert(0, doc_sections.pop(best_idx))
|
||||||
|
|
||||||
# 从每个文档取前 1-2 个 section
|
# 从每个文档取前 1-2 个 section
|
||||||
|
# ⭐ 修复:当查询是操作指南(如何/怎么/教程)时,合并同一文档的所有相关 section
|
||||||
|
# 避免"如何通过官网更新皮肤"只返回最后一个步骤
|
||||||
|
is_guide_query = any(w in query_stripped for w in ['如何', '怎么', '教程', '步骤', '操作', '指南'])
|
||||||
|
if is_guide_query and len(doc_sections) > 1:
|
||||||
|
# 检查是否为操作指南类文档(多个步骤 section)
|
||||||
|
step_headings = {'1.', '2.', '3.', '4.', '5.', '步骤', '第一', '第二', '第三', '第四', '第五'}
|
||||||
|
step_count = sum(1 for _, doc, _ in doc_sections if any(s in doc.get('heading', '') for s in step_headings))
|
||||||
|
if step_count >= 3:
|
||||||
|
# 合并所有步骤 section 的 body
|
||||||
|
merged_body = ""
|
||||||
|
for idx, doc, score in doc_sections:
|
||||||
|
heading = doc.get('heading', '')
|
||||||
|
body = doc.get('body', doc.get('content', ''))
|
||||||
|
if body and len(body) > 10:
|
||||||
|
merged_body += f"\n{heading}\n{body}\n"
|
||||||
|
if merged_body:
|
||||||
|
# 创建合并后的文档
|
||||||
|
merged_doc = doc_sections[0][1].copy()
|
||||||
|
merged_doc['body'] = merged_body.strip()
|
||||||
|
merged_doc['content'] = merged_body.strip()
|
||||||
|
# 删除 answer 字段,使用 body
|
||||||
|
if 'answer' in merged_doc:
|
||||||
|
del merged_doc['answer']
|
||||||
|
results.append((merged_doc, float(doc_sections[0][2])))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 默认:从每个文档取前 1-2 个 section
|
||||||
sub_k = 2 if len(results) < top_k - 1 else top_k - len(results)
|
sub_k = 2 if len(results) < top_k - 1 else top_k - len(results)
|
||||||
for i, (idx, doc, score) in enumerate(doc_sections):
|
for i, (idx, doc, score) in enumerate(doc_sections):
|
||||||
if i >= sub_k:
|
if i >= sub_k:
|
||||||
|
|||||||
+43
-54
@@ -492,14 +492,7 @@ class MessageMonitor:
|
|||||||
self._emergency_thread.join(timeout=2)
|
self._emergency_thread.join(timeout=2)
|
||||||
self._emergency_thread = None
|
self._emergency_thread = None
|
||||||
# 杀死 EasyOCR 子进程(防止 GPU 推理残留吃资源/冻电脑)
|
# 杀死 EasyOCR 子进程(防止 GPU 推理残留吃资源/冻电脑)
|
||||||
try:
|
self._kill_easyocr_proc("stop")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
self._easyocr_proc.wait(timeout=3)
|
|
||||||
logger.info("[stop] EasyOCR 子进程已杀死")
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"[stop] 杀 EasyOCR 异常: {e}")
|
|
||||||
self._easyocr_proc = None
|
|
||||||
# ⭐ 重置 broken 标记:stop 后 start 应能重新拉起 EasyOCR
|
# ⭐ 重置 broken 标记:stop 后 start 应能重新拉起 EasyOCR
|
||||||
self._easyocr_broken = False
|
self._easyocr_broken = False
|
||||||
self._easyocr_retry_until = 0.0 # 清除崩溃冷却
|
self._easyocr_retry_until = 0.0 # 清除崩溃冷却
|
||||||
@@ -524,6 +517,40 @@ class MessageMonitor:
|
|||||||
def is_running(self) -> bool:
|
def is_running(self) -> bool:
|
||||||
return self._running
|
return self._running
|
||||||
|
|
||||||
|
def _kill_easyocr_proc(self, reason: str = ""):
|
||||||
|
"""安全杀死 EasyOCR 子进程:terminate→等待→kill→GC,防 CUDA 内存泄漏。
|
||||||
|
|
||||||
|
terminate() 发 SIGTERM 让 worker 有机会执行 atexit/signal 清理 CUDA 缓存;
|
||||||
|
3s 后仍存活则 kill() 强杀;最后 gc.collect() 释放主进程侧引用。
|
||||||
|
"""
|
||||||
|
import gc
|
||||||
|
if not self._easyocr_proc or self._easyocr_proc.poll() is not None:
|
||||||
|
self._easyocr_proc = None
|
||||||
|
return
|
||||||
|
_tag = f"[ocr] 杀 EasyOCR 子进程({reason})" if reason else "[ocr] 杀 EasyOCR 子进程"
|
||||||
|
try:
|
||||||
|
# 1) 先 terminate 让 worker 执行 GPU 清理(atexit/signal handler)
|
||||||
|
self._easyocr_proc.terminate()
|
||||||
|
try:
|
||||||
|
self._easyocr_proc.wait(timeout=3)
|
||||||
|
logger.debug(f"{_tag}: terminate 后正常退出")
|
||||||
|
except Exception:
|
||||||
|
# 2) 3s 内未退出 → 强杀
|
||||||
|
try:
|
||||||
|
self._easyocr_proc.kill()
|
||||||
|
self._easyocr_proc.wait(timeout=3)
|
||||||
|
logger.warning(f"{_tag}: terminate 无效,已 kill")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"{_tag} 异常: {e}")
|
||||||
|
self._easyocr_proc = None
|
||||||
|
# 3) 主进程 GC:释放 PIL Image / numpy array 等残留引用
|
||||||
|
try:
|
||||||
|
gc.collect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
# ── 内存看门狗(诊断用,定位泄漏源)──────────────────────────
|
# ── 内存看门狗(诊断用,定位泄漏源)──────────────────────────
|
||||||
|
|
||||||
def _start_memory_watchdog(self):
|
def _start_memory_watchdog(self):
|
||||||
@@ -709,14 +736,7 @@ class MessageMonitor:
|
|||||||
self._sending_event.clear()
|
self._sending_event.clear()
|
||||||
|
|
||||||
# 杀死 EasyOCR 子进程(GPU 推理可能卡死不响应)
|
# 杀死 EasyOCR 子进程(GPU 推理可能卡死不响应)
|
||||||
try:
|
self._kill_easyocr_proc("急停")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
self._easyocr_proc.wait(timeout=2)
|
|
||||||
logger.warning("[急停] EasyOCR 子进程已杀死")
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"[急停] 杀 EasyOCR 异常: {e}")
|
|
||||||
self._easyocr_proc = None
|
|
||||||
|
|
||||||
# ⭐ 清零全部 OCR/EasyOCR 计数器:急停后不残留(下次启动从头评估 GPU/CPU)
|
# ⭐ 清零全部 OCR/EasyOCR 计数器:急停后不残留(下次启动从头评估 GPU/CPU)
|
||||||
# CUDA 报错降级 CPU 的计数、连续超时计数、截图指纹等全部归零
|
# CUDA 报错降级 CPU 的计数、连续超时计数、截图指纹等全部归零
|
||||||
@@ -1644,9 +1664,7 @@ class MessageMonitor:
|
|||||||
line = _q.get(timeout=60)
|
line = _q.get(timeout=60)
|
||||||
except _QueueEmpty:
|
except _QueueEmpty:
|
||||||
logger.error("[ocr] EasyOCR 子进程初始化超时(60s),杀死进程,30s 后重试")
|
logger.error("[ocr] EasyOCR 子进程初始化超时(60s),杀死进程,30s 后重试")
|
||||||
self._easyocr_proc.kill()
|
self._kill_easyocr_proc("初始化超时")
|
||||||
self._easyocr_proc.wait(timeout=3)
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_retry_until = time.time() + 30
|
self._easyocr_retry_until = time.time() + 30
|
||||||
return False
|
return False
|
||||||
if not line:
|
if not line:
|
||||||
@@ -1685,19 +1703,12 @@ class MessageMonitor:
|
|||||||
pass
|
pass
|
||||||
# 非 ready 响应 → 协议异常
|
# 非 ready 响应 → 协议异常
|
||||||
logger.error(f"[ocr] EasyOCR worker 异常响应: {line[:100]!r}")
|
logger.error(f"[ocr] EasyOCR worker 异常响应: {line[:100]!r}")
|
||||||
self._easyocr_proc.kill()
|
self._kill_easyocr_proc("协议异常")
|
||||||
self._easyocr_proc.wait(timeout=3)
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_retry_until = time.time() + 30
|
self._easyocr_retry_until = time.time() + 30
|
||||||
return False
|
return False
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[ocr] EasyOCR 握手失败: {e}")
|
logger.error(f"[ocr] EasyOCR 握手失败: {e}")
|
||||||
try:
|
self._kill_easyocr_proc("握手失败")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_retry_until = time.time() + 30
|
self._easyocr_retry_until = time.time() + 30
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -1782,12 +1793,7 @@ class MessageMonitor:
|
|||||||
self._easyocr_consecutive_failures += 1
|
self._easyocr_consecutive_failures += 1
|
||||||
if self._easyocr_consecutive_failures >= self._easyocr_max_failures:
|
if self._easyocr_consecutive_failures >= self._easyocr_max_failures:
|
||||||
logger.debug(f"[ocr] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来")
|
logger.debug(f"[ocr] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来")
|
||||||
try:
|
self._kill_easyocr_proc("连续超时")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_consecutive_failures = 0
|
self._easyocr_consecutive_failures = 0
|
||||||
self._easyocr_consecutive_startup_failures = 0 # 重置启动失败计数
|
self._easyocr_consecutive_startup_failures = 0 # 重置启动失败计数
|
||||||
else:
|
else:
|
||||||
@@ -1812,13 +1818,7 @@ class MessageMonitor:
|
|||||||
self._easyocr_call_count += 1
|
self._easyocr_call_count += 1
|
||||||
if self._easyocr_call_count >= self._easyocr_max_calls:
|
if self._easyocr_call_count >= self._easyocr_max_calls:
|
||||||
logger.debug(f"[ocr] EasyOCR 正常资源释放:{self._easyocr_max_calls}次调用完成,重启子进程清GPU缓存(非错误,不计入失败计数器)")
|
logger.debug(f"[ocr] EasyOCR 正常资源释放:{self._easyocr_max_calls}次调用完成,重启子进程清GPU缓存(非错误,不计入失败计数器)")
|
||||||
try:
|
self._kill_easyocr_proc("正常资源释放")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
self._easyocr_proc.wait(timeout=3)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_call_count = 0
|
self._easyocr_call_count = 0
|
||||||
return result["text"]
|
return result["text"]
|
||||||
else:
|
else:
|
||||||
@@ -1831,13 +1831,7 @@ class MessageMonitor:
|
|||||||
logger.warning(
|
logger.warning(
|
||||||
f"[ocr] EasyOCR CUDA 错误(累计{self._easyocr_cuda_oom_count}次),"
|
f"[ocr] EasyOCR CUDA 错误(累计{self._easyocr_cuda_oom_count}次),"
|
||||||
f"杀子进程+冷却,{('下次强制CPU' if self._easyocr_cuda_oom_count >= 2 else '保持GPU')}")
|
f"杀子进程+冷却,{('下次强制CPU' if self._easyocr_cuda_oom_count >= 2 else '保持GPU')}")
|
||||||
try:
|
self._kill_easyocr_proc("CUDA错误")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
self._easyocr_proc.wait(timeout=3)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_retry_until = time.time() + 30
|
self._easyocr_retry_until = time.time() + 30
|
||||||
return None
|
return None
|
||||||
except (BrokenPipeError, ConnectionError, OSError) as e:
|
except (BrokenPipeError, ConnectionError, OSError) as e:
|
||||||
@@ -2111,12 +2105,7 @@ class MessageMonitor:
|
|||||||
self._easyocr_consecutive_failures += 1
|
self._easyocr_consecutive_failures += 1
|
||||||
if self._easyocr_consecutive_failures >= self._easyocr_max_failures:
|
if self._easyocr_consecutive_failures >= self._easyocr_max_failures:
|
||||||
logger.warning(f"[{platform}] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来")
|
logger.warning(f"[{platform}] EasyOCR 连续{self._easyocr_consecutive_failures}次超时,杀子进程重来")
|
||||||
try:
|
self._kill_easyocr_proc("双引擎连续超时")
|
||||||
if self._easyocr_proc and self._easyocr_proc.poll() is None:
|
|
||||||
self._easyocr_proc.kill()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
self._easyocr_proc = None
|
|
||||||
self._easyocr_consecutive_failures = 0
|
self._easyocr_consecutive_failures = 0
|
||||||
self._easyocr_consecutive_startup_failures = 0
|
self._easyocr_consecutive_startup_failures = 0
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user