[Bugfix] exit when oom (#1163)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu authored and GitHub committed 2026-02-03 15:03:54 +08:00
1 parent 9505ba30ab
commit 2ec1fc3c9c
2 files changed
+31 -11

No files matched your search

+24
View File
@@ -471,6 +471,29 @@ class AsyncOmni(OmniBase):
else: else:
logger.debug(f"[{self._name}] Request {req_id} fully completed") logger.debug(f"[{self._name}] Request {req_id} fully completed")
def handle_oom(self, result: dict[str, Any], stage: OmniStage, stage_id: int) -> None:
if getattr(stage, "stage_type", None) == "diffusion":
error_text = str(result.get("error", ""))
error_lower = error_text.lower()
is_oom = "out of memory" in error_lower or "cuda oom" in error_lower
if is_oom:
logger.critical(
f"[{self._name}] Diffusion stage {stage_id} reported OOM; stopping stage worker.",
)
try:
stage.stop_stage_worker()
except Exception as e:
logger.warning(
f"[{self._name}] Failed to stop diffusion stage {stage_id}: {e}",
)
logger.critical(
f"[{self._name}] Exiting API process due to diffusion stage {stage_id} OOM.",
)
import os as _os
_os._exit(1)
def _process_single_result( def _process_single_result(
self, self,
result: dict[str, Any], result: dict[str, Any],
@@ -493,6 +516,7 @@ class AsyncOmni(OmniBase):
logger.error( logger.error(
f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}", f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}",
) )
self.handle_oom(result, stage, stage_id) # only temporary workaround for oom
raise RuntimeError(result) raise RuntimeError(result)
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm") engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
+7 -11
View File
@@ -147,17 +147,13 @@ class AsyncOmniDiffusion:
# Run engine in thread pool # Run engine in thread pool
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
try: # In async mode, only a single request is submitted at a time
# In async mode, only a single request is submitted at a time result = await loop.run_in_executor(
result = await loop.run_in_executor( self._executor,
self._executor, self.engine.step,
self.engine.step, request,
request, )
) result = result[0]
result = result[0]
except Exception as e:
logger.error("Generation failed for request %s: %s", request_id, e)
raise RuntimeError(f"Diffusion generation failed: {e}") from e
# Update request_id if needed # Update request_id if needed
if not result.request_id: if not result.request_id: