[Bugfix] exit when oom (#1163)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
9505ba30ab
commit
2ec1fc3c9c
2 files changed
+31
-11
No files matched your search
@@ -471,6 +471,29 @@ class AsyncOmni(OmniBase):
|
|||||||
else:
|
else:
|
||||||
logger.debug(f"[{self._name}] Request {req_id} fully completed")
|
logger.debug(f"[{self._name}] Request {req_id} fully completed")
|
||||||
|
|
||||||
|
def handle_oom(self, result: dict[str, Any], stage: OmniStage, stage_id: int) -> None:
|
||||||
|
if getattr(stage, "stage_type", None) == "diffusion":
|
||||||
|
error_text = str(result.get("error", ""))
|
||||||
|
error_lower = error_text.lower()
|
||||||
|
is_oom = "out of memory" in error_lower or "cuda oom" in error_lower
|
||||||
|
if is_oom:
|
||||||
|
logger.critical(
|
||||||
|
f"[{self._name}] Diffusion stage {stage_id} reported OOM; stopping stage worker.",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
stage.stop_stage_worker()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
f"[{self._name}] Failed to stop diffusion stage {stage_id}: {e}",
|
||||||
|
)
|
||||||
|
logger.critical(
|
||||||
|
f"[{self._name}] Exiting API process due to diffusion stage {stage_id} OOM.",
|
||||||
|
)
|
||||||
|
|
||||||
|
import os as _os
|
||||||
|
|
||||||
|
_os._exit(1)
|
||||||
|
|
||||||
def _process_single_result(
|
def _process_single_result(
|
||||||
self,
|
self,
|
||||||
result: dict[str, Any],
|
result: dict[str, Any],
|
||||||
@@ -493,6 +516,7 @@ class AsyncOmni(OmniBase):
|
|||||||
logger.error(
|
logger.error(
|
||||||
f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}",
|
f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}",
|
||||||
)
|
)
|
||||||
|
self.handle_oom(result, stage, stage_id) # only temporary workaround for oom
|
||||||
raise RuntimeError(result)
|
raise RuntimeError(result)
|
||||||
|
|
||||||
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
|
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
|
||||||
|
|||||||
@@ -147,17 +147,13 @@ class AsyncOmniDiffusion:
|
|||||||
|
|
||||||
# Run engine in thread pool
|
# Run engine in thread pool
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
try:
|
# In async mode, only a single request is submitted at a time
|
||||||
# In async mode, only a single request is submitted at a time
|
result = await loop.run_in_executor(
|
||||||
result = await loop.run_in_executor(
|
self._executor,
|
||||||
self._executor,
|
self.engine.step,
|
||||||
self.engine.step,
|
request,
|
||||||
request,
|
)
|
||||||
)
|
result = result[0]
|
||||||
result = result[0]
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("Generation failed for request %s: %s", request_id, e)
|
|
||||||
raise RuntimeError(f"Diffusion generation failed: {e}") from e
|
|
||||||
|
|
||||||
# Update request_id if needed
|
# Update request_id if needed
|
||||||
if not result.request_id:
|
if not result.request_id:
|
||||||
|
|||||||
Reference in new issue
Block a user