[Enhancement] Engine runtime errors (#2426)

Signed-off-by: Daniel Huang <daniel1.huang@intel.com>
Signed-off-by: Daniel Huang <pilotflyer824@gmail.com>
Signed-off-by: bjf-frz <frz123db@gmail.com>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-04-21 14:46:05 +08:00
1 parent c1ba86ab5b
commit 52b53367ef
23 files changed
+1597 -98

No files matched your search

@@ -1,17 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import multiprocessing as mp
import queue
import threading
import time
from types import SimpleNamespace
from unittest.mock import MagicMock, Mock
import pytest
import torch
import zmq
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
from vllm_omni.diffusion.executor.multiproc_executor import MultiprocDiffusionExecutor
from vllm_omni.diffusion.sched import RequestScheduler
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.diffusion, pytest.mark.core_model, pytest.mark.cpu]
@@ -51,6 +59,8 @@ def _make_executor(num_gpus: int = 1):
executor._result_mq = mock_rmq
executor._closed = False
executor._processes = []
executor.is_failed = False
executor._failure_callbacks = []
return executor, req_q, res_q
@@ -334,8 +344,9 @@ class TestSerialEngineOperations:
class TestCollectiveRpcTimeoutWhileLockHeld:
"""``collective_rpc(timeout=...)`` must honour its timeout even when
another thread holds ``engine._rpc_lock`` indefinitely (e.g. a stalled
``add_req`` waiting on an unresponsive worker).
another thread holds ``engine._rpc_lock`` indefinitely (e.g. request
execution stalled on ``add_req_and_wait_for_response`` → ``execute_fn``
→ ``collective_rpc`` while blocked on an unresponsive worker).
"""
def test_rpc_times_out_when_lock_held_directly(self):
@@ -359,10 +370,10 @@ class TestCollectiveRpcTimeoutWhileLockHeld:
with pytest.raises(TimeoutError):
engine.collective_rpc("health", timeout=0.5)
def test_rpc_times_out_when_add_req_stalled_on_worker(self):
def test_rpc_times_out_when_request_execution_stalled_on_worker(self):
"""Real-world scenario the bot flagged:
``add_req`` holds ``_rpc_lock`` while blocked on
The scheduler/execute path holds ``_rpc_lock`` while blocked on
``executor._result_mq.dequeue()`` because the worker never replies.
A concurrent ``collective_rpc(timeout=...)`` must still time out
instead of hanging forever waiting for the lock.
@@ -428,3 +439,353 @@ class TestCollectiveRpcTimeoutWhileLockHeld:
t.join(5)
assert result.error == "ok"
# ───────── error handling: EngineDeadError propagation through layers ─────
class TestMultiprocExecutorRaisesEngineDeadError:
"""``collective_rpc`` raises ``EngineDeadError`` when the engine is failed."""
def test_collective_rpc_raises_when_is_failed(self):
executor = object.__new__(MultiprocDiffusionExecutor)
executor._closed = False
executor._broadcast_mq = MagicMock()
executor._result_mq = MagicMock()
executor._result_mq.dequeue = MagicMock(side_effect=TimeoutError)
executor.is_failed = True
with pytest.raises(EngineDeadError):
executor.collective_rpc(
"generate",
args=(MagicMock(),),
unique_reply_rank=0,
exec_all_ranks=True,
)
def test_collective_rpc_raises_mid_dequeue_when_is_failed(self):
"""Worker dies while we are polling the dequeue loop."""
executor, _, res_q = _make_executor()
call_count = 0
orig_dequeue = executor._result_mq.dequeue
def _dying_dequeue(timeout=None):
nonlocal call_count
call_count += 1
if call_count == 1:
executor.is_failed = True
raise TimeoutError
return orig_dequeue(timeout=timeout)
executor._result_mq.dequeue = _dying_dequeue
with pytest.raises(EngineDeadError):
executor.collective_rpc(
"generate",
args=(MagicMock(),),
unique_reply_rank=0,
exec_all_ranks=True,
)
class TestDiffusionEngineDeadErrorPassthrough:
"""``DiffusionEngine.add_req_and_wait_for_response`` re-raises
``EngineDeadError`` from executor and wraps other errors."""
def test_engine_dead_error_propagates(self):
engine, executor, _, _ = _make_engine()
engine.execute_fn = Mock(side_effect=EngineDeadError())
with pytest.raises(EngineDeadError):
engine.add_req_and_wait_for_response(_mock_request("dead"))
def test_runtime_error_wrapped_in_output(self):
engine, executor, _, _ = _make_engine()
engine.execute_fn = Mock(side_effect=RuntimeError("gpu fault"))
out = engine.add_req_and_wait_for_response(_mock_request("fault"))
assert isinstance(out, DiffusionOutput)
assert "gpu fault" in out.error
class TestStageDiffusionClientErrorPropagation:
"""Error surface behaviour of ``StageDiffusionClient``.
Uses ``object.__new__`` to construct a client without spawning a real
subprocess, then manually sets the fields needed for each test.
"""
def _make_client(self, *, engine_dead=False, proc_alive=True):
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
client = object.__new__(StageDiffusionClient)
client.stage_id = 0
client.final_output = True
client.final_output_type = "image"
client.default_sampling_params = None
client.custom_process_input_func = None
client.engine_input_source = None
client._output_queue = asyncio.Queue()
client._rpc_results = {}
client._pending_rpcs = set()
client._tasks = {}
client._shutting_down = False
client._engine_dead = engine_dead
client._owns_process = True
client._proc = MagicMock(
is_alive=MagicMock(return_value=proc_alive),
exitcode=1,
)
client._request_socket = MagicMock()
client._response_socket = MagicMock()
client._encoder = MagicMock()
client._decoder = MagicMock()
return client
@pytest.mark.asyncio
async def test_add_request_raises_when_dead(self):
client = self._make_client(engine_dead=True)
with pytest.raises(EngineDeadError):
await client.add_request_async("req-3", "test prompt", None)
def test_check_health_raises_when_dead(self):
client = self._make_client(engine_dead=True)
with pytest.raises(EngineDeadError):
client.check_health()
def test_check_health_ok_when_alive(self):
client = self._make_client()
client.check_health()
def test_get_output_raises_engine_dead_when_dead(self):
"""When ``_engine_dead`` is True and the output queue is empty,
``get_diffusion_output_nowait`` must raise ``EngineDeadError``."""
client = self._make_client(engine_dead=True)
# Simulate _drain_responses as a no-op (no ZMQ socket)
client._response_socket.recv.side_effect = zmq.Again
with pytest.raises(EngineDeadError):
client.get_diffusion_output_nowait()
def test_get_output_returns_none_when_alive_and_empty(self):
"""When the engine is alive and the queue is empty, return None."""
client = self._make_client()
client._response_socket.recv.side_effect = zmq.Again
assert client.get_diffusion_output_nowait() is None
def test_check_health_raises_when_proc_dead(self):
"""``check_health`` detects a dead subprocess via ``_proc.is_alive()``
and raises ``EngineDeadError``, setting ``_engine_dead`` as a
side effect."""
client = self._make_client(proc_alive=False)
with pytest.raises(EngineDeadError, match="not alive"):
client.check_health()
assert client._engine_dead is True
def test_get_output_raises_when_proc_dead(self):
"""When the subprocess has died (non-signal exit) and the output
queue is empty, ``get_diffusion_output_nowait`` must raise
``EngineDeadError`` with the exit code."""
client = self._make_client(proc_alive=False)
client._response_socket.recv.side_effect = zmq.Again
with pytest.raises(EngineDeadError, match="exit code"):
client.get_diffusion_output_nowait()
assert client._engine_dead is True
def test_get_output_returns_none_on_signal_death(self):
"""When the subprocess was killed by a signal (exit code > 128),
``get_diffusion_output_nowait`` returns ``None`` and sets
``_shutting_down`` instead of raising."""
client = self._make_client(proc_alive=False)
client._proc.exitcode = 137 # SIGKILL (128 + 9)
client._response_socket.recv.side_effect = zmq.Again
result = client.get_diffusion_output_nowait()
assert result is None
assert client._shutting_down is True
assert client._engine_dead is True
# ───────── monitor thread & death sentinel integration tests ─────────
def _poll_flag(get_flag, *, timeout=5.0, interval=0.05) -> bool:
"""Poll until ``get_flag()`` returns True or *timeout* elapses."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if get_flag():
return True
time.sleep(interval)
return False
def _make_short_lived_process() -> mp.Process:
"""Spawn a real subprocess that exits immediately.
The process must be started with ``"fork"`` (or the platform default)
so that it can use a plain ``lambda`` as its target — ``"spawn"`` would
fail to pickle it.
"""
ctx = mp.get_context("fork")
p = ctx.Process(target=lambda: None, name="ShortLivedWorker-0")
p.start()
return p
class TestMultiprocExecutorWorkerMonitor:
"""Integration tests for ``start_worker_monitor``.
Uses real short-lived subprocesses so that OS-level sentinel fd
readiness is exercised end-to-end.
"""
def test_worker_monitor_sets_is_failed_and_calls_callbacks_on_death(self):
"""When a worker process dies, the monitor thread must:
1. Set ``is_failed = True``
2. Call ``shutdown()`` (which sets ``_closed = True``)
3. Invoke all registered failure callbacks
"""
executor = object.__new__(MultiprocDiffusionExecutor)
executor._closed = False
executor.is_failed = False
executor._failure_callbacks = []
executor._broadcast_mq = None
executor._result_mq = None
executor.resources = None
# Use a no-op so shutdown() doesn't crash on None resources.
executor._finalizer = lambda: None
proc = _make_short_lived_process()
executor._processes = [proc]
callback_called = threading.Event()
executor.register_failure_callback(callback_called.set)
executor.start_worker_monitor()
# Wait for the process to exit and the monitor to react.
proc.join(5)
assert _poll_flag(lambda: executor.is_failed), "is_failed was not set"
assert executor._closed, "shutdown() was not called"
assert callback_called.wait(timeout=2), "failure callback was not invoked"
def test_worker_monitor_noop_when_already_closed(self):
"""If ``_closed`` is already True when the process dies (orderly
shutdown), the monitor must *not* set ``is_failed``."""
executor = object.__new__(MultiprocDiffusionExecutor)
executor._closed = True # already shut down
executor.is_failed = False
executor._failure_callbacks = []
executor._broadcast_mq = None
executor._result_mq = None
executor.resources = None
executor._finalizer = lambda: None
proc = _make_short_lived_process()
executor._processes = [proc]
executor.start_worker_monitor()
proc.join(5)
# Give the monitor thread a chance to run (it should early-return).
time.sleep(0.3)
assert not executor.is_failed, "is_failed should remain False on orderly shutdown"
class TestStageDiffusionClientProcMonitor:
"""Integration test for ``StageDiffusionClient._start_proc_monitor``.
Uses a real short-lived subprocess to verify the sentinel-based
detection pipeline.
"""
def test_proc_monitor_sets_engine_dead_on_process_death(self):
"""When the subprocess dies, the monitor thread must set
``_engine_dead = True``."""
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
client = object.__new__(StageDiffusionClient)
client.stage_id = 0
client._shutting_down = False
client._engine_dead = False
proc = _make_short_lived_process()
client._proc = proc
client._start_proc_monitor()
proc.join(5)
assert _poll_flag(lambda: client._engine_dead), "_engine_dead was not set"
class TestDrainResponsesDeathSentinel:
"""Tests for death sentinel and error routing in
``StageDiffusionClient._drain_responses()``.
"""
def _make_client(self):
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
client = object.__new__(StageDiffusionClient)
client.stage_id = 0
client._engine_dead = False
client._shutting_down = False
client._output_queue = asyncio.Queue()
client._rpc_results = {}
client._pending_rpcs = set()
client._response_socket = MagicMock()
client._decoder = MagicMock()
return client
def test_drain_responses_sets_engine_dead_on_death_sentinel(self):
"""When ``_drain_responses`` receives the ``DIFFUSION_PROC_DEAD``
sentinel, it must set ``_engine_dead = True`` and stop draining
(decoder is never called)."""
client = self._make_client()
# First recv returns the death sentinel, second would be a normal
# message but should never be reached.
client._response_socket.recv.side_effect = [
StageDiffusionProc.DIFFUSION_PROC_DEAD,
b"should-not-be-reached",
]
client._drain_responses()
assert client._engine_dead is True
client._decoder.decode.assert_not_called()
def test_drain_responses_routes_error_as_omni_request_output(self):
"""When ``_drain_responses`` receives a ``{"type": "error"}`` message
with a ``request_id``, it must place an ``OmniRequestOutput`` with
the error on ``_output_queue``."""
client = self._make_client()
error_msg = {
"type": "error",
"request_id": "req-fail",
"error": "gpu fault",
}
# First recv returns the encoded error, second raises zmq.Again.
client._response_socket.recv.side_effect = [b"encoded-error", zmq.Again]
client._decoder.decode.return_value = error_msg
client._drain_responses()
assert not client._output_queue.empty()
output = client._output_queue.get_nowait()
assert isinstance(output, OmniRequestOutput)
assert output.request_id == "req-fail"
assert output.error == "gpu fault"
assert output.finished is True
@@ -63,3 +63,39 @@ async def test_try_get_output_async_raises_after_orchestrator_dies(mocker: Mocke
with pytest.raises(RuntimeError, match="Orchestrator died unexpectedly"):
await engine.try_get_output_async()
def test_fatal_error_message_surfaces_through_try_get_output(mocker: MockerFixture):
"""When the orchestrator thread crashes, it enqueues a fatal error message.
``try_get_output`` must return this message so the caller
(``OmniBase._handle_output_message``) can detect the fatal flag.
"""
fatal_msg = {"type": "error", "error": "Orchestrator thread crashed", "fatal": True}
mock_queue = mocker.MagicMock()
mock_queue.sync_q.get.return_value = fatal_msg
engine = _make_engine(mock_queue, mocker, thread_alive=False)
msg = engine.try_get_output()
assert msg is not None
assert msg["type"] == "error"
assert msg["fatal"] is True
assert "crashed" in msg["error"]
@pytest.mark.asyncio
async def test_fatal_error_message_surfaces_through_try_get_output_async(mocker: MockerFixture):
"""Async variant of the fatal error message test."""
fatal_msg = {"type": "error", "error": "Orchestrator thread crashed", "fatal": True}
mock_queue = mocker.MagicMock()
mock_queue.sync_q.get_nowait.return_value = fatal_msg
engine = _make_engine(mock_queue, mocker, thread_alive=False)
msg = await engine.try_get_output_async()
assert msg is not None
assert msg["type"] == "error"
assert msg["fatal"] is True
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for error propagation paths within the Orchestrator.
Covers:
- EngineDeadError from an LLM stage poll → fatal error broadcast + shutdown
- Diffusion stage error output (OmniRequestOutput.from_error) → routed correctly
"""
from __future__ import annotations
import asyncio
import queue
import time
from types import SimpleNamespace
import pytest
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from .test_orchestrator import (
FakeStageClient,
OrchestratorFixture,
_build_harness,
_enqueue_add_request,
_wait_for,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _sampling_params(max_tokens: int = 4):
from vllm.sampling_params import SamplingParams
return SamplingParams(max_tokens=max_tokens)
async def _get_any_output_message(fixture: OrchestratorFixture, *, timeout: float = 2.0) -> dict:
"""Like _get_output_message but returns any message type (including errors)."""
deadline = time.monotonic() + timeout
while True:
if time.monotonic() >= deadline:
raise AssertionError("Timed out waiting for orchestrator output")
try:
return fixture.output_sync_q.get_nowait()
except queue.Empty:
await asyncio.sleep(0.01)
@pytest.fixture
def orchestrator_factory():
fixtures: list[OrchestratorFixture] = []
def _factory(*args, **kwargs) -> OrchestratorFixture:
fixture = _build_harness(*args, **kwargs)
fixtures.append(fixture)
return fixture
yield _factory
for fixture in fixtures:
if fixture.thread.is_alive():
fixture.request_sync_q.put_nowait({"type": "shutdown"})
fixture.thread.join(timeout=5)
for q in fixture.queues:
q.close()
# ───────── EngineDeadError from LLM stage poll ─────────
class FakeDeadLLMStageClient(FakeStageClient):
"""LLM stage client that raises EngineDeadError on get_output_async."""
async def get_output_async(self):
raise EngineDeadError("Stage-0 engine core is dead")
@pytest.mark.asyncio
async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_factory) -> None:
"""When a stage raises EngineDeadError during poll, the orchestrator must:
1. Enqueue a fatal error message for each affected request
2. Shut itself down (thread exits)
"""
stage0 = FakeDeadLLMStageClient(stage_type="llm", final_output=True)
orchestrator_fixture = orchestrator_factory([stage0])
request = SimpleNamespace(request_id="req-dead", prompt_token_ids=[1, 2])
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-dead",
prompt=request,
original_prompt={"prompt": "hello"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
# Collect the fatal error message.
msg = await _get_any_output_message(orchestrator_fixture)
assert msg["type"] == "error"
assert msg["fatal"] is True
assert msg["request_id"] == "req-dead"
assert "Stage-0 engine core is dead" in msg["error"]
# The orchestrator thread should exit after the fatal error.
orchestrator_fixture.thread.join(timeout=5)
assert not orchestrator_fixture.thread.is_alive()
# Request state should be cleaned up.
assert "req-dead" not in orchestrator_fixture.orchestrator.request_states
finally:
if orchestrator_fixture.thread.is_alive():
orchestrator_fixture.request_sync_q.put_nowait({"type": "shutdown"})
orchestrator_fixture.thread.join(timeout=5)
# ───────── Diffusion stage error output routing ─────────
@pytest.mark.asyncio
async def test_diffusion_error_output_routed_as_finished(orchestrator_factory) -> None:
"""When a diffusion stage returns an OmniRequestOutput with a non-None
error, the orchestrator must route it as a finished output message and
clean up the request state.
"""
stage0 = FakeStageClient(stage_type="diffusion", final_output=True, final_output_type="image")
orchestrator_fixture = orchestrator_factory([stage0])
params = OmniDiffusionSamplingParams()
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-err",
prompt={"prompt": "draw a cat"},
original_prompt={"prompt": "draw a cat"},
sampling_params_list=[params],
final_stage_id=0,
)
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
# Push an error output from the diffusion stage.
stage0.push_diffusion_output(OmniRequestOutput.from_error("req-err", "gpu fault"))
msg = await _get_any_output_message(orchestrator_fixture)
assert msg["type"] == "output"
assert msg["request_id"] == "req-err"
assert msg["finished"] is True
assert msg["engine_outputs"].error == "gpu fault"
# Request state should be cleaned up.
await _wait_for(lambda: "req-err" not in orchestrator_fixture.orchestrator.request_states)
finally:
orchestrator_fixture.request_sync_q.put_nowait({"type": "shutdown"})
orchestrator_fixture.thread.join(timeout=5)
@@ -0,0 +1,46 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for StageEngineCoreClient.check_health().
Uses object.__new__ to construct a minimal client — check_health only
touches self.resources, self.stage_id, and self._proc.
"""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_client(*, engine_dead=False, proc_alive=True):
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
client.resources = SimpleNamespace(engine_dead=engine_dead)
client._proc = MagicMock(is_alive=MagicMock(return_value=proc_alive), exitcode=1)
return client
def test_check_health_passes_when_alive():
client = _make_client(engine_dead=False, proc_alive=True)
client.check_health() # no exception
def test_check_health_raises_when_resources_engine_dead():
client = _make_client(engine_dead=True, proc_alive=True)
with pytest.raises(EngineDeadError, match="engine core is dead"):
client.check_health()
def test_check_health_raises_when_proc_not_alive():
client = _make_client(engine_dead=False, proc_alive=False)
with pytest.raises(EngineDeadError, match="not alive"):
client.check_health()
# Verify it set resources.engine_dead as a side effect
assert client.resources.engine_dead is True
@@ -407,6 +407,29 @@ def test_health_endpoint_no_engine():
assert data["status"] == "unhealthy"
def test_health_endpoint_dead_engine():
"""Health returns 503 when the engine raises EngineDeadError."""
from unittest.mock import AsyncMock
from fastapi import FastAPI
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.entrypoints.openai.api_server import router
app = FastAPI()
app.include_router(router)
dead_engine = AsyncMock()
dead_engine.check_health = AsyncMock(side_effect=EngineDeadError())
app.state.engine_client = dead_engine
client = TestClient(app)
response = client.get("/health")
assert response.status_code == 503
data = response.json()
assert data["status"] == "unhealthy"
def test_models_endpoint(test_client):
"""Test /v1/models endpoint for diffusion mode"""
response = test_client.get("/v1/models")
@@ -133,7 +133,6 @@ def _wait_for_status(client: TestClient, video_id: str, status: str, timeout_s:
last_payload = None
while time.time() < deadline:
response = client.get(f"/v1/videos/{video_id}")
assert response.status_code == 200
last_payload = response.json()
if last_payload["status"] == status:
return last_payload
@@ -644,7 +643,7 @@ def test_invalid_lora_returns_400(test_client):
assert response.status_code == 200
video_id = response.json()["id"]
failed = _wait_for_status(test_client, video_id, VideoGenerationStatus.FAILED.value)
assert failed["error"]["code"] == "HTTPException"
assert failed["error"]["code"] == 400
assert "lora object" in failed["error"]["message"].lower()
+263
View File
@@ -4,11 +4,13 @@ import queue
from collections.abc import Callable
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.omni import Omni
@@ -163,6 +165,15 @@ def _patch_engine(monkeypatch: pytest.MonkeyPatch, engine: FakeAsyncOmniEngine)
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
def _make_base():
from vllm_omni.entrypoints.omni_base import OmniBase
obj = object.__new__(OmniBase)
obj.engine = MagicMock()
obj.request_states = {}
return obj
def _stage_spec(
stage_id: int,
*,
@@ -687,3 +698,255 @@ def test_omni_forces_final_only_on_llm_stages(monkeypatch: pytest.MonkeyPatch):
assert submitted_params[1].output_kind == RequestOutputKind.FINAL_ONLY
assert submitted_params[2].output_kind == original_diffusion_output_kind
assert len(outputs) == 2
def test_fatal_error_raises_engine_dead():
base = _make_base()
msg = {"type": "error", "error": "orchestrator crashed", "fatal": True}
with pytest.raises(EngineDeadError, match="orchestrator crashed"):
base._handle_output_message(msg)
def test_non_fatal_error_raises_runtime():
base = _make_base()
msg = {"type": "error", "error": "something wrong"}
with pytest.raises(RuntimeError, match="something wrong"):
base._handle_output_message(msg)
def test_async_omni_errored_property_alive():
omni = object.__new__(AsyncOmni)
omni.engine = SimpleNamespace(
is_alive=lambda: True,
stage_clients=[SimpleNamespace(is_comprehension=False)],
)
assert omni.errored is False
def test_async_omni_errored_property_dead_engine():
omni = object.__new__(AsyncOmni)
omni.engine = SimpleNamespace(
is_alive=lambda: False,
stage_clients=[SimpleNamespace(is_comprehension=False)],
)
assert omni.errored is True
def test_async_omni_errored_property_dead_stage():
omni = object.__new__(AsyncOmni)
dead_stage = SimpleNamespace(is_comprehension=False, _engine_dead=True)
omni.engine = SimpleNamespace(
is_alive=lambda: True,
stage_clients=[dead_stage],
)
assert omni.errored is True
def _enqueue_stage_error(
engine: FakeAsyncOmniEngine,
msg,
*,
error_text: str,
kill_engine: bool = False,
):
"""Enqueue a stage error output, optionally killing the engine."""
if kill_engine:
engine._alive = False
engine.output_q.put_nowait(
{
"type": "output",
"request_id": msg["request_id"],
"stage_id": 0,
"engine_outputs": SimpleNamespace(
payload="",
finished=True,
images=[],
stage_durations={},
error=error_text,
),
"finished": False,
}
)
@pytest.mark.asyncio
async def test_async_omni_propagates_engine_dead_error(monkeypatch: pytest.MonkeyPatch):
"""When the engine is dead and an error output arrives, ``generate()``
must raise ``EngineDeadError`` (not plain ``RuntimeError``)."""
engine = FakeAsyncOmniEngine(
stage_metadata=THREE_STAGE_META,
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="worker OOM", kill_engine=True),
)
_patch_engine(monkeypatch, engine)
app = AsyncOmni("dummy-model")
try:
with pytest.raises(EngineDeadError, match="worker OOM"):
async for _ in app.generate(prompt="hello", request_id="req-dead"):
pass
finally:
app.shutdown()
@pytest.mark.asyncio
async def test_async_omni_propagates_engine_generate_error(monkeypatch: pytest.MonkeyPatch):
"""When the engine is alive but a stage error occurs, ``generate()``
must raise ``EngineGenerateError`` (recoverable, not ``EngineDeadError``)."""
engine = FakeAsyncOmniEngine(
stage_metadata=THREE_STAGE_META,
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="diffusion step failed"),
)
_patch_engine(monkeypatch, engine)
app = AsyncOmni("dummy-model")
try:
with pytest.raises(EngineGenerateError):
async for _ in app.generate(prompt="hello", request_id="req-recover"):
pass
finally:
app.shutdown()
# ───────── OmniBase.check_health() aggregation ─────────
def test_check_health_passes_when_all_healthy():
base = _make_base()
healthy_stage = MagicMock()
healthy_stage.check_health = MagicMock()
base.engine.is_alive.return_value = True
base.engine.stage_clients = [healthy_stage]
base.check_health() # should not raise
def test_check_health_raises_when_stage_dead():
base = _make_base()
dead_stage = MagicMock()
dead_stage.check_health = MagicMock(side_effect=EngineDeadError("Stage-1 dead"))
base.engine.is_alive.return_value = True
base.engine.stage_clients = [dead_stage]
with pytest.raises(EngineDeadError, match="Stage-1 dead"):
base.check_health()
def test_check_health_raises_when_orchestrator_dead():
base = _make_base()
base.engine.is_alive.return_value = False
base.engine.stage_clients = []
with pytest.raises(EngineDeadError, match="not alive"):
base.check_health()
# ───────── OmniBase.errored property ─────────
def test_omni_base_errored_false_when_alive():
base = _make_base()
base.engine.is_alive.return_value = True
base.engine.stage_clients = [SimpleNamespace()]
assert base.errored is False
def test_omni_base_errored_true_when_orchestrator_dead():
base = _make_base()
base.engine.is_alive.return_value = False
base.engine.stage_clients = []
assert base.errored is True
def test_omni_base_errored_true_when_stage_engine_dead():
base = _make_base()
base.engine.is_alive.return_value = True
dead_stage = SimpleNamespace(_engine_dead=True)
base.engine.stage_clients = [dead_stage]
assert base.errored is True
def test_omni_base_errored_true_when_stage_resources_engine_dead():
base = _make_base()
base.engine.is_alive.return_value = True
dead_stage = SimpleNamespace(resources=SimpleNamespace(engine_dead=True))
base.engine.stage_clients = [dead_stage]
assert base.errored is True
# ───────── Omni (sync) EngineDeadError / EngineGenerateError ─────────
def test_omni_propagates_engine_dead_error(monkeypatch: pytest.MonkeyPatch):
"""When the engine is dead and a stage error output arrives,
``Omni.generate()`` must raise ``EngineDeadError``."""
engine = FakeAsyncOmniEngine(
stage_metadata=THREE_STAGE_META,
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="worker OOM", kill_engine=True),
)
_patch_engine(monkeypatch, engine)
app = Omni("dummy-model")
try:
with pytest.raises(EngineDeadError, match="worker OOM"):
list(app.generate(["hello"], py_generator=False, use_tqdm=False))
finally:
app.shutdown()
def test_omni_propagates_engine_generate_error(monkeypatch: pytest.MonkeyPatch):
"""When the engine is alive but a stage error occurs,
``Omni.generate()`` must raise ``EngineGenerateError`` (recoverable)."""
engine = FakeAsyncOmniEngine(
stage_metadata=THREE_STAGE_META,
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="diffusion step failed"),
)
_patch_engine(monkeypatch, engine)
app = Omni("dummy-model")
try:
with pytest.raises(EngineGenerateError):
list(app.generate(["hello"], py_generator=False, use_tqdm=False))
finally:
app.shutdown()
def test_omni_errored_property_alive(monkeypatch: pytest.MonkeyPatch):
"""Omni.errored (inherited from OmniBase) returns False when healthy."""
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
_patch_engine(monkeypatch, engine)
app = Omni("dummy-model")
try:
assert app.errored is False
finally:
app.shutdown()
def test_omni_errored_property_dead_engine(monkeypatch: pytest.MonkeyPatch):
"""Omni.errored returns True when the orchestrator is dead."""
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
_patch_engine(monkeypatch, engine)
app = Omni("dummy-model")
try:
engine._alive = False
assert app.errored is True
finally:
app.shutdown()
def test_omni_errored_property_dead_stage(monkeypatch: pytest.MonkeyPatch):
"""Omni.errored returns True when a stage client is marked dead."""
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
_patch_engine(monkeypatch, engine)
app = Omni("dummy-model")
try:
engine.stage_clients[0]._engine_dead = True
assert app.errored is True
finally:
app.shutdown()