[Enhancement] Engine runtime errors (#2426)
Signed-off-by: Daniel Huang <daniel1.huang@intel.com> Signed-off-by: Daniel Huang <pilotflyer824@gmail.com> Signed-off-by: bjf-frz <frz123db@gmail.com> Co-authored-by: Chenguang Zheng <645327136@qq.com> Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
23 files changed
+1597
-98
No files matched your search
@@ -1,17 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import multiprocessing as mp
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import zmq
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||
from vllm_omni.diffusion.executor.multiproc_executor import MultiprocDiffusionExecutor
|
||||
from vllm_omni.diffusion.sched import RequestScheduler
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -51,6 +59,8 @@ def _make_executor(num_gpus: int = 1):
|
||||
executor._result_mq = mock_rmq
|
||||
executor._closed = False
|
||||
executor._processes = []
|
||||
executor.is_failed = False
|
||||
executor._failure_callbacks = []
|
||||
return executor, req_q, res_q
|
||||
|
||||
|
||||
@@ -334,8 +344,9 @@ class TestSerialEngineOperations:
|
||||
|
||||
class TestCollectiveRpcTimeoutWhileLockHeld:
|
||||
"""``collective_rpc(timeout=...)`` must honour its timeout even when
|
||||
another thread holds ``engine._rpc_lock`` indefinitely (e.g. a stalled
|
||||
``add_req`` waiting on an unresponsive worker).
|
||||
another thread holds ``engine._rpc_lock`` indefinitely (e.g. request
|
||||
execution stalled on ``add_req_and_wait_for_response`` → ``execute_fn``
|
||||
→ ``collective_rpc`` while blocked on an unresponsive worker).
|
||||
"""
|
||||
|
||||
def test_rpc_times_out_when_lock_held_directly(self):
|
||||
@@ -359,10 +370,10 @@ class TestCollectiveRpcTimeoutWhileLockHeld:
|
||||
with pytest.raises(TimeoutError):
|
||||
engine.collective_rpc("health", timeout=0.5)
|
||||
|
||||
def test_rpc_times_out_when_add_req_stalled_on_worker(self):
|
||||
def test_rpc_times_out_when_request_execution_stalled_on_worker(self):
|
||||
"""Real-world scenario the bot flagged:
|
||||
|
||||
``add_req`` holds ``_rpc_lock`` while blocked on
|
||||
The scheduler/execute path holds ``_rpc_lock`` while blocked on
|
||||
``executor._result_mq.dequeue()`` because the worker never replies.
|
||||
A concurrent ``collective_rpc(timeout=...)`` must still time out
|
||||
instead of hanging forever waiting for the lock.
|
||||
@@ -428,3 +439,353 @@ class TestCollectiveRpcTimeoutWhileLockHeld:
|
||||
t.join(5)
|
||||
|
||||
assert result.error == "ok"
|
||||
|
||||
|
||||
# ───────── error handling: EngineDeadError propagation through layers ─────
|
||||
|
||||
|
||||
class TestMultiprocExecutorRaisesEngineDeadError:
|
||||
"""``collective_rpc`` raises ``EngineDeadError`` when the engine is failed."""
|
||||
|
||||
def test_collective_rpc_raises_when_is_failed(self):
|
||||
executor = object.__new__(MultiprocDiffusionExecutor)
|
||||
executor._closed = False
|
||||
executor._broadcast_mq = MagicMock()
|
||||
executor._result_mq = MagicMock()
|
||||
executor._result_mq.dequeue = MagicMock(side_effect=TimeoutError)
|
||||
executor.is_failed = True
|
||||
|
||||
with pytest.raises(EngineDeadError):
|
||||
executor.collective_rpc(
|
||||
"generate",
|
||||
args=(MagicMock(),),
|
||||
unique_reply_rank=0,
|
||||
exec_all_ranks=True,
|
||||
)
|
||||
|
||||
def test_collective_rpc_raises_mid_dequeue_when_is_failed(self):
|
||||
"""Worker dies while we are polling the dequeue loop."""
|
||||
executor, _, res_q = _make_executor()
|
||||
|
||||
call_count = 0
|
||||
orig_dequeue = executor._result_mq.dequeue
|
||||
|
||||
def _dying_dequeue(timeout=None):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
executor.is_failed = True
|
||||
raise TimeoutError
|
||||
return orig_dequeue(timeout=timeout)
|
||||
|
||||
executor._result_mq.dequeue = _dying_dequeue
|
||||
|
||||
with pytest.raises(EngineDeadError):
|
||||
executor.collective_rpc(
|
||||
"generate",
|
||||
args=(MagicMock(),),
|
||||
unique_reply_rank=0,
|
||||
exec_all_ranks=True,
|
||||
)
|
||||
|
||||
|
||||
class TestDiffusionEngineDeadErrorPassthrough:
|
||||
"""``DiffusionEngine.add_req_and_wait_for_response`` re-raises
|
||||
``EngineDeadError`` from executor and wraps other errors."""
|
||||
|
||||
def test_engine_dead_error_propagates(self):
|
||||
engine, executor, _, _ = _make_engine()
|
||||
engine.execute_fn = Mock(side_effect=EngineDeadError())
|
||||
|
||||
with pytest.raises(EngineDeadError):
|
||||
engine.add_req_and_wait_for_response(_mock_request("dead"))
|
||||
|
||||
def test_runtime_error_wrapped_in_output(self):
|
||||
engine, executor, _, _ = _make_engine()
|
||||
engine.execute_fn = Mock(side_effect=RuntimeError("gpu fault"))
|
||||
|
||||
out = engine.add_req_and_wait_for_response(_mock_request("fault"))
|
||||
assert isinstance(out, DiffusionOutput)
|
||||
assert "gpu fault" in out.error
|
||||
|
||||
|
||||
class TestStageDiffusionClientErrorPropagation:
|
||||
"""Error surface behaviour of ``StageDiffusionClient``.
|
||||
|
||||
Uses ``object.__new__`` to construct a client without spawning a real
|
||||
subprocess, then manually sets the fields needed for each test.
|
||||
"""
|
||||
|
||||
def _make_client(self, *, engine_dead=False, proc_alive=True):
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
|
||||
client = object.__new__(StageDiffusionClient)
|
||||
client.stage_id = 0
|
||||
client.final_output = True
|
||||
client.final_output_type = "image"
|
||||
client.default_sampling_params = None
|
||||
client.custom_process_input_func = None
|
||||
client.engine_input_source = None
|
||||
|
||||
client._output_queue = asyncio.Queue()
|
||||
client._rpc_results = {}
|
||||
client._pending_rpcs = set()
|
||||
client._tasks = {}
|
||||
client._shutting_down = False
|
||||
client._engine_dead = engine_dead
|
||||
client._owns_process = True
|
||||
client._proc = MagicMock(
|
||||
is_alive=MagicMock(return_value=proc_alive),
|
||||
exitcode=1,
|
||||
)
|
||||
client._request_socket = MagicMock()
|
||||
client._response_socket = MagicMock()
|
||||
client._encoder = MagicMock()
|
||||
client._decoder = MagicMock()
|
||||
|
||||
return client
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_request_raises_when_dead(self):
|
||||
client = self._make_client(engine_dead=True)
|
||||
|
||||
with pytest.raises(EngineDeadError):
|
||||
await client.add_request_async("req-3", "test prompt", None)
|
||||
|
||||
def test_check_health_raises_when_dead(self):
|
||||
client = self._make_client(engine_dead=True)
|
||||
|
||||
with pytest.raises(EngineDeadError):
|
||||
client.check_health()
|
||||
|
||||
def test_check_health_ok_when_alive(self):
|
||||
client = self._make_client()
|
||||
client.check_health()
|
||||
|
||||
def test_get_output_raises_engine_dead_when_dead(self):
|
||||
"""When ``_engine_dead`` is True and the output queue is empty,
|
||||
``get_diffusion_output_nowait`` must raise ``EngineDeadError``."""
|
||||
client = self._make_client(engine_dead=True)
|
||||
# Simulate _drain_responses as a no-op (no ZMQ socket)
|
||||
client._response_socket.recv.side_effect = zmq.Again
|
||||
|
||||
with pytest.raises(EngineDeadError):
|
||||
client.get_diffusion_output_nowait()
|
||||
|
||||
def test_get_output_returns_none_when_alive_and_empty(self):
|
||||
"""When the engine is alive and the queue is empty, return None."""
|
||||
client = self._make_client()
|
||||
client._response_socket.recv.side_effect = zmq.Again
|
||||
|
||||
assert client.get_diffusion_output_nowait() is None
|
||||
|
||||
def test_check_health_raises_when_proc_dead(self):
|
||||
"""``check_health`` detects a dead subprocess via ``_proc.is_alive()``
|
||||
and raises ``EngineDeadError``, setting ``_engine_dead`` as a
|
||||
side effect."""
|
||||
client = self._make_client(proc_alive=False)
|
||||
|
||||
with pytest.raises(EngineDeadError, match="not alive"):
|
||||
client.check_health()
|
||||
|
||||
assert client._engine_dead is True
|
||||
|
||||
def test_get_output_raises_when_proc_dead(self):
|
||||
"""When the subprocess has died (non-signal exit) and the output
|
||||
queue is empty, ``get_diffusion_output_nowait`` must raise
|
||||
``EngineDeadError`` with the exit code."""
|
||||
client = self._make_client(proc_alive=False)
|
||||
client._response_socket.recv.side_effect = zmq.Again
|
||||
|
||||
with pytest.raises(EngineDeadError, match="exit code"):
|
||||
client.get_diffusion_output_nowait()
|
||||
|
||||
assert client._engine_dead is True
|
||||
|
||||
def test_get_output_returns_none_on_signal_death(self):
|
||||
"""When the subprocess was killed by a signal (exit code > 128),
|
||||
``get_diffusion_output_nowait`` returns ``None`` and sets
|
||||
``_shutting_down`` instead of raising."""
|
||||
client = self._make_client(proc_alive=False)
|
||||
client._proc.exitcode = 137 # SIGKILL (128 + 9)
|
||||
client._response_socket.recv.side_effect = zmq.Again
|
||||
|
||||
result = client.get_diffusion_output_nowait()
|
||||
|
||||
assert result is None
|
||||
assert client._shutting_down is True
|
||||
assert client._engine_dead is True
|
||||
|
||||
|
||||
# ───────── monitor thread & death sentinel integration tests ─────────
|
||||
|
||||
|
||||
def _poll_flag(get_flag, *, timeout=5.0, interval=0.05) -> bool:
|
||||
"""Poll until ``get_flag()`` returns True or *timeout* elapses."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if get_flag():
|
||||
return True
|
||||
time.sleep(interval)
|
||||
return False
|
||||
|
||||
|
||||
def _make_short_lived_process() -> mp.Process:
|
||||
"""Spawn a real subprocess that exits immediately.
|
||||
|
||||
The process must be started with ``"fork"`` (or the platform default)
|
||||
so that it can use a plain ``lambda`` as its target — ``"spawn"`` would
|
||||
fail to pickle it.
|
||||
"""
|
||||
ctx = mp.get_context("fork")
|
||||
p = ctx.Process(target=lambda: None, name="ShortLivedWorker-0")
|
||||
p.start()
|
||||
return p
|
||||
|
||||
|
||||
class TestMultiprocExecutorWorkerMonitor:
|
||||
"""Integration tests for ``start_worker_monitor``.
|
||||
|
||||
Uses real short-lived subprocesses so that OS-level sentinel fd
|
||||
readiness is exercised end-to-end.
|
||||
"""
|
||||
|
||||
def test_worker_monitor_sets_is_failed_and_calls_callbacks_on_death(self):
|
||||
"""When a worker process dies, the monitor thread must:
|
||||
1. Set ``is_failed = True``
|
||||
2. Call ``shutdown()`` (which sets ``_closed = True``)
|
||||
3. Invoke all registered failure callbacks
|
||||
"""
|
||||
executor = object.__new__(MultiprocDiffusionExecutor)
|
||||
executor._closed = False
|
||||
executor.is_failed = False
|
||||
executor._failure_callbacks = []
|
||||
executor._broadcast_mq = None
|
||||
executor._result_mq = None
|
||||
executor.resources = None
|
||||
# Use a no-op so shutdown() doesn't crash on None resources.
|
||||
executor._finalizer = lambda: None
|
||||
|
||||
proc = _make_short_lived_process()
|
||||
executor._processes = [proc]
|
||||
|
||||
callback_called = threading.Event()
|
||||
executor.register_failure_callback(callback_called.set)
|
||||
|
||||
executor.start_worker_monitor()
|
||||
|
||||
# Wait for the process to exit and the monitor to react.
|
||||
proc.join(5)
|
||||
assert _poll_flag(lambda: executor.is_failed), "is_failed was not set"
|
||||
assert executor._closed, "shutdown() was not called"
|
||||
assert callback_called.wait(timeout=2), "failure callback was not invoked"
|
||||
|
||||
def test_worker_monitor_noop_when_already_closed(self):
|
||||
"""If ``_closed`` is already True when the process dies (orderly
|
||||
shutdown), the monitor must *not* set ``is_failed``."""
|
||||
executor = object.__new__(MultiprocDiffusionExecutor)
|
||||
executor._closed = True # already shut down
|
||||
executor.is_failed = False
|
||||
executor._failure_callbacks = []
|
||||
executor._broadcast_mq = None
|
||||
executor._result_mq = None
|
||||
executor.resources = None
|
||||
executor._finalizer = lambda: None
|
||||
|
||||
proc = _make_short_lived_process()
|
||||
executor._processes = [proc]
|
||||
|
||||
executor.start_worker_monitor()
|
||||
proc.join(5)
|
||||
|
||||
# Give the monitor thread a chance to run (it should early-return).
|
||||
time.sleep(0.3)
|
||||
assert not executor.is_failed, "is_failed should remain False on orderly shutdown"
|
||||
|
||||
|
||||
class TestStageDiffusionClientProcMonitor:
|
||||
"""Integration test for ``StageDiffusionClient._start_proc_monitor``.
|
||||
|
||||
Uses a real short-lived subprocess to verify the sentinel-based
|
||||
detection pipeline.
|
||||
"""
|
||||
|
||||
def test_proc_monitor_sets_engine_dead_on_process_death(self):
|
||||
"""When the subprocess dies, the monitor thread must set
|
||||
``_engine_dead = True``."""
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
|
||||
client = object.__new__(StageDiffusionClient)
|
||||
client.stage_id = 0
|
||||
client._shutting_down = False
|
||||
client._engine_dead = False
|
||||
|
||||
proc = _make_short_lived_process()
|
||||
client._proc = proc
|
||||
|
||||
client._start_proc_monitor()
|
||||
proc.join(5)
|
||||
|
||||
assert _poll_flag(lambda: client._engine_dead), "_engine_dead was not set"
|
||||
|
||||
|
||||
class TestDrainResponsesDeathSentinel:
|
||||
"""Tests for death sentinel and error routing in
|
||||
``StageDiffusionClient._drain_responses()``.
|
||||
"""
|
||||
|
||||
def _make_client(self):
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
|
||||
client = object.__new__(StageDiffusionClient)
|
||||
client.stage_id = 0
|
||||
client._engine_dead = False
|
||||
client._shutting_down = False
|
||||
client._output_queue = asyncio.Queue()
|
||||
client._rpc_results = {}
|
||||
client._pending_rpcs = set()
|
||||
client._response_socket = MagicMock()
|
||||
client._decoder = MagicMock()
|
||||
return client
|
||||
|
||||
def test_drain_responses_sets_engine_dead_on_death_sentinel(self):
|
||||
"""When ``_drain_responses`` receives the ``DIFFUSION_PROC_DEAD``
|
||||
sentinel, it must set ``_engine_dead = True`` and stop draining
|
||||
(decoder is never called)."""
|
||||
client = self._make_client()
|
||||
|
||||
# First recv returns the death sentinel, second would be a normal
|
||||
# message but should never be reached.
|
||||
client._response_socket.recv.side_effect = [
|
||||
StageDiffusionProc.DIFFUSION_PROC_DEAD,
|
||||
b"should-not-be-reached",
|
||||
]
|
||||
|
||||
client._drain_responses()
|
||||
|
||||
assert client._engine_dead is True
|
||||
client._decoder.decode.assert_not_called()
|
||||
|
||||
def test_drain_responses_routes_error_as_omni_request_output(self):
|
||||
"""When ``_drain_responses`` receives a ``{"type": "error"}`` message
|
||||
with a ``request_id``, it must place an ``OmniRequestOutput`` with
|
||||
the error on ``_output_queue``."""
|
||||
client = self._make_client()
|
||||
|
||||
error_msg = {
|
||||
"type": "error",
|
||||
"request_id": "req-fail",
|
||||
"error": "gpu fault",
|
||||
}
|
||||
# First recv returns the encoded error, second raises zmq.Again.
|
||||
client._response_socket.recv.side_effect = [b"encoded-error", zmq.Again]
|
||||
client._decoder.decode.return_value = error_msg
|
||||
|
||||
client._drain_responses()
|
||||
|
||||
assert not client._output_queue.empty()
|
||||
output = client._output_queue.get_nowait()
|
||||
assert isinstance(output, OmniRequestOutput)
|
||||
assert output.request_id == "req-fail"
|
||||
assert output.error == "gpu fault"
|
||||
assert output.finished is True
|
||||
@@ -63,3 +63,39 @@ async def test_try_get_output_async_raises_after_orchestrator_dies(mocker: Mocke
|
||||
|
||||
with pytest.raises(RuntimeError, match="Orchestrator died unexpectedly"):
|
||||
await engine.try_get_output_async()
|
||||
|
||||
|
||||
def test_fatal_error_message_surfaces_through_try_get_output(mocker: MockerFixture):
|
||||
"""When the orchestrator thread crashes, it enqueues a fatal error message.
|
||||
|
||||
``try_get_output`` must return this message so the caller
|
||||
(``OmniBase._handle_output_message``) can detect the fatal flag.
|
||||
"""
|
||||
fatal_msg = {"type": "error", "error": "Orchestrator thread crashed", "fatal": True}
|
||||
|
||||
mock_queue = mocker.MagicMock()
|
||||
mock_queue.sync_q.get.return_value = fatal_msg
|
||||
|
||||
engine = _make_engine(mock_queue, mocker, thread_alive=False)
|
||||
|
||||
msg = engine.try_get_output()
|
||||
assert msg is not None
|
||||
assert msg["type"] == "error"
|
||||
assert msg["fatal"] is True
|
||||
assert "crashed" in msg["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fatal_error_message_surfaces_through_try_get_output_async(mocker: MockerFixture):
|
||||
"""Async variant of the fatal error message test."""
|
||||
fatal_msg = {"type": "error", "error": "Orchestrator thread crashed", "fatal": True}
|
||||
|
||||
mock_queue = mocker.MagicMock()
|
||||
mock_queue.sync_q.get_nowait.return_value = fatal_msg
|
||||
|
||||
engine = _make_engine(mock_queue, mocker, thread_alive=False)
|
||||
|
||||
msg = await engine.try_get_output_async()
|
||||
assert msg is not None
|
||||
assert msg["type"] == "error"
|
||||
assert msg["fatal"] is True
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for error propagation paths within the Orchestrator.
|
||||
|
||||
Covers:
|
||||
- EngineDeadError from an LLM stage poll → fatal error broadcast + shutdown
|
||||
- Diffusion stage error output (OmniRequestOutput.from_error) → routed correctly
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
from .test_orchestrator import (
|
||||
FakeStageClient,
|
||||
OrchestratorFixture,
|
||||
_build_harness,
|
||||
_enqueue_add_request,
|
||||
_wait_for,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _sampling_params(max_tokens: int = 4):
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
return SamplingParams(max_tokens=max_tokens)
|
||||
|
||||
|
||||
async def _get_any_output_message(fixture: OrchestratorFixture, *, timeout: float = 2.0) -> dict:
|
||||
"""Like _get_output_message but returns any message type (including errors)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
raise AssertionError("Timed out waiting for orchestrator output")
|
||||
try:
|
||||
return fixture.output_sync_q.get_nowait()
|
||||
except queue.Empty:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def orchestrator_factory():
|
||||
fixtures: list[OrchestratorFixture] = []
|
||||
|
||||
def _factory(*args, **kwargs) -> OrchestratorFixture:
|
||||
fixture = _build_harness(*args, **kwargs)
|
||||
fixtures.append(fixture)
|
||||
return fixture
|
||||
|
||||
yield _factory
|
||||
|
||||
for fixture in fixtures:
|
||||
if fixture.thread.is_alive():
|
||||
fixture.request_sync_q.put_nowait({"type": "shutdown"})
|
||||
fixture.thread.join(timeout=5)
|
||||
for q in fixture.queues:
|
||||
q.close()
|
||||
|
||||
|
||||
# ───────── EngineDeadError from LLM stage poll ─────────
|
||||
|
||||
|
||||
class FakeDeadLLMStageClient(FakeStageClient):
|
||||
"""LLM stage client that raises EngineDeadError on get_output_async."""
|
||||
|
||||
async def get_output_async(self):
|
||||
raise EngineDeadError("Stage-0 engine core is dead")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_factory) -> None:
|
||||
"""When a stage raises EngineDeadError during poll, the orchestrator must:
|
||||
1. Enqueue a fatal error message for each affected request
|
||||
2. Shut itself down (thread exits)
|
||||
"""
|
||||
stage0 = FakeDeadLLMStageClient(stage_type="llm", final_output=True)
|
||||
orchestrator_fixture = orchestrator_factory([stage0])
|
||||
request = SimpleNamespace(request_id="req-dead", prompt_token_ids=[1, 2])
|
||||
|
||||
try:
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-dead",
|
||||
prompt=request,
|
||||
original_prompt={"prompt": "hello"},
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
|
||||
# Collect the fatal error message.
|
||||
msg = await _get_any_output_message(orchestrator_fixture)
|
||||
|
||||
assert msg["type"] == "error"
|
||||
assert msg["fatal"] is True
|
||||
assert msg["request_id"] == "req-dead"
|
||||
assert "Stage-0 engine core is dead" in msg["error"]
|
||||
|
||||
# The orchestrator thread should exit after the fatal error.
|
||||
orchestrator_fixture.thread.join(timeout=5)
|
||||
assert not orchestrator_fixture.thread.is_alive()
|
||||
|
||||
# Request state should be cleaned up.
|
||||
assert "req-dead" not in orchestrator_fixture.orchestrator.request_states
|
||||
finally:
|
||||
if orchestrator_fixture.thread.is_alive():
|
||||
orchestrator_fixture.request_sync_q.put_nowait({"type": "shutdown"})
|
||||
orchestrator_fixture.thread.join(timeout=5)
|
||||
|
||||
|
||||
# ───────── Diffusion stage error output routing ─────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diffusion_error_output_routed_as_finished(orchestrator_factory) -> None:
|
||||
"""When a diffusion stage returns an OmniRequestOutput with a non-None
|
||||
error, the orchestrator must route it as a finished output message and
|
||||
clean up the request state.
|
||||
"""
|
||||
stage0 = FakeStageClient(stage_type="diffusion", final_output=True, final_output_type="image")
|
||||
orchestrator_fixture = orchestrator_factory([stage0])
|
||||
params = OmniDiffusionSamplingParams()
|
||||
|
||||
try:
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-err",
|
||||
prompt={"prompt": "draw a cat"},
|
||||
original_prompt={"prompt": "draw a cat"},
|
||||
sampling_params_list=[params],
|
||||
final_stage_id=0,
|
||||
)
|
||||
|
||||
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
|
||||
|
||||
# Push an error output from the diffusion stage.
|
||||
stage0.push_diffusion_output(OmniRequestOutput.from_error("req-err", "gpu fault"))
|
||||
|
||||
msg = await _get_any_output_message(orchestrator_fixture)
|
||||
|
||||
assert msg["type"] == "output"
|
||||
assert msg["request_id"] == "req-err"
|
||||
assert msg["finished"] is True
|
||||
assert msg["engine_outputs"].error == "gpu fault"
|
||||
|
||||
# Request state should be cleaned up.
|
||||
await _wait_for(lambda: "req-err" not in orchestrator_fixture.orchestrator.request_states)
|
||||
finally:
|
||||
orchestrator_fixture.request_sync_q.put_nowait({"type": "shutdown"})
|
||||
orchestrator_fixture.thread.join(timeout=5)
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for StageEngineCoreClient.check_health().
|
||||
|
||||
Uses object.__new__ to construct a minimal client — check_health only
|
||||
touches self.resources, self.stage_id, and self._proc.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_client(*, engine_dead=False, proc_alive=True):
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
client.resources = SimpleNamespace(engine_dead=engine_dead)
|
||||
client._proc = MagicMock(is_alive=MagicMock(return_value=proc_alive), exitcode=1)
|
||||
return client
|
||||
|
||||
|
||||
def test_check_health_passes_when_alive():
|
||||
client = _make_client(engine_dead=False, proc_alive=True)
|
||||
client.check_health() # no exception
|
||||
|
||||
|
||||
def test_check_health_raises_when_resources_engine_dead():
|
||||
client = _make_client(engine_dead=True, proc_alive=True)
|
||||
with pytest.raises(EngineDeadError, match="engine core is dead"):
|
||||
client.check_health()
|
||||
|
||||
|
||||
def test_check_health_raises_when_proc_not_alive():
|
||||
client = _make_client(engine_dead=False, proc_alive=False)
|
||||
with pytest.raises(EngineDeadError, match="not alive"):
|
||||
client.check_health()
|
||||
# Verify it set resources.engine_dead as a side effect
|
||||
assert client.resources.engine_dead is True
|
||||
@@ -407,6 +407,29 @@ def test_health_endpoint_no_engine():
|
||||
assert data["status"] == "unhealthy"
|
||||
|
||||
|
||||
def test_health_endpoint_dead_engine():
|
||||
"""Health returns 503 when the engine raises EngineDeadError."""
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from fastapi import FastAPI
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from vllm_omni.entrypoints.openai.api_server import router
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
|
||||
dead_engine = AsyncMock()
|
||||
dead_engine.check_health = AsyncMock(side_effect=EngineDeadError())
|
||||
app.state.engine_client = dead_engine
|
||||
|
||||
client = TestClient(app)
|
||||
response = client.get("/health")
|
||||
assert response.status_code == 503
|
||||
data = response.json()
|
||||
assert data["status"] == "unhealthy"
|
||||
|
||||
|
||||
def test_models_endpoint(test_client):
|
||||
"""Test /v1/models endpoint for diffusion mode"""
|
||||
response = test_client.get("/v1/models")
|
||||
|
||||
@@ -133,7 +133,6 @@ def _wait_for_status(client: TestClient, video_id: str, status: str, timeout_s:
|
||||
last_payload = None
|
||||
while time.time() < deadline:
|
||||
response = client.get(f"/v1/videos/{video_id}")
|
||||
assert response.status_code == 200
|
||||
last_payload = response.json()
|
||||
if last_payload["status"] == status:
|
||||
return last_payload
|
||||
@@ -644,7 +643,7 @@ def test_invalid_lora_returns_400(test_client):
|
||||
assert response.status_code == 200
|
||||
video_id = response.json()["id"]
|
||||
failed = _wait_for_status(test_client, video_id, VideoGenerationStatus.FAILED.value)
|
||||
assert failed["error"]["code"] == "HTTPException"
|
||||
assert failed["error"]["code"] == 400
|
||||
assert "lora object" in failed["error"]["message"].lower()
|
||||
|
||||
|
||||
|
||||
@@ -4,11 +4,13 @@ import queue
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -163,6 +165,15 @@ def _patch_engine(monkeypatch: pytest.MonkeyPatch, engine: FakeAsyncOmniEngine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
|
||||
|
||||
|
||||
def _make_base():
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
obj = object.__new__(OmniBase)
|
||||
obj.engine = MagicMock()
|
||||
obj.request_states = {}
|
||||
return obj
|
||||
|
||||
|
||||
def _stage_spec(
|
||||
stage_id: int,
|
||||
*,
|
||||
@@ -687,3 +698,255 @@ def test_omni_forces_final_only_on_llm_stages(monkeypatch: pytest.MonkeyPatch):
|
||||
assert submitted_params[1].output_kind == RequestOutputKind.FINAL_ONLY
|
||||
assert submitted_params[2].output_kind == original_diffusion_output_kind
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
def test_fatal_error_raises_engine_dead():
|
||||
base = _make_base()
|
||||
msg = {"type": "error", "error": "orchestrator crashed", "fatal": True}
|
||||
|
||||
with pytest.raises(EngineDeadError, match="orchestrator crashed"):
|
||||
base._handle_output_message(msg)
|
||||
|
||||
|
||||
def test_non_fatal_error_raises_runtime():
|
||||
base = _make_base()
|
||||
msg = {"type": "error", "error": "something wrong"}
|
||||
|
||||
with pytest.raises(RuntimeError, match="something wrong"):
|
||||
base._handle_output_message(msg)
|
||||
|
||||
|
||||
def test_async_omni_errored_property_alive():
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
is_alive=lambda: True,
|
||||
stage_clients=[SimpleNamespace(is_comprehension=False)],
|
||||
)
|
||||
|
||||
assert omni.errored is False
|
||||
|
||||
|
||||
def test_async_omni_errored_property_dead_engine():
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
is_alive=lambda: False,
|
||||
stage_clients=[SimpleNamespace(is_comprehension=False)],
|
||||
)
|
||||
|
||||
assert omni.errored is True
|
||||
|
||||
|
||||
def test_async_omni_errored_property_dead_stage():
|
||||
omni = object.__new__(AsyncOmni)
|
||||
dead_stage = SimpleNamespace(is_comprehension=False, _engine_dead=True)
|
||||
omni.engine = SimpleNamespace(
|
||||
is_alive=lambda: True,
|
||||
stage_clients=[dead_stage],
|
||||
)
|
||||
|
||||
assert omni.errored is True
|
||||
|
||||
|
||||
def _enqueue_stage_error(
|
||||
engine: FakeAsyncOmniEngine,
|
||||
msg,
|
||||
*,
|
||||
error_text: str,
|
||||
kill_engine: bool = False,
|
||||
):
|
||||
"""Enqueue a stage error output, optionally killing the engine."""
|
||||
if kill_engine:
|
||||
engine._alive = False
|
||||
engine.output_q.put_nowait(
|
||||
{
|
||||
"type": "output",
|
||||
"request_id": msg["request_id"],
|
||||
"stage_id": 0,
|
||||
"engine_outputs": SimpleNamespace(
|
||||
payload="",
|
||||
finished=True,
|
||||
images=[],
|
||||
stage_durations={},
|
||||
error=error_text,
|
||||
),
|
||||
"finished": False,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_propagates_engine_dead_error(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When the engine is dead and an error output arrives, ``generate()``
|
||||
must raise ``EngineDeadError`` (not plain ``RuntimeError``)."""
|
||||
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="worker OOM", kill_engine=True),
|
||||
)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = AsyncOmni("dummy-model")
|
||||
try:
|
||||
with pytest.raises(EngineDeadError, match="worker OOM"):
|
||||
async for _ in app.generate(prompt="hello", request_id="req-dead"):
|
||||
pass
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_propagates_engine_generate_error(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When the engine is alive but a stage error occurs, ``generate()``
|
||||
must raise ``EngineGenerateError`` (recoverable, not ``EngineDeadError``)."""
|
||||
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="diffusion step failed"),
|
||||
)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = AsyncOmni("dummy-model")
|
||||
try:
|
||||
with pytest.raises(EngineGenerateError):
|
||||
async for _ in app.generate(prompt="hello", request_id="req-recover"):
|
||||
pass
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
# ───────── OmniBase.check_health() aggregation ─────────
|
||||
|
||||
|
||||
def test_check_health_passes_when_all_healthy():
|
||||
base = _make_base()
|
||||
healthy_stage = MagicMock()
|
||||
healthy_stage.check_health = MagicMock()
|
||||
base.engine.is_alive.return_value = True
|
||||
base.engine.stage_clients = [healthy_stage]
|
||||
base.check_health() # should not raise
|
||||
|
||||
|
||||
def test_check_health_raises_when_stage_dead():
|
||||
base = _make_base()
|
||||
dead_stage = MagicMock()
|
||||
dead_stage.check_health = MagicMock(side_effect=EngineDeadError("Stage-1 dead"))
|
||||
base.engine.is_alive.return_value = True
|
||||
base.engine.stage_clients = [dead_stage]
|
||||
with pytest.raises(EngineDeadError, match="Stage-1 dead"):
|
||||
base.check_health()
|
||||
|
||||
|
||||
def test_check_health_raises_when_orchestrator_dead():
|
||||
base = _make_base()
|
||||
base.engine.is_alive.return_value = False
|
||||
base.engine.stage_clients = []
|
||||
with pytest.raises(EngineDeadError, match="not alive"):
|
||||
base.check_health()
|
||||
|
||||
|
||||
# ───────── OmniBase.errored property ─────────
|
||||
|
||||
|
||||
def test_omni_base_errored_false_when_alive():
|
||||
base = _make_base()
|
||||
base.engine.is_alive.return_value = True
|
||||
base.engine.stage_clients = [SimpleNamespace()]
|
||||
assert base.errored is False
|
||||
|
||||
|
||||
def test_omni_base_errored_true_when_orchestrator_dead():
|
||||
base = _make_base()
|
||||
base.engine.is_alive.return_value = False
|
||||
base.engine.stage_clients = []
|
||||
assert base.errored is True
|
||||
|
||||
|
||||
def test_omni_base_errored_true_when_stage_engine_dead():
|
||||
base = _make_base()
|
||||
base.engine.is_alive.return_value = True
|
||||
dead_stage = SimpleNamespace(_engine_dead=True)
|
||||
base.engine.stage_clients = [dead_stage]
|
||||
assert base.errored is True
|
||||
|
||||
|
||||
def test_omni_base_errored_true_when_stage_resources_engine_dead():
|
||||
base = _make_base()
|
||||
base.engine.is_alive.return_value = True
|
||||
dead_stage = SimpleNamespace(resources=SimpleNamespace(engine_dead=True))
|
||||
base.engine.stage_clients = [dead_stage]
|
||||
assert base.errored is True
|
||||
|
||||
|
||||
# ───────── Omni (sync) EngineDeadError / EngineGenerateError ─────────
|
||||
|
||||
|
||||
def test_omni_propagates_engine_dead_error(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When the engine is dead and a stage error output arrives,
|
||||
``Omni.generate()`` must raise ``EngineDeadError``."""
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="worker OOM", kill_engine=True),
|
||||
)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
try:
|
||||
with pytest.raises(EngineDeadError, match="worker OOM"):
|
||||
list(app.generate(["hello"], py_generator=False, use_tqdm=False))
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
def test_omni_propagates_engine_generate_error(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When the engine is alive but a stage error occurs,
|
||||
``Omni.generate()`` must raise ``EngineGenerateError`` (recoverable)."""
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=lambda eng, msg: _enqueue_stage_error(eng, msg, error_text="diffusion step failed"),
|
||||
)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
try:
|
||||
with pytest.raises(EngineGenerateError):
|
||||
list(app.generate(["hello"], py_generator=False, use_tqdm=False))
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
def test_omni_errored_property_alive(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Omni.errored (inherited from OmniBase) returns False when healthy."""
|
||||
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
try:
|
||||
assert app.errored is False
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
def test_omni_errored_property_dead_engine(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Omni.errored returns True when the orchestrator is dead."""
|
||||
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
try:
|
||||
engine._alive = False
|
||||
assert app.errored is True
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
def test_omni_errored_property_dead_stage(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Omni.errored returns True when a stage client is marked dead."""
|
||||
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
try:
|
||||
engine.stage_clients[0]._engine_dead = True
|
||||
assert app.errored is True
|
||||
finally:
|
||||
app.shutdown()
|
||||
Reference in new issue
Block a user