[Bugfix] Per-replica fault isolation: keep API server alive on single-stage death (#4285) (#4583)

Signed-off-by: Shenglei Fu <sfu@confluent.io>
Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-08-10 23:00:18 -04:00
1 parent c27623c29c
commit ee118ab492
9 files changed
+1157 -290

No files matched your search

+641 -30
View File
@@ -3,7 +3,7 @@
"""Tests for error propagation paths within the Orchestrator.
Covers:
- EngineDeadError from an LLM stage poll → fatal error broadcast + shutdown
- EngineDeadError from an LLM stage poll → fatal error broadcast + replica eviction
- Diffusion stage error output (OmniRequestOutput.from_error) → routed correctly
"""
@@ -14,17 +14,30 @@ import queue
import time
from types import SimpleNamespace
import janus
import pytest
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.engine.messages import EngineQueueMessage, ErrorMessage, ShutdownRequestMessage
from vllm_omni.engine.messages import (
AddCompanionRequestMessage,
EngineQueueMessage,
ErrorMessage,
ShutdownRequestMessage,
StageSubmissionMessage,
)
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.engine.stage_pool import StageUnavailableError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from .test_orchestrator import (
FakeOutputProcessor,
FakeStageClient,
OrchestratorFixture,
_build_harness,
_build_request_output,
_build_stage_pools,
_engine_core_outputs,
_enqueue_add_request,
_wait_for,
)
@@ -50,6 +63,22 @@ async def _get_any_output_message(fixture: OrchestratorFixture, *, timeout: floa
await asyncio.sleep(0.01)
async def _wait_for_error_message(
fixture: OrchestratorFixture,
*,
request_id: str | None = None,
max_messages: int = 50,
) -> ErrorMessage:
"""Drain output messages until the ErrorMessage arrives, skipping the
non-error outputs (e.g. stage metrics, forwarded stage results) that a
driven pipeline can interleave ahead of the failure."""
for _ in range(max_messages):
msg = await _get_any_output_message(fixture)
if isinstance(msg, ErrorMessage) and (request_id is None or msg.request_id == request_id):
return msg
raise AssertionError(f"No ErrorMessage for req={request_id} within {max_messages} messages")
@pytest.fixture
def orchestrator_factory():
fixtures: list[OrchestratorFixture] = []
@@ -72,32 +101,14 @@ def orchestrator_factory():
# ───────── EngineDeadError from LLM stage poll ─────────
class FakeDeadLLMStageClient(FakeStageClient):
"""LLM stage client that raises EngineDeadError once a request is in-flight.
Raising on the first poll races with ``add_request``: under suite load the
orchestrator can die and drain before the test enqueues anything, so no
fatal ErrorMessage is ever published. Keep returning empty outputs until
``add_request_async`` has been called, then fail on the next poll.
"""
async def get_output_async(self):
if not self.add_request_calls:
return SimpleNamespace(
outputs=[],
scheduler_stats=None,
finished_requests=None,
)
raise EngineDeadError("Stage-0 engine core is dead")
@pytest.mark.asyncio
async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_factory) -> None:
"""When a stage raises EngineDeadError during poll, the orchestrator must:
async def test_engine_dead_error_evicts_replica_and_keeps_running(orchestrator_factory) -> None:
"""When a stage replica raises EngineDeadError during poll, the orchestrator must:
1. Enqueue a fatal error message for each affected request
2. Shut itself down (thread exits)
2. Evict the dead replica and keep running so the API server stays up
(per-replica fault isolation, #4285)
"""
stage0 = FakeDeadLLMStageClient(stage_type="llm", final_output=True)
stage0 = _DieOnDemandLLMStageClient(stage_type="llm", final_output=True)
orchestrator_fixture = orchestrator_factory([stage0])
request = SimpleNamespace(request_id="req-dead", prompt_token_ids=[1, 2])
@@ -110,6 +121,12 @@ async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_fa
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
# Let the request dispatch before the replica dies; a replica dead on
# its very first poll can be evicted before dispatch, which routes the
# failure through the add-request guard instead (that ordering is
# covered by test_add_request_to_dead_stage_fails_request_not_server).
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
stage0.die = True
# Collect the fatal error message.
msg = await _get_any_output_message(orchestrator_fixture)
@@ -118,13 +135,17 @@ async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_fa
assert msg.type == "error"
assert msg.fatal is True
assert msg.request_id == "req-dead"
assert "Stage-0 engine core is dead" in msg.error
assert "Stage-0 replica-0 is dead" in msg.error
# The orchestrator thread should exit after the fatal error.
orchestrator_fixture.thread.join(timeout=5)
assert not orchestrator_fixture.thread.is_alive()
# The dead replica is evicted; the orchestrator keeps running.
pool = orchestrator_fixture.orchestrator.stage_pools[0]
deadline = time.monotonic() + 5.0
while pool.live_replica_ids() and time.monotonic() < deadline:
await asyncio.sleep(0.02)
assert pool.live_replica_ids() == []
assert orchestrator_fixture.thread.is_alive()
# Request state should be cleaned up.
# Affected request state should be cleaned up.
assert "req-dead" not in orchestrator_fixture.orchestrator.request_states
finally:
if orchestrator_fixture.thread.is_alive():
@@ -132,6 +153,596 @@ async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_fa
orchestrator_fixture.thread.join(timeout=5)
class _DieOnDemandLLMStageClient(FakeStageClient):
"""LLM stage replica that raises EngineDeadError on poll once ``die`` is set."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.die = False
async def get_output_async(self):
if self.die:
raise EngineDeadError(f"Stage-{self.stage_id} replica-{self.replica_id} is dead")
return await super().get_output_async()
class _DieOnDemandDiffusionStageClient(FakeStageClient):
"""Diffusion stage replica that raises EngineDeadError on poll once ``die`` is set."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.die = False
def get_diffusion_output_nowait(self):
if self.die:
raise EngineDeadError(f"Stage-{self.stage_id} diffusion replica-{self.replica_id} is dead")
return super().get_diffusion_output_nowait()
class _UnavailableDiffusionStageClient(FakeStageClient):
"""Diffusion replica that stays live (so the pool keeps stage_type) but whose
dispatch fails with StageUnavailableError, as a downstream prewarm would when
its stage has no capacity."""
async def add_request_async(self, *args, **kwargs) -> None:
raise StageUnavailableError(f"stage {self.stage_id} has no live replica")
@pytest.mark.asyncio
async def test_engine_dead_error_fails_only_dead_replica_requests(orchestrator_factory) -> None:
"""Multi-replica stage: one replica dying must fail only the requests bound
to it. Requests bound to a surviving replica keep running, and the stage
(still having a live replica) is not torn down (per-replica fault isolation,
#4285).
"""
r0 = _DieOnDemandLLMStageClient(stage_type="llm", final_output=True)
r1 = FakeStageClient(stage_type="llm", final_output=True)
stage_pools = _build_stage_pools([[r0, r1]])
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
pool = orchestrator_fixture.orchestrator.stage_pools[0]
try:
# req-0 → replica 0 (round-robin starts at 0)
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-0",
prompt=SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]),
original_prompt={"prompt": "a"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await _wait_for(lambda: len(r0.add_request_calls) == 1)
# req-1 → replica 1
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-1",
prompt=SimpleNamespace(request_id="req-1", prompt_token_ids=[3, 4]),
original_prompt={"prompt": "b"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await _wait_for(lambda: len(r1.add_request_calls) == 1)
assert pool.get_bound_replica_id("req-0") == 0
assert pool.get_bound_replica_id("req-1") == 1
# Kill replica 0 on its next poll.
r0.die = True
msg = await _get_any_output_message(orchestrator_fixture)
assert isinstance(msg, ErrorMessage)
assert msg.fatal is True
assert msg.request_id == "req-0"
# Replica 0 evicted, replica 1 still serving; req-1 untouched.
await _wait_for(lambda: pool.live_replica_ids() == [1])
assert orchestrator_fixture.thread.is_alive()
assert "req-0" not in orchestrator_fixture.orchestrator.request_states
assert "req-1" in orchestrator_fixture.orchestrator.request_states
finally:
if orchestrator_fixture.thread.is_alive():
orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
orchestrator_fixture.thread.join(timeout=5)
@pytest.mark.asyncio
async def test_forward_to_dead_downstream_stage_fails_request_not_server(orchestrator_factory) -> None:
"""A request mid-pipeline whose downstream stage has lost all replicas must
fail that request, not tear down the orchestrator. Forwarding to a stage with
no live replica is handled gracefully (per-replica fault isolation, #4285).
"""
stage0 = FakeStageClient(stage_type="llm", final_output=False, next_inputs=[{"prompt_token_ids": [7, 8]}])
stage1 = _DieOnDemandLLMStageClient(stage_type="llm", final_output=True)
proc0 = FakeOutputProcessor(request_outputs=[_build_request_output("req-x", token_ids=[3], finished=True)])
proc1 = FakeOutputProcessor()
stage_pools = _build_stage_pools([[stage0], [stage1]], output_processors=[proc0, proc1])
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
pool1 = orchestrator_fixture.orchestrator.stage_pools[1]
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-x",
prompt=SimpleNamespace(request_id="req-x", prompt_token_ids=[1, 2]),
original_prompt={"prompt": "hello"},
sampling_params_list=[_sampling_params(), _sampling_params()],
final_stage_id=1,
)
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
# Kill stage-1's only replica before stage-0 output is forwarded.
stage1.die = True
await _wait_for(lambda: pool1.live_replica_ids() == [])
# Completing stage 0 triggers a forward to the now-empty stage 1.
stage0.push_engine_core_outputs(_engine_core_outputs("s0-raw", 1.0))
# Completing stage 0 also emits its normal outputs; skip past them to
# the downstream-forward failure.
error_msg = await _wait_for_error_message(orchestrator_fixture, request_id="req-x")
assert error_msg.fatal is True
assert error_msg.stage_id == 1
assert orchestrator_fixture.thread.is_alive()
assert "req-x" not in orchestrator_fixture.orchestrator.request_states
finally:
if orchestrator_fixture.thread.is_alive():
orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
orchestrator_fixture.thread.join(timeout=5)
@pytest.mark.asyncio
async def test_add_request_to_dead_stage_fails_request_not_server(orchestrator_factory) -> None:
"""A new request entering a stage that already lost all replicas fails that
request instead of raising out of the request handler and tearing the server
down. The guard runs before request state is registered, so nothing leaks.
"""
r0 = _DieOnDemandLLMStageClient(stage_type="llm", final_output=True)
stage_pools = _build_stage_pools([[r0]])
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
pool = orchestrator_fixture.orchestrator.stage_pools[0]
try:
# Send a first request so the replica is polled and dies → evicted.
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-0",
prompt=SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]),
original_prompt={"prompt": "a"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
r0.die = True
await _wait_for(lambda: pool.live_replica_ids() == [])
# A new request now enters a stage with no live replica.
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-after",
prompt=SimpleNamespace(request_id="req-after", prompt_token_ids=[3, 4]),
original_prompt={"prompt": "b"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
error_msg = await _wait_for_error_message(orchestrator_fixture, request_id="req-after")
assert error_msg.fatal is True
assert error_msg.stage_id == 0
assert orchestrator_fixture.thread.is_alive()
# Guard runs before registration, so the failed request never enters state.
assert "req-after" not in orchestrator_fixture.orchestrator.request_states
finally:
if orchestrator_fixture.thread.is_alive():
orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
orchestrator_fixture.thread.join(timeout=5)
@pytest.mark.asyncio
async def test_diffusion_replica_death_on_poll_keeps_server(orchestrator_factory) -> None:
"""A diffusion stage replica raising EngineDeadError during poll must be
evicted while the orchestrator keeps serving. The per-replica catch has to
cover the diffusion polling path, not just the LLM one (#4285).
"""
stage0 = _DieOnDemandDiffusionStageClient(stage_type="diffusion", final_output=True, final_output_type="image")
stage_pools = _build_stage_pools([[stage0]])
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
pool = orchestrator_fixture.orchestrator.stage_pools[0]
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-dit",
prompt={"prompt": "a cat"},
original_prompt={"prompt": "a cat"},
sampling_params_list=[OmniDiffusionSamplingParams()],
final_stage_id=0,
)
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
stage0.die = True
error_msg = await _wait_for_error_message(orchestrator_fixture, request_id="req-dit")
assert error_msg.fatal is True
assert error_msg.stage_id == 0
await _wait_for(lambda: pool.live_replica_ids() == [])
assert orchestrator_fixture.thread.is_alive()
assert "req-dit" not in orchestrator_fixture.orchestrator.request_states
finally:
if orchestrator_fixture.thread.is_alive():
orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
orchestrator_fixture.thread.join(timeout=5)
@pytest.mark.asyncio
async def test_async_chunk_prewarm_failure_reports_failing_downstream_stage(orchestrator_factory) -> None:
"""When an async-chunk prewarm dispatch to a downstream stage fails, the
failure must be attributed to that stage, not stage 0 (the prewarm's own
dispatch stage). Regression for the guard using stage_id=0.
"""
stage0 = FakeStageClient(stage_type="llm", final_output=False)
stage1 = FakeStageClient(stage_type="diffusion", final_output=False, final_output_type="image")
stage2 = _UnavailableDiffusionStageClient(stage_type="diffusion", final_output=True, final_output_type="image")
stage_pools = _build_stage_pools([[stage0], [stage1], [stage2]])
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools, async_chunk=True)
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-x",
prompt=SimpleNamespace(request_id="req-x", prompt_token_ids=[1, 2, 3]),
original_prompt={"prompt": "a cat"},
sampling_params_list=[_sampling_params(), OmniDiffusionSamplingParams(), OmniDiffusionSamplingParams()],
final_stage_id=2,
)
error_msg = await _wait_for_error_message(orchestrator_fixture, request_id="req-x")
assert error_msg.fatal is True
assert error_msg.stage_id == 2
assert "Stage-2" in error_msg.error
# Stage 1 prewarm ran before the stage-2 failure; server stays up.
assert len(stage1.add_request_calls) == 1
assert orchestrator_fixture.thread.is_alive()
finally:
if orchestrator_fixture.thread.is_alive():
orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
orchestrator_fixture.thread.join(timeout=5)
# ───────── Direct unit tests for the fault-isolation helpers ─────────
def _build_bare_orchestrator(stage_pools) -> tuple[Orchestrator, tuple[janus.Queue, ...]]:
"""Construct an Orchestrator on the current event loop without running it.
Lets tests await the fault-isolation helpers directly instead of driving
them through the orchestration loop.
"""
queues = (janus.Queue(), janus.Queue(), janus.Queue())
orchestrator = Orchestrator(
request_async_queue=queues[0].async_q,
output_async_queue=queues[1].async_q,
rpc_async_queue=queues[2].async_q,
stage_pools=stage_pools,
)
return orchestrator, queues
def _register_request(orchestrator: Orchestrator, request_id: str, *, submitted_stage: int | None = 0) -> None:
state = OrchestratorRequestState(request_id=request_id)
if submitted_stage is not None:
state.stage_submit_ts[submitted_stage] = time.time()
orchestrator.request_states[request_id] = state
@pytest.mark.asyncio
async def test_handle_dead_replica_fails_only_bound_requests() -> None:
"""Direct helper test: with a surviving replica, only requests bound to the
dead replica fail; the survivor's requests and bindings are untouched.
"""
r0 = FakeStageClient(stage_type="llm", final_output=True)
r1 = FakeStageClient(stage_type="llm", final_output=True)
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[r0, r1]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-0")
_register_request(orchestrator, "req-1")
assert pool.select_replica_id("req-0") == 0
assert pool.select_replica_id("req-1") == 1
await orchestrator._handle_dead_replica(0, 0, EngineDeadError("replica-0 dead"))
assert pool.live_replica_ids() == [1]
assert r0.shutdown_calls == 1
msg = queues[1].async_q.get_nowait()
assert isinstance(msg, ErrorMessage)
assert msg.fatal is True
assert msg.request_id == "req-0"
assert msg.stage_id == 0
assert "replica-0 dead" in msg.error
assert queues[1].async_q.empty()
assert "req-0" not in orchestrator.request_states
assert "req-1" in orchestrator.request_states
assert pool.get_bound_replica_id("req-0") is None
assert pool.get_bound_replica_id("req-1") == 1
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_handle_dead_replica_last_replica_fails_all_stage_requests() -> None:
"""Direct helper test: when the dead replica was the stage's last one, every
request routed through the stage fails — bound or not — while requests that
never entered the stage survive.
"""
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[FakeStageClient(stage_type="llm")]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-bound")
_register_request(orchestrator, "req-unbound")
_register_request(orchestrator, "req-elsewhere", submitted_stage=None)
assert pool.select_replica_id("req-bound") == 0
await orchestrator._handle_dead_replica(0, 0, EngineDeadError("last replica dead"))
assert pool.live_replica_ids() == []
failed = {queues[1].async_q.get_nowait().request_id for _ in range(2)}
assert failed == {"req-bound", "req-unbound"}
assert queues[1].async_q.empty()
assert set(orchestrator.request_states) == {"req-elsewhere"}
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_fail_request_dead_stage_cleans_registered_request() -> None:
"""Direct helper test: a registered request failed against a dead stage gets
a fatal ErrorMessage, its state and bindings are released, and its
in-flight work on live replicas is aborted.
"""
r0 = FakeStageClient(stage_type="llm")
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[r0]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-a")
assert pool.select_replica_id("req-a") == 0
await orchestrator._fail_request_dead_stage("req-a", 0)
msg = queues[1].async_q.get_nowait()
assert isinstance(msg, ErrorMessage)
assert msg.error == "Stage-0 has no live replica"
assert msg.fatal is True
assert msg.request_id == "req-a"
assert msg.stage_id == 0
assert "req-a" not in orchestrator.request_states
assert pool.get_bound_replica_id("req-a") is None
# abort=True: the failed request's work on the still-live replica is
# stopped, not left to run to completion.
assert r0.abort_calls == [["req-a"]]
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_streaming_update_racing_eviction_fails_request_not_server() -> None:
"""A streaming_update can find its request state still present while the
stage has already lost all replicas (_handle_dead_replica awaits between
eviction and cleanup). Submitting then raises from replica selection; the
handler must fail that request instead of letting the raise escape
_request_handler and shut the server down (#4285).
"""
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[FakeStageClient(stage_type="llm")]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-s")
orchestrator.request_states["req-s"].sampling_params_list = [_sampling_params()]
assert pool.select_replica_id("req-s") == 0
pool.evict_replica(0)
msg = StageSubmissionMessage(
type="streaming_update",
request_id="req-s",
prompt=SimpleNamespace(request_id="req-s", prompt_token_ids=[1]),
original_prompt={"prompt": "hi"},
output_prompt_text=None,
sampling_params_list=[],
final_stage_id=0,
preprocess_ms=0.0,
request_timestamp=time.time(),
enqueue_ts=time.time(),
)
await orchestrator._handle_streaming_update(msg)
err = queues[1].async_q.get_nowait()
assert isinstance(err, ErrorMessage)
assert err.fatal is True
assert err.request_id == "req-s"
assert "req-s" not in orchestrator.request_states
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_streaming_update_unrelated_error_is_not_masked() -> None:
"""The dispatch guard converts only StageUnavailableError/EngineDeadError;
a genuine bug surfacing as a plain RuntimeError must propagate (fail
fast), not be misreported as a dead-stage request failure.
"""
class _BuggyClient(FakeStageClient):
async def add_request_async(self, *args, **kwargs):
raise RuntimeError("boom: unrelated bug")
orchestrator, queues = _build_bare_orchestrator(
_build_stage_pools([[_BuggyClient(stage_type="llm", final_output=True)]])
)
try:
_register_request(orchestrator, "req-b")
orchestrator.request_states["req-b"].sampling_params_list = [_sampling_params()]
msg = StageSubmissionMessage(
type="streaming_update",
request_id="req-b",
prompt=SimpleNamespace(request_id="req-b", prompt_token_ids=[1]),
original_prompt={"prompt": "hi"},
output_prompt_text=None,
sampling_params_list=[],
final_stage_id=0,
preprocess_ms=0.0,
request_timestamp=time.time(),
enqueue_ts=time.time(),
)
with pytest.raises(RuntimeError, match="unrelated bug"):
await orchestrator._handle_streaming_update(msg)
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_add_companion_racing_eviction_fails_parent_not_server() -> None:
"""A CFG companion submission hitting a fully dead stage must fail the
parent request (whose cleanup also releases the companion), not raise out
of _request_handler (#4285).
"""
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[FakeStageClient(stage_type="llm")]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-p")
orchestrator.request_states["req-p"].sampling_params_list = [_sampling_params()]
pool.evict_replica(0)
msg = AddCompanionRequestMessage(
companion_id="req-p-cfg",
parent_id="req-p",
role="negative",
prompt=SimpleNamespace(request_id="req-p-cfg", prompt_token_ids=[2]),
companion_prompt_text=None,
sampling_params_list=[_sampling_params()],
)
await orchestrator._handle_add_companion(msg)
err = queues[1].async_q.get_nowait()
assert isinstance(err, ErrorMessage)
assert err.fatal is True
assert err.request_id == "req-p"
assert "req-p" not in orchestrator.request_states
assert "req-p-cfg" not in orchestrator.request_states
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_fail_request_dead_stage_is_noop_safe_for_unregistered_request() -> None:
"""Direct helper test: the stage-0 dispatch guard calls the helper before the
request is registered; the cleanup must be a safe no-op and still emit the
fatal ErrorMessage.
"""
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[FakeStageClient(stage_type="llm")]]))
try:
await orchestrator._fail_request_dead_stage("req-unknown", 0)
msg = queues[1].async_q.get_nowait()
assert isinstance(msg, ErrorMessage)
assert msg.error == "Stage-0 has no live replica"
assert msg.fatal is True
assert msg.request_id == "req-unknown"
assert orchestrator.request_states == {}
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_dispatch_death_evicts_so_subsequent_requests_reroute_to_survivor() -> None:
"""A replica dying mid-submit (EngineDeadError from dispatch, before the poll
loop evicts it) is evicted by the guard so subsequent requests reroute to the
survivor instead of round-robining back onto the dead slot until the poll
catches up. The survivor and its request are untouched (#4285).
"""
r0 = FakeStageClient(stage_type="llm")
r1 = FakeStageClient(stage_type="llm")
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[r0, r1]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-dead")
_register_request(orchestrator, "req-live")
assert pool.select_replica_id("req-dead") == 0
assert pool.select_replica_id("req-live") == 1
async def _die() -> None:
raise EngineDeadError("stage-0 replica-0 died on submit")
failed = await orchestrator._dispatch_or_fail_request(
_die, req_id="req-dead", stage_id=0, operation="add_request"
)
assert failed is False
# The dead replica is evicted; only the request on it fails, the survivor
# and its request are untouched.
assert pool.live_replica_ids() == [1]
assert r0.shutdown_calls == 1
assert r1.shutdown_calls == 0
msg = queues[1].async_q.get_nowait()
assert isinstance(msg, ErrorMessage) and msg.fatal is True and msg.request_id == "req-dead"
assert queues[1].async_q.empty()
assert "req-dead" not in orchestrator.request_states
assert "req-live" in orchestrator.request_states and pool.get_bound_replica_id("req-live") == 1
# The point of eviction: a burst of subsequent requests all route to the
# survivor and never land back on the evicted replica.
for rid in ("next-1", "next-2", "next-3"):
assert pool.select_replica_id(rid) == 1
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_dispatch_death_evicts_so_subsequent_requests_fast_fail() -> None:
"""When the replica dying on submit was the stage's only one, the guard evicts
it (leaving the stage empty). A subsequent request then fast-fails at routing —
the ``_handle_add_request`` entry guard turns ``StageUnavailableError`` into a
fatal 5xx immediately, rather than dispatching onto the dead replica (#4285).
"""
r0 = FakeStageClient(stage_type="llm")
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[r0]]))
try:
pool = orchestrator.stage_pools[0]
_register_request(orchestrator, "req-a")
assert pool.select_replica_id("req-a") == 0
async def _die() -> None:
raise EngineDeadError("stage-0 replica-0 died on submit")
failed = await orchestrator._dispatch_or_fail_request(_die, req_id="req-a", stage_id=0, operation="add_request")
assert failed is False
assert pool.live_replica_ids() == []
assert r0.shutdown_calls == 1
msg = queues[1].async_q.get_nowait()
assert isinstance(msg, ErrorMessage) and msg.request_id == "req-a"
assert queues[1].async_q.empty()
assert "req-a" not in orchestrator.request_states
# A subsequent request cannot be routed at all: the stage has no live
# replica, so selection raises fast instead of dispatching onto the dead
# slot. _handle_add_request converts this to a fatal 5xx at its entry guard.
with pytest.raises(StageUnavailableError):
pool.select_replica_id("req-next")
finally:
for q in queues:
q.close()
# ───────── Diffusion stage error output routing ─────────