[Bugfix] Fail a duplex session that outgrows max_model_len instead of killing the EngineCore (#7277)

Signed-off-by: Tianyao Wu <rayroy31@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-16 20:08:25 +08:00
1 parent e6cebf8557
commit fa639b889e
7 files changed
+582 -3

No files matched your search

@@ -149,6 +149,9 @@ def _make_scheduler_stub(requests: list[_Request]) -> SimpleNamespace:
for name in _MIXIN_UPDATE_HELPERS:
setattr(scheduler, name, MethodType(getattr(OmniSchedulerMixin, name), scheduler))
scheduler._cleanup_kv_tracking = MethodType(OmniARScheduler._cleanup_kv_tracking, scheduler)
scheduler._emit_streaming_context_overflow_outputs = MethodType(
OmniARScheduler._emit_streaming_context_overflow_outputs, scheduler
)
scheduler.make_spec_decoding_stats = lambda *args, **kwargs: None
scheduler.make_stats = lambda *args, **kwargs: None
return scheduler
@@ -5,6 +5,7 @@
from __future__ import annotations
from collections import defaultdict
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -17,6 +18,8 @@ import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.engine import FinishReason
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm_omni.core.sched.omni_ar_scheduler import OmniARAsyncScheduler, OmniARScheduler
from vllm_omni.distributed.omni_connectors.transfer_adapter.chunk_transfer_adapter import (
@@ -958,3 +961,303 @@ def test_chunk_segment_cleanup_keeps_explicit_update_stage_parked(
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
sched.skipped_waiting.remove_requests.assert_not_called()
sched._enqueue_waiting_request.assert_not_called()
def _make_live_session_scheduler(*, max_model_len: int) -> OmniARScheduler:
"""A scheduler whose finish path is real: the queues, ``finish_requests``,
``_free_request`` and the finished-request bookkeeping that wakes an idle
engine all run. Only block freeing and the encoder cache are mocked."""
sched = _make_admission_scheduler(max_model_len=max_model_len)
del sched._free_request # the admission helper mocks it; run the real one
sched._omits_kv_transfer_cache = {}
sched.connector = None
sched.perf_metrics = None
sched.recompute_kv_load_failures = False
sched.finished_req_ids = set()
sched.finished_req_ids_dict = defaultdict(set)
sched.transfer_triggered_requests = set()
sched.active_kv_transfers = set()
sched.waiting_for_transfer_free = set()
sched.pending_stop_after_extraction = set()
sched.requests_needing_kv_transfer = {}
sched._kv_wait_start_ts = {}
sched.kv_cache_manager = SimpleNamespace(take_events=lambda: None, estimate_cached_tokens=lambda _request: 0)
return sched
def _run_idle_step(sched: OmniARScheduler):
"""One ``update_from_output`` with nothing scheduled: the step an idle
engine runs once ``has_finished_requests()`` reports a freed session."""
scheduler_output = SimpleNamespace(
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
num_invalid_spec_tokens=0,
)
model_runner_output = SimpleNamespace(
sampled_token_ids=[],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=None,
num_nans_in_logits=None,
kv_connector_output=None,
cudagraph_stats=None,
req_id_to_index={},
routed_experts=None,
)
return OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)
def _park_session(sched: OmniARScheduler, session: Request) -> None:
"""Put a session where a 1 fps video stream leaves it between appends:
prompt partly computed, waiting for streaming input, in admission."""
sched.requests[session.request_id] = session
session.append_output_token_ids([7, 8, 9])
session.num_computed_tokens = 6
session.num_output_placeholders = 0
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
sched.skipped_waiting.add_request(session)
sched.num_waiting_for_streaming_input = 1
def test_stage0_streaming_update_that_overflows_max_model_len_finishes_the_session() -> None:
"""A native duplex session grows its prompt on every append; once the
extension cannot fit the model the worker crashes copying the prompt.
The scheduler must drop the update and fail only this session, at once,
through the real finish path: a parked session does not make the engine
schedule, so what wakes the engine is the freed request showing up in
``has_finished_requests()``, and the step that follows must carry one
explicit ERROR rather than the synthesized ABORT."""
sched = _make_live_session_scheduler(max_model_len=8)
session = _make_request()
_park_session(sched, session)
sched._update_request_as_session(session, _make_update([10, 20, 30]))
assert session.prompt_token_ids == [1, 2, 3]
assert session.num_prompt_tokens == 3
assert session.status == RequestStatus.FINISHED_ERROR
assert session.request_id not in sched.requests
assert len(sched.waiting) == 0
assert len(sched.skipped_waiting) == 0
assert sched.num_waiting_for_streaming_input == 0
sched._free_request_blocks.assert_called_once_with(session)
assert sched.finished_req_ids == {session.request_id}
assert sched.has_finished_requests()
client_index, reason = sched._streaming_context_overflow[session.request_id]
assert client_index == session.client_index
assert reason.startswith("context_length_exceeded: ")
assert "9 tokens" in reason and "max_model_len 8" in reason
engine_core_outputs = _run_idle_step(sched)
(output,) = engine_core_outputs[session.client_index].outputs
assert output.request_id == session.request_id
assert output.finish_reason == FinishReason.ERROR
assert output.stop_reason == reason
assert output.new_token_ids == []
assert engine_core_outputs[session.client_index].finished_requests == {session.request_id}
assert sched._streaming_context_overflow == {}
# Nothing is left to report once the error went out.
assert _run_idle_step(sched) == {}
def test_stage0_streaming_update_that_fills_max_model_len_exactly_finishes_the_session() -> None:
"""An exactly full prompt leaves no room for the token every duplex step
samples: upstream's running budget drops to -1, which ``schedule()`` does
not catch, so the exact fill is an overflow too."""
sched = _make_scheduler(stage_id=0)
sched.max_model_len = 8
sched.finish_requests = MagicMock()
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids([7, 8, 9])
session.num_computed_tokens = 6
session.num_output_placeholders = 0
sched._update_request_as_session(session, _make_update([10, 20]))
assert session.prompt_token_ids == [1, 2, 3]
assert session.num_prompt_tokens == 3
sched.finish_requests.assert_called_once_with((session.request_id,), RequestStatus.FINISHED_ERROR)
_, reason = sched._streaming_context_overflow[session.request_id]
assert "8 tokens" in reason and "max_model_len 8" in reason
def test_stage0_streaming_update_that_leaves_room_to_sample_is_applied() -> None:
sched = _make_scheduler(stage_id=0)
sched.max_model_len = 9
sched.finish_requests = MagicMock()
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids([7, 8, 9])
session.num_computed_tokens = 6
session.num_output_placeholders = 0
sched._update_request_as_session(session, _make_update([10, 20]))
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
assert session.num_prompt_tokens == 8
assert session.status == RequestStatus.WAITING
sched.finish_requests.assert_not_called()
assert not getattr(sched, "_streaming_context_overflow", {})
def test_stage0_streaming_update_keeps_room_for_every_token_sampled_per_step() -> None:
"""With speculative decoding a step samples more than one token, and the
prompt must leave room for all of them."""
sched = _make_scheduler(stage_id=0)
sched.max_model_len = 9
sched.num_sampled_tokens_per_step = 2
sched.finish_requests = MagicMock()
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids([7, 8, 9])
session.num_computed_tokens = 6
session.num_output_placeholders = 0
sched._update_request_as_session(session, _make_update([10, 20]))
assert session.prompt_token_ids == [1, 2, 3]
sched.finish_requests.assert_called_once_with((session.request_id,), RequestStatus.FINISHED_ERROR)
assert "8 tokens" in sched._streaming_context_overflow[session.request_id][1]
def test_stage0_streaming_update_overflow_counts_an_uncomputed_prompt() -> None:
"""A preempted session keeps its prompt even though num_computed_tokens
fell behind it; the projection must use the longer of the two."""
sched = _make_scheduler(stage_id=0)
sched.max_model_len = 4
sched.finish_requests = MagicMock()
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.num_computed_tokens = 0
sched._update_request_as_session(session, _make_update([10, 20]))
assert session.prompt_token_ids == [1, 2, 3]
sched.finish_requests.assert_called_once()
assert "5 tokens" in sched._streaming_context_overflow[session.request_id][1]
def _make_admission_scheduler(*, max_model_len: int) -> OmniARScheduler:
"""A scheduler with real admission queues, so nothing about the terminal
handling below is mocked away except block freeing."""
sched = _make_scheduler(stage_id=0)
sched.max_model_len = max_model_len
sched.policy = SchedulingPolicy.FCFS
sched.waiting = create_request_queue(sched.policy)
sched.skipped_waiting = create_request_queue(sched.policy)
sched.running = []
sched.requests = {}
sched._free_request = MagicMock() # needs a KV manager; not what this covers
return sched
def _make_queued_stop(session: Request, update: StreamingUpdate) -> None:
"""Put the request in the state upstream hands to _handle_stopped_request:
an append that arrived during generation is queued, and the stop status is
still on the request."""
from collections import deque
session.resumable = True
session.streaming_queue = deque([update])
session.status = RequestStatus.FINISHED_STOPPED
def test_queued_streaming_update_that_overflows_does_not_return_to_admission() -> None:
sched = _make_admission_scheduler(max_model_len=4)
session = _make_request()
sched.requests[session.request_id] = session
session.num_computed_tokens = 3
_make_queued_stop(session, _make_update([10, 20]))
finished = sched._handle_stopped_request(session)
assert finished is True
assert len(sched.waiting) == 0
assert len(sched.skipped_waiting) == 0
assert session.status == RequestStatus.FINISHED_ERROR
assert session.resumable is False
assert "context_length_exceeded: " in sched._streaming_context_overflow[session.request_id][1]
# update_from_output frees every request reported as finished; freeing here
# as well deleted it from self.requests twice (KeyError in _free_blocks).
sched._free_request.assert_not_called()
def test_queued_streaming_update_that_fits_still_resumes_the_session() -> None:
sched = _make_admission_scheduler(max_model_len=64)
session = _make_request()
sched.requests[session.request_id] = session
session.num_computed_tokens = 3
_make_queued_stop(session, _make_update([10, 20]))
finished = sched._handle_stopped_request(session)
assert finished is False
assert len(sched.waiting) == 1
assert session.status == RequestStatus.WAITING
assert not getattr(sched, "_streaming_context_overflow", {})
sched._free_request.assert_not_called()
def test_queued_stop_with_a_recorded_overflow_still_leaves_admission() -> None:
"""The overflow may already be on record when the stop is handled (an
earlier update of the same session). Upstream still re-enqueues the
request, so the override must take it back out either way instead of
leaving a FINISHED_ERROR request for admission to trip over."""
sched = _make_admission_scheduler(max_model_len=64)
session = _make_request()
sched.requests[session.request_id] = session
session.num_computed_tokens = 3
sched._streaming_context_overflow = {session.request_id: (session.client_index, "context_length_exceeded: earlier")}
_make_queued_stop(session, _make_update([10, 20])) # this update itself fits
finished = sched._handle_stopped_request(session)
assert finished is True
assert len(sched.waiting) == 0
assert len(sched.skipped_waiting) == 0
assert session.status == RequestStatus.FINISHED_ERROR
assert session.resumable is False
sched._free_request.assert_not_called()
def test_parked_stop_with_a_recorded_overflow_keeps_the_streaming_counter_balanced() -> None:
"""Same as above with nothing queued: upstream parks the request as
waiting for streaming input and counts it; the dequeue must uncount it."""
from collections import deque
sched = _make_admission_scheduler(max_model_len=64)
session = _make_request()
sched.requests[session.request_id] = session
session.num_computed_tokens = 3
sched._streaming_context_overflow = {session.request_id: (session.client_index, "context_length_exceeded: earlier")}
session.resumable = True
session.streaming_queue = deque()
session.status = RequestStatus.FINISHED_STOPPED
finished = sched._handle_stopped_request(session)
assert finished is True
assert len(sched.skipped_waiting) == 0
assert sched.num_waiting_for_streaming_input == 0
assert session.status == RequestStatus.FINISHED_ERROR
sched._free_request.assert_not_called()
def test_context_overflow_emits_an_error_output_with_the_reason() -> None:
sched = _make_scheduler(stage_id=0)
sched._streaming_context_overflow = {"req-a": (2, "context_length_exceeded: too long")}
outputs: dict[int, list] = {}
sched._emit_streaming_context_overflow_outputs(outputs)
(output,) = outputs[2]
assert output.request_id == "req-a"
assert output.finish_reason == FinishReason.ERROR
assert output.stop_reason == "context_length_exceeded: too long"
assert output.new_token_ids == []
assert sched._streaming_context_overflow == {}
sched._emit_streaming_context_overflow_outputs(outputs)
assert len(outputs[2]) == 1