[Bugfix] Fail a duplex session that outgrows max_model_len instead of killing the EngineCore (#7277)
Signed-off-by: Tianyao Wu <rayroy31@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
7 files changed
+582
-3
No files matched your search
@@ -149,6 +149,9 @@ def _make_scheduler_stub(requests: list[_Request]) -> SimpleNamespace:
|
||||
for name in _MIXIN_UPDATE_HELPERS:
|
||||
setattr(scheduler, name, MethodType(getattr(OmniSchedulerMixin, name), scheduler))
|
||||
scheduler._cleanup_kv_tracking = MethodType(OmniARScheduler._cleanup_kv_tracking, scheduler)
|
||||
scheduler._emit_streaming_context_overflow_outputs = MethodType(
|
||||
OmniARScheduler._emit_streaming_context_overflow_outputs, scheduler
|
||||
)
|
||||
scheduler.make_spec_decoding_stats = lambda *args, **kwargs: None
|
||||
scheduler.make_stats = lambda *args, **kwargs: None
|
||||
return scheduler
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -17,6 +18,8 @@ import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARAsyncScheduler, OmniARScheduler
|
||||
from vllm_omni.distributed.omni_connectors.transfer_adapter.chunk_transfer_adapter import (
|
||||
@@ -958,3 +961,303 @@ def test_chunk_segment_cleanup_keeps_explicit_update_stage_parked(
|
||||
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
|
||||
sched.skipped_waiting.remove_requests.assert_not_called()
|
||||
sched._enqueue_waiting_request.assert_not_called()
|
||||
|
||||
|
||||
def _make_live_session_scheduler(*, max_model_len: int) -> OmniARScheduler:
|
||||
"""A scheduler whose finish path is real: the queues, ``finish_requests``,
|
||||
``_free_request`` and the finished-request bookkeeping that wakes an idle
|
||||
engine all run. Only block freeing and the encoder cache are mocked."""
|
||||
sched = _make_admission_scheduler(max_model_len=max_model_len)
|
||||
del sched._free_request # the admission helper mocks it; run the real one
|
||||
sched._omits_kv_transfer_cache = {}
|
||||
sched.connector = None
|
||||
sched.perf_metrics = None
|
||||
sched.recompute_kv_load_failures = False
|
||||
sched.finished_req_ids = set()
|
||||
sched.finished_req_ids_dict = defaultdict(set)
|
||||
sched.transfer_triggered_requests = set()
|
||||
sched.active_kv_transfers = set()
|
||||
sched.waiting_for_transfer_free = set()
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.requests_needing_kv_transfer = {}
|
||||
sched._kv_wait_start_ts = {}
|
||||
sched.kv_cache_manager = SimpleNamespace(take_events=lambda: None, estimate_cached_tokens=lambda _request: 0)
|
||||
return sched
|
||||
|
||||
|
||||
def _run_idle_step(sched: OmniARScheduler):
|
||||
"""One ``update_from_output`` with nothing scheduled: the step an idle
|
||||
engine runs once ``has_finished_requests()`` reports a freed session."""
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_invalid_spec_tokens=0,
|
||||
)
|
||||
model_runner_output = SimpleNamespace(
|
||||
sampled_token_ids=[],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=None,
|
||||
num_nans_in_logits=None,
|
||||
kv_connector_output=None,
|
||||
cudagraph_stats=None,
|
||||
req_id_to_index={},
|
||||
routed_experts=None,
|
||||
)
|
||||
return OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
|
||||
def _park_session(sched: OmniARScheduler, session: Request) -> None:
|
||||
"""Put a session where a 1 fps video stream leaves it between appends:
|
||||
prompt partly computed, waiting for streaming input, in admission."""
|
||||
sched.requests[session.request_id] = session
|
||||
session.append_output_token_ids([7, 8, 9])
|
||||
session.num_computed_tokens = 6
|
||||
session.num_output_placeholders = 0
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
sched.skipped_waiting.add_request(session)
|
||||
sched.num_waiting_for_streaming_input = 1
|
||||
|
||||
|
||||
def test_stage0_streaming_update_that_overflows_max_model_len_finishes_the_session() -> None:
|
||||
"""A native duplex session grows its prompt on every append; once the
|
||||
extension cannot fit the model the worker crashes copying the prompt.
|
||||
The scheduler must drop the update and fail only this session, at once,
|
||||
through the real finish path: a parked session does not make the engine
|
||||
schedule, so what wakes the engine is the freed request showing up in
|
||||
``has_finished_requests()``, and the step that follows must carry one
|
||||
explicit ERROR rather than the synthesized ABORT."""
|
||||
sched = _make_live_session_scheduler(max_model_len=8)
|
||||
session = _make_request()
|
||||
_park_session(sched, session)
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20, 30]))
|
||||
|
||||
assert session.prompt_token_ids == [1, 2, 3]
|
||||
assert session.num_prompt_tokens == 3
|
||||
assert session.status == RequestStatus.FINISHED_ERROR
|
||||
assert session.request_id not in sched.requests
|
||||
assert len(sched.waiting) == 0
|
||||
assert len(sched.skipped_waiting) == 0
|
||||
assert sched.num_waiting_for_streaming_input == 0
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
assert sched.finished_req_ids == {session.request_id}
|
||||
assert sched.has_finished_requests()
|
||||
client_index, reason = sched._streaming_context_overflow[session.request_id]
|
||||
assert client_index == session.client_index
|
||||
assert reason.startswith("context_length_exceeded: ")
|
||||
assert "9 tokens" in reason and "max_model_len 8" in reason
|
||||
|
||||
engine_core_outputs = _run_idle_step(sched)
|
||||
|
||||
(output,) = engine_core_outputs[session.client_index].outputs
|
||||
assert output.request_id == session.request_id
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
assert output.stop_reason == reason
|
||||
assert output.new_token_ids == []
|
||||
assert engine_core_outputs[session.client_index].finished_requests == {session.request_id}
|
||||
assert sched._streaming_context_overflow == {}
|
||||
# Nothing is left to report once the error went out.
|
||||
assert _run_idle_step(sched) == {}
|
||||
|
||||
|
||||
def test_stage0_streaming_update_that_fills_max_model_len_exactly_finishes_the_session() -> None:
|
||||
"""An exactly full prompt leaves no room for the token every duplex step
|
||||
samples: upstream's running budget drops to -1, which ``schedule()`` does
|
||||
not catch, so the exact fill is an overflow too."""
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = 8
|
||||
sched.finish_requests = MagicMock()
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids([7, 8, 9])
|
||||
session.num_computed_tokens = 6
|
||||
session.num_output_placeholders = 0
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.prompt_token_ids == [1, 2, 3]
|
||||
assert session.num_prompt_tokens == 3
|
||||
sched.finish_requests.assert_called_once_with((session.request_id,), RequestStatus.FINISHED_ERROR)
|
||||
_, reason = sched._streaming_context_overflow[session.request_id]
|
||||
assert "8 tokens" in reason and "max_model_len 8" in reason
|
||||
|
||||
|
||||
def test_stage0_streaming_update_that_leaves_room_to_sample_is_applied() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = 9
|
||||
sched.finish_requests = MagicMock()
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids([7, 8, 9])
|
||||
session.num_computed_tokens = 6
|
||||
session.num_output_placeholders = 0
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||
assert session.num_prompt_tokens == 8
|
||||
assert session.status == RequestStatus.WAITING
|
||||
sched.finish_requests.assert_not_called()
|
||||
assert not getattr(sched, "_streaming_context_overflow", {})
|
||||
|
||||
|
||||
def test_stage0_streaming_update_keeps_room_for_every_token_sampled_per_step() -> None:
|
||||
"""With speculative decoding a step samples more than one token, and the
|
||||
prompt must leave room for all of them."""
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = 9
|
||||
sched.num_sampled_tokens_per_step = 2
|
||||
sched.finish_requests = MagicMock()
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids([7, 8, 9])
|
||||
session.num_computed_tokens = 6
|
||||
session.num_output_placeholders = 0
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.prompt_token_ids == [1, 2, 3]
|
||||
sched.finish_requests.assert_called_once_with((session.request_id,), RequestStatus.FINISHED_ERROR)
|
||||
assert "8 tokens" in sched._streaming_context_overflow[session.request_id][1]
|
||||
|
||||
|
||||
def test_stage0_streaming_update_overflow_counts_an_uncomputed_prompt() -> None:
|
||||
"""A preempted session keeps its prompt even though num_computed_tokens
|
||||
fell behind it; the projection must use the longer of the two."""
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = 4
|
||||
sched.finish_requests = MagicMock()
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.num_computed_tokens = 0
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.prompt_token_ids == [1, 2, 3]
|
||||
sched.finish_requests.assert_called_once()
|
||||
assert "5 tokens" in sched._streaming_context_overflow[session.request_id][1]
|
||||
|
||||
|
||||
def _make_admission_scheduler(*, max_model_len: int) -> OmniARScheduler:
|
||||
"""A scheduler with real admission queues, so nothing about the terminal
|
||||
handling below is mocked away except block freeing."""
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = max_model_len
|
||||
sched.policy = SchedulingPolicy.FCFS
|
||||
sched.waiting = create_request_queue(sched.policy)
|
||||
sched.skipped_waiting = create_request_queue(sched.policy)
|
||||
sched.running = []
|
||||
sched.requests = {}
|
||||
sched._free_request = MagicMock() # needs a KV manager; not what this covers
|
||||
return sched
|
||||
|
||||
|
||||
def _make_queued_stop(session: Request, update: StreamingUpdate) -> None:
|
||||
"""Put the request in the state upstream hands to _handle_stopped_request:
|
||||
an append that arrived during generation is queued, and the stop status is
|
||||
still on the request."""
|
||||
from collections import deque
|
||||
|
||||
session.resumable = True
|
||||
session.streaming_queue = deque([update])
|
||||
session.status = RequestStatus.FINISHED_STOPPED
|
||||
|
||||
|
||||
def test_queued_streaming_update_that_overflows_does_not_return_to_admission() -> None:
|
||||
sched = _make_admission_scheduler(max_model_len=4)
|
||||
session = _make_request()
|
||||
sched.requests[session.request_id] = session
|
||||
session.num_computed_tokens = 3
|
||||
_make_queued_stop(session, _make_update([10, 20]))
|
||||
|
||||
finished = sched._handle_stopped_request(session)
|
||||
|
||||
assert finished is True
|
||||
assert len(sched.waiting) == 0
|
||||
assert len(sched.skipped_waiting) == 0
|
||||
assert session.status == RequestStatus.FINISHED_ERROR
|
||||
assert session.resumable is False
|
||||
assert "context_length_exceeded: " in sched._streaming_context_overflow[session.request_id][1]
|
||||
# update_from_output frees every request reported as finished; freeing here
|
||||
# as well deleted it from self.requests twice (KeyError in _free_blocks).
|
||||
sched._free_request.assert_not_called()
|
||||
|
||||
|
||||
def test_queued_streaming_update_that_fits_still_resumes_the_session() -> None:
|
||||
sched = _make_admission_scheduler(max_model_len=64)
|
||||
session = _make_request()
|
||||
sched.requests[session.request_id] = session
|
||||
session.num_computed_tokens = 3
|
||||
_make_queued_stop(session, _make_update([10, 20]))
|
||||
|
||||
finished = sched._handle_stopped_request(session)
|
||||
|
||||
assert finished is False
|
||||
assert len(sched.waiting) == 1
|
||||
assert session.status == RequestStatus.WAITING
|
||||
assert not getattr(sched, "_streaming_context_overflow", {})
|
||||
sched._free_request.assert_not_called()
|
||||
|
||||
|
||||
def test_queued_stop_with_a_recorded_overflow_still_leaves_admission() -> None:
|
||||
"""The overflow may already be on record when the stop is handled (an
|
||||
earlier update of the same session). Upstream still re-enqueues the
|
||||
request, so the override must take it back out either way instead of
|
||||
leaving a FINISHED_ERROR request for admission to trip over."""
|
||||
sched = _make_admission_scheduler(max_model_len=64)
|
||||
session = _make_request()
|
||||
sched.requests[session.request_id] = session
|
||||
session.num_computed_tokens = 3
|
||||
sched._streaming_context_overflow = {session.request_id: (session.client_index, "context_length_exceeded: earlier")}
|
||||
_make_queued_stop(session, _make_update([10, 20])) # this update itself fits
|
||||
|
||||
finished = sched._handle_stopped_request(session)
|
||||
|
||||
assert finished is True
|
||||
assert len(sched.waiting) == 0
|
||||
assert len(sched.skipped_waiting) == 0
|
||||
assert session.status == RequestStatus.FINISHED_ERROR
|
||||
assert session.resumable is False
|
||||
sched._free_request.assert_not_called()
|
||||
|
||||
|
||||
def test_parked_stop_with_a_recorded_overflow_keeps_the_streaming_counter_balanced() -> None:
|
||||
"""Same as above with nothing queued: upstream parks the request as
|
||||
waiting for streaming input and counts it; the dequeue must uncount it."""
|
||||
from collections import deque
|
||||
|
||||
sched = _make_admission_scheduler(max_model_len=64)
|
||||
session = _make_request()
|
||||
sched.requests[session.request_id] = session
|
||||
session.num_computed_tokens = 3
|
||||
sched._streaming_context_overflow = {session.request_id: (session.client_index, "context_length_exceeded: earlier")}
|
||||
session.resumable = True
|
||||
session.streaming_queue = deque()
|
||||
session.status = RequestStatus.FINISHED_STOPPED
|
||||
|
||||
finished = sched._handle_stopped_request(session)
|
||||
|
||||
assert finished is True
|
||||
assert len(sched.skipped_waiting) == 0
|
||||
assert sched.num_waiting_for_streaming_input == 0
|
||||
assert session.status == RequestStatus.FINISHED_ERROR
|
||||
sched._free_request.assert_not_called()
|
||||
|
||||
|
||||
def test_context_overflow_emits_an_error_output_with_the_reason() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched._streaming_context_overflow = {"req-a": (2, "context_length_exceeded: too long")}
|
||||
outputs: dict[int, list] = {}
|
||||
|
||||
sched._emit_streaming_context_overflow_outputs(outputs)
|
||||
|
||||
(output,) = outputs[2]
|
||||
assert output.request_id == "req-a"
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
assert output.stop_reason == "context_length_exceeded: too long"
|
||||
assert output.new_token_ids == []
|
||||
assert sched._streaming_context_overflow == {}
|
||||
|
||||
sched._emit_streaming_context_overflow_outputs(outputs)
|
||||
assert len(outputs[2]) == 1
|
||||
Reference in new issue
Block a user