[Bugfix][MiniCPM-o] Prevent native duplex Stage-1 prompt overflow

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-08-24 05:13:31 +08:00
1 parent 1c917b2751
commit 5edee1393a
32 files changed
+1591 -99

No files matched your search

@@ -0,0 +1,115 @@
"""Tests for ``OmniARScheduler._drop_aborted_queued_requests``.
``schedule()`` sweeps ``FINISHED_ABORTED`` requests out of the queues before
handing control to upstream, because upstream ``Scheduler.schedule()`` raises
``RuntimeError: Invalid request status: FINISHED_ABORTED`` for any request it
admits in a finished state -- which kills the stage's engine core, not just
the request.
Regression for the sweep missing ``skipped_waiting``, the third queue
upstream admits from. An aborted duplex session parked there was re-selected
by ``_select_waiting_queue_for_scheduling`` on a later tick and crashed the
stage.
"""
from __future__ import annotations
import pytest
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
from vllm.v1.request import RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_POLICIES = [
pytest.param(SchedulingPolicy.FCFS, id="fcfs"),
pytest.param(SchedulingPolicy.PRIORITY, id="priority"),
]
class _StubRequest:
"""Minimal ``Request`` stub with the surface the sweep exercises.
Carries the ``(priority, arrival_time)`` ordering ``PriorityRequestQueue``
relies on so the same stub works under both scheduling policies.
"""
def __init__(self, request_id: str, status: RequestStatus, arrival_time: float = 0.0) -> None:
self.request_id = request_id
self.status = status
self.priority = 0
self.arrival_time = arrival_time
def __lt__(self, other: _StubRequest) -> bool:
return (self.priority, self.arrival_time) < (other.priority, other.arrival_time)
def _make_scheduler(policy: SchedulingPolicy, *, waiting=(), skipped_waiting=(), running=()):
"""Bare scheduler with just the surface the sweep reads."""
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler.waiting = create_request_queue(policy)
for req in waiting:
scheduler.waiting.add_request(req)
scheduler.skipped_waiting = create_request_queue(policy)
for req in skipped_waiting:
scheduler.skipped_waiting.add_request(req)
scheduler.running = list(running)
return scheduler
@pytest.mark.parametrize("policy", _POLICIES)
def test_sweep_drops_aborted_request_from_skipped_waiting(policy: SchedulingPolicy) -> None:
"""The sweep must cover ``skipped_waiting``.
Upstream ``schedule()`` admits from ``skipped_waiting`` as readily as from
``waiting``, so an aborted request left there is re-selected and raises.
"""
aborted = _StubRequest("req-aborted", RequestStatus.FINISHED_ABORTED)
scheduler = _make_scheduler(policy, skipped_waiting=[aborted])
scheduler._drop_aborted_queued_requests()
assert list(scheduler.skipped_waiting) == []
@pytest.mark.parametrize("policy", _POLICIES)
def test_sweep_drops_aborted_requests_from_waiting_and_running(policy: SchedulingPolicy) -> None:
"""``waiting`` and ``running`` stay covered alongside the new queue."""
aborted_waiting = _StubRequest("req-waiting", RequestStatus.FINISHED_ABORTED)
aborted_running = _StubRequest("req-running", RequestStatus.FINISHED_ABORTED)
scheduler = _make_scheduler(
policy,
waiting=[aborted_waiting],
running=[aborted_running],
)
scheduler._drop_aborted_queued_requests()
assert list(scheduler.waiting) == []
assert scheduler.running == []
@pytest.mark.parametrize("policy", _POLICIES)
def test_sweep_leaves_live_requests_intact(policy: SchedulingPolicy) -> None:
"""The sweep must not over-sweep.
A request parked in ``skipped_waiting`` on a blocked waiting status is one
upstream still intends to schedule; only the aborted entry may go.
"""
blocked = _StubRequest("req-blocked", RequestStatus.WAITING_FOR_STREAMING_REQ, arrival_time=0.0)
aborted = _StubRequest("req-aborted", RequestStatus.FINISHED_ABORTED, arrival_time=1.0)
live_waiting = _StubRequest("req-live-waiting", RequestStatus.WAITING)
live_running = _StubRequest("req-live-running", RequestStatus.RUNNING)
scheduler = _make_scheduler(
policy,
waiting=[live_waiting],
skipped_waiting=[blocked, aborted],
running=[live_running],
)
scheduler._drop_aborted_queued_requests()
assert list(scheduler.skipped_waiting) == [blocked]
assert list(scheduler.waiting) == [live_waiting]
assert scheduler.running == [live_running]
@@ -28,7 +28,9 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_scheduler(*, stage_id: int = 0) -> OmniARScheduler:
sched = OmniARScheduler.__new__(OmniARScheduler)
sched._new_prompt_len_snapshot = {}
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=stage_id))
sched.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(stage_id=stage_id, max_model_len=None),
)
sched.num_waiting_for_streaming_input = 0
sched.log_stats = False
sched.chunk_transfer_adapter = None
@@ -210,6 +212,81 @@ def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> Non
sched.chunk_transfer_adapter.save_async.assert_not_called()
def test_stale_async_frame_is_dropped_before_output_processing() -> None:
session = _make_request()
session.status = RequestStatus.RUNNING
session.num_in_flight_tokens = 2
session.num_computed_tokens = session.num_tokens + 1
session.num_output_placeholders = 1
session.async_tokens_to_discard = 1
session.sampling_params = SimpleNamespace(num_logprobs=1)
num_computed_tokens = session.num_computed_tokens
num_output_placeholders = session.num_output_placeholders
sched = MagicMock()
sched.requests = {session.request_id: session}
sched.perf_metrics = None
sched.structured_output_manager.should_advance.return_value = False
def discard_stale_output(request: Request, token_ids: list[int]) -> tuple[list[int], bool]:
request.async_tokens_to_discard = 0
return token_ids, False
sched._update_request_with_output.side_effect = discard_stale_output
sched._process_kv_transfer_trigger.return_value = False
sched.chunk_transfer_adapter = MagicMock()
sched.running = [session]
sched.waiting_for_transfer_free = set()
sched.transfer_triggered_requests = set()
sched.active_kv_transfers = set()
sched.pending_stop_after_extraction = set()
sched.connector = None
sched.kv_cache_manager.take_events.return_value = None
sched.finished_req_ids_dict = {}
sched.make_stats.return_value = None
scheduler_output = MagicMock(spec=SchedulerOutput)
scheduler_output.num_scheduled_tokens = {session.request_id: 1}
scheduler_output.scheduled_spec_decode_tokens = {}
scheduler_output.num_invalid_spec_tokens = 0
model_runner_output = MagicMock(spec=ModelRunnerOutput)
model_runner_output.sampled_token_ids = [[42]]
model_runner_output.logprobs = None
model_runner_output.prompt_logprobs_dict = {}
model_runner_output.pooler_output = None
model_runner_output.multimodal_outputs = None
model_runner_output.inter_stage_outputs = [{"hidden": object()}]
model_runner_output.num_nans_in_logits = None
model_runner_output.kv_connector_output = None
model_runner_output.cudagraph_stats = None
model_runner_output.req_id_to_index = {session.request_id: 0}
model_runner_output.routed_experts = None
OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)
assert session.async_tokens_to_discard == 0
assert session.status == RequestStatus.RUNNING
assert session.num_computed_tokens == num_computed_tokens
assert session.num_output_placeholders == num_output_placeholders
sched.chunk_transfer_adapter.save_async.assert_not_called()
session.sampling_params = SamplingParams(max_tokens=8)
next_payload = {"hidden": object()}
sched._update_request_with_output.side_effect = None
sched._update_request_with_output.return_value = ([43], False)
model_runner_output.sampled_token_ids = [[43]]
model_runner_output.inter_stage_outputs = [next_payload]
OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)
sched.chunk_transfer_adapter.save_async.assert_called_once_with(
next_payload,
session,
False,
)
def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None:
sched = _make_scheduler(stage_id=0)
session = _make_request()
@@ -354,3 +431,34 @@ def test_ready_async_chunk_prompt_replacement_releases_stale_kv_once() -> None:
assert sched.chunk_transfer_adapter.replaced_streaming_prompt_ids == set()
assert sched.chunk_transfer_adapter.requests_with_ready_chunks == {session.request_id}
assert sched.chunk_transfer_adapter.requests_num_chunks_sent == {}
def test_native_duplex_prompt_reset_prevents_stage1_context_overflow() -> None:
sched = _make_scheduler(stage_id=1)
sched.vllm_config.model_config.max_model_len = 4096
sched.chunk_transfer_adapter = SimpleNamespace(
receives_chunks=False,
segment_finished_requests=set(),
)
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.resumable = True
session.prompt_token_ids = [0] * 3
session._all_token_ids.clear()
session._all_token_ids.extend([0] * 4095)
session._output_token_ids.extend([1] * 4080)
session.num_prompt_tokens = 3
session.num_computed_tokens = 4095
update = _make_update([0] * 16)
update.model_intermediate_buffer = {"native_duplex": True}
sched._update_request_as_session(session, update)
assert session.prompt_token_ids == [0] * 16
assert list(session._all_token_ids) == [0] * 16
assert session._output_token_ids == []
assert session.num_prompt_tokens == 16
assert session.num_computed_tokens == 0
assert session.model_intermediate_buffer == update.model_intermediate_buffer
assert session.status == RequestStatus.WAITING
@@ -76,6 +76,9 @@ def _make_scheduler(
else:
hf_config.voxcpm2_runtime_config = runtime_config
sched.vllm_config = SimpleNamespace(model_config=model_config)
# Upstream ``Scheduler.__init__`` always creates ``skipped_waiting``; the
# abort sweep at the top of ``schedule()`` reads it.
sched.skipped_waiting = _MockQueue()
return sched
@@ -0,0 +1,254 @@
"""Tests for the ``num_waiting_for_streaming_input`` resync placed in
``OmniARScheduler.finish_requests`` / ``OmniGenerationScheduler.finish_requests``
and in ``OmniARScheduler.schedule``.
Upstream keeps that counter incrementally (``+1`` when a resumable request
parks in a waiting queue as ``WAITING_FOR_STREAMING_REQ``, ``-1`` when the
next update arrives or the request is finished) and
``Scheduler.get_num_unfinished_requests`` *subtracts* it from the waiting
queues. Omni rewrites ``request.status`` outside both hooks -- the
chunk-transfer adapter's park/restore and
``_realign_request_status_to_queues`` -- so a counted request can leave
``WAITING_FOR_STREAMING_REQ`` without the matching decrement and inflate the
counter for the lifetime of the process.
An inflated counter does not degrade throughput, it hangs the stage:
``EngineCore.has_work()`` reads false while a live request sits in
``waiting``, so the engine blocks in ``input_queue.get()`` and never calls
``schedule()`` again. Regression for the Qwen3-Omni duplex symptom where the
server answered the first conversation after a boot and was silent from the
second on while still reporting healthy.
"""
from __future__ import annotations
import pytest
from vllm.v1.request import RequestStatus
import vllm_omni.core.sched.omni_ar_scheduler as ar_sched_mod
import vllm_omni.core.sched.omni_generation_scheduler as gen_sched_mod
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _StubRequest:
def __init__(self, request_id: str, status: RequestStatus) -> None:
self.request_id = request_id
self.status = status
def is_finished(self) -> bool:
return RequestStatus.is_finished(self.status)
_SCHEDULER_PARAMS = [
pytest.param(OmniARScheduler, ar_sched_mod, id="ar"),
pytest.param(OmniGenerationScheduler, gen_sched_mod, id="generation"),
]
def _make_scheduler(scheduler_cls, *, requests, running, waiting, counter, skipped=None):
scheduler = scheduler_cls.__new__(scheduler_cls)
scheduler.chunk_transfer_adapter = None
scheduler.input_coordinator = None
scheduler.requests = requests
scheduler.running = running
scheduler.waiting = waiting
scheduler.skipped_waiting = skipped if skipped is not None else []
scheduler.num_waiting_for_streaming_input = counter
return scheduler
def _upstream_num_unfinished(scheduler) -> int:
"""Mirror of ``Scheduler.get_num_unfinished_requests`` arithmetic."""
num_waiting = (
len(scheduler.waiting) + len(scheduler.skipped_waiting) - scheduler.num_waiting_for_streaming_input
)
return num_waiting + len(scheduler.running)
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
def test_finish_requests_clears_counter_left_by_a_stomped_status(
monkeypatch: pytest.MonkeyPatch,
scheduler_cls,
scheduler_mod,
) -> None:
"""The abort case that wedged the talker.
The request was counted while parked for streaming input, then had its
status rewritten to ``RUNNING`` by omni's queue surgery, so upstream's
``finish_requests`` took the running branch and never decremented. With
no parked requests left, the counter must land at 0.
"""
aborted = _StubRequest("req-aborted", RequestStatus.RUNNING)
scheduler = _make_scheduler(
scheduler_cls,
requests={}, # upstream already dropped it
running=[],
waiting=[],
counter=1,
)
monkeypatch.setattr(
scheduler_mod.VLLMScheduler,
"finish_requests",
lambda self, request_ids, finished_status: [aborted],
)
scheduler_cls.finish_requests(scheduler, [aborted.request_id], RequestStatus.FINISHED_ABORTED)
assert scheduler.num_waiting_for_streaming_input == 0
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
def test_next_request_is_visible_as_work_after_the_abort(
monkeypatch: pytest.MonkeyPatch,
scheduler_cls,
scheduler_mod,
) -> None:
"""The consequence, stated as the engine sees it.
A fresh request queued behind the leak must read as unfinished work.
Without the resync this is ``(1 + 0 - 1) + 0 == 0`` and the engine parks
in ``input_queue.get()`` with a live request in ``waiting``.
"""
aborted = _StubRequest("req-aborted", RequestStatus.RUNNING)
incoming = _StubRequest("req-next-session", RequestStatus.WAITING)
scheduler = _make_scheduler(
scheduler_cls,
requests={incoming.request_id: incoming},
running=[],
waiting=[incoming],
counter=1,
)
monkeypatch.setattr(
scheduler_mod.VLLMScheduler,
"finish_requests",
lambda self, request_ids, finished_status: [aborted],
)
scheduler_cls.finish_requests(scheduler, [aborted.request_id], RequestStatus.FINISHED_ABORTED)
assert _upstream_num_unfinished(scheduler) == 1
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
def test_finish_requests_keeps_counting_a_genuinely_parked_request(
monkeypatch: pytest.MonkeyPatch,
scheduler_cls,
scheduler_mod,
) -> None:
"""The resync must not over-correct.
A request still parked as ``WAITING_FOR_STREAMING_REQ`` in ``waiting`` is
exactly what the counter is for -- it must stay counted, so the engine
does not spin on a request that has no input to process.
"""
parked = _StubRequest("req-parked", RequestStatus.WAITING_FOR_STREAMING_REQ)
leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED)
scheduler = _make_scheduler(
scheduler_cls,
requests={parked.request_id: parked},
running=[],
waiting=[parked],
counter=1,
)
monkeypatch.setattr(
scheduler_mod.VLLMScheduler,
"finish_requests",
lambda self, request_ids, finished_status: [leaving],
)
scheduler_cls.finish_requests(scheduler, [leaving.request_id], RequestStatus.FINISHED_STOPPED)
assert scheduler.num_waiting_for_streaming_input == 1
assert _upstream_num_unfinished(scheduler) == 0
def test_resync_counts_parked_requests_in_skipped_waiting() -> None:
"""``get_num_unfinished_requests`` sums ``waiting`` and ``skipped_waiting``,
so the resync has to look at both or it under-counts and reintroduces the
spin the counter exists to prevent."""
parked = _StubRequest("req-skipped", RequestStatus.WAITING_FOR_STREAMING_REQ)
scheduler = _make_scheduler(
OmniARScheduler,
requests={parked.request_id: parked},
running=[],
waiting=[],
counter=0,
skipped=[parked],
)
scheduler._resync_streaming_input_counter()
assert scheduler.num_waiting_for_streaming_input == 1
def test_resync_is_a_no_op_when_upstream_accounting_is_correct() -> None:
"""In the healthy case the derived value equals upstream's incremental one,
so the helper must leave it alone rather than fight it."""
parked = _StubRequest("req-parked", RequestStatus.WAITING_FOR_STREAMING_REQ)
running = _StubRequest("req-running", RequestStatus.RUNNING)
scheduler = _make_scheduler(
OmniARScheduler,
requests={r.request_id: r for r in (parked, running)},
running=[running],
waiting=[parked],
counter=1,
)
scheduler._resync_streaming_input_counter()
assert scheduler.num_waiting_for_streaming_input == 1
def test_resync_tolerates_a_scheduler_without_the_counter() -> None:
"""Guard for vLLM versions that predate ``num_waiting_for_streaming_input``:
the helper must no-op rather than invent the attribute."""
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler.waiting = []
scheduler.skipped_waiting = []
scheduler._resync_streaming_input_counter()
assert not hasattr(scheduler, "num_waiting_for_streaming_input")
def test_ar_schedule_resyncs_before_delegating_upstream(monkeypatch: pytest.MonkeyPatch) -> None:
"""Pins placement in ``schedule``: the chunk-transfer adapter rewrites
status during ``process_pending_chunks``, so the resync has to run after
that and before upstream schedules, while ``self.waiting`` is still the
real queue."""
parked = _StubRequest("req-resumed", RequestStatus.RUNNING)
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler.chunk_transfer_adapter = None
scheduler.input_coordinator = None
scheduler.requests = {parked.request_id: parked}
scheduler.running = [parked]
scheduler.waiting = []
scheduler.skipped_waiting = []
scheduler.num_waiting_for_streaming_input = 1 # leaked by the status rewrite
scheduler.policy = "fcfs"
seen: dict[str, int] = {}
monkeypatch.setattr(ar_sched_mod.OmniSchedulerMixin, "_consume_pending_connector_output", lambda self, model_mode: None)
monkeypatch.setattr(ar_sched_mod.OmniSchedulerMixin, "_process_pending_input_timeouts", lambda self: None)
monkeypatch.setattr(OmniARScheduler, "_should_defer_waiting_admission", lambda self: False)
def fake_schedule(self, throttle_prefills=False):
seen["counter"] = self.num_waiting_for_streaming_input
raise _StopScheduleError
class _StopScheduleError(Exception):
pass
monkeypatch.setattr(ar_sched_mod.VLLMScheduler, "schedule", fake_schedule)
with pytest.raises(_StopScheduleError):
OmniARScheduler.schedule(scheduler)
assert seen["counter"] == 0