[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
Zhou Taichang authored and GitHub committed 2026-10-05 14:11:01 +08:00
1 parent 1f47fa2f29
commit 0b0d2d7009
141 files changed
+2885 -721

No files matched your search

@@ -171,6 +171,7 @@ def test_cancelled_notification_is_filtered_before_coordinator(monkeypatch, mock
s, _ = scheduler(monkeypatch, [event])
s._generation_max_wait_s = 0
s.waiting = []
s.kv_holding_waiting = []
s.input_coordinator._async_chunk = True
s.input_coordinator.update_request_metadata = mocker.Mock()
s.input_coordinator.process_pending_chunks = mocker.Mock()
@@ -178,7 +179,11 @@ def test_cancelled_notification_is_filtered_before_coordinator(monkeypatch, mock
s.input_coordinator.update_request_metadata.assert_called_once_with(
s.requests, {"0": {"code_predictor_codes": [7]}}, model_mode="generation"
)
s.input_coordinator.process_pending_chunks.assert_called_once_with(s.waiting, s.running, {"0"}, set())
s.input_coordinator.process_pending_chunks.assert_called_once()
waiting, running, ready, finished = s.input_coordinator.process_pending_chunks.call_args.args
assert list(waiting) == []
assert running is s.running
assert ready == {"0"} and finished == set()
@pytest.mark.parametrize("target,wait", [(0, 1), (257, 1), (4, -1), (4, float("nan")), (4, float("inf"))])
@@ -11,7 +11,9 @@ from types import SimpleNamespace
import pytest
import torch
from vllm import SamplingParams
from vllm.distributed.aux_output_connector.connector import AuxOutputSchedulerConnector
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.output import CachedRequestData
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
from vllm.v1.request import Request, RequestStatus
@@ -53,12 +55,14 @@ class FakeAdapter:
def _make_generation_scheduler(waiting_request, *, use_v2_model_runner=False):
scheduler = OmniGenerationScheduler.__new__(OmniGenerationScheduler)
scheduler.max_num_scheduled_tokens = 8
scheduler.max_num_active_reqs = 1
scheduler.max_num_running_reqs = 1
scheduler._pause_state = PauseState.UNPAUSED
scheduler.running = []
scheduler.waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.waiting.add_request(waiting_request)
scheduler.skipped_waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.kv_holding_waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.deferred_waiting = set()
scheduler.requests = {waiting_request.request_id: waiting_request}
scheduler.policy = SchedulingPolicy.FCFS
scheduler.chunk_transfer_adapter = FakeAdapter()
@@ -86,6 +90,7 @@ def _make_generation_scheduler(waiting_request, *, use_v2_model_runner=False):
get_manager_metadata=lambda: None,
)
scheduler.connector = None
scheduler.aux_output_connector = None
scheduler.ec_connector = None
scheduler.prev_step_scheduled_req_ids = set()
scheduler._pending_finish_reqs = []
@@ -105,6 +110,45 @@ def _make_generation_scheduler(waiting_request, *, use_v2_model_runner=False):
return scheduler
def test_generation_scheduler_preserves_auxiliary_output_metadata() -> None:
request = Request("aux-contract", [1, 2], SamplingParams(max_tokens=2), pooling_params=None)
request.block_hashes.append(b"block-hash")
scheduler = _make_generation_scheduler(request, use_v2_model_runner=True)
scheduler.aux_output_connector = AuxOutputSchedulerConnector()
output = scheduler._build_generation_scheduler_output(
new_reqs_data=[],
cached_reqs_data=CachedRequestData.make_empty(),
num_scheduled_tokens={request.request_id: 1},
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
)
metadata = output.aux_output_connector_metadata
assert metadata is not None
assert metadata.requests == {request.request_id: 0}
assert list(metadata.block_hashes[request.request_id]) == request.block_hashes
@pytest.mark.parametrize(("runner_slots", "active_limit"), [(1, 2), (2, 1)])
def test_generation_admits_kv_holder_before_fresh_requests(runner_slots, active_limit) -> None:
fresh = Request("fresh", [1, 2], SamplingParams(max_tokens=2), pooling_params=None)
holding = Request("holding", [3, 4], SamplingParams(max_tokens=2), pooling_params=None)
holding.num_computed_tokens = 1
scheduler = _make_generation_scheduler(fresh)
scheduler.max_num_running_reqs = runner_slots
scheduler.max_num_active_reqs = active_limit
scheduler.requests[holding.request_id] = holding
scheduler.kv_holding_waiting.add_request(holding)
output = scheduler.schedule()
assert set(output.num_scheduled_tokens) == {holding.request_id}
assert list(scheduler.waiting) == [fresh]
assert list(scheduler.kv_holding_waiting) == []
def _chunk_request(request_id, **kwargs):
defaults = dict(
request_id=request_id,
@@ -123,7 +167,13 @@ def _chunk_request(request_id, **kwargs):
record_event=lambda *args, **kwargs: None,
)
defaults.update(kwargs)
return SimpleNamespace(**defaults)
return _HashableChunkRequest(**defaults)
class _HashableChunkRequest(SimpleNamespace):
# Deferred queues require identity hashing; SimpleNamespace types it as None.
__hash__ = object.__hash__ # type: ignore[assignment]
__eq__ = object.__eq__
def test_chunk_lifecycle_no_resubmit_and_state_survives_requeue(monkeypatch) -> None:
@@ -138,7 +188,8 @@ def test_chunk_lifecycle_no_resubmit_and_state_survives_requeue(monkeypatch) ->
)
scheduler.running = []
scheduler.waiting = create_request_queue(scheduler.policy)
scheduler.skipped_waiting = create_request_queue(scheduler.policy)
scheduler.kv_holding_waiting = create_request_queue(scheduler.policy)
scheduler.deferred_waiting = set()
completed = Request("completed", [1, 2], SamplingParams(max_tokens=4), pooling_params=None)
completed.status = RequestStatus.RUNNING
@@ -292,6 +343,7 @@ def _express_scheduler():
scheduler = _make_generation_scheduler(continuation, use_v2_model_runner=True)
scheduler.waiting.add_request(first)
scheduler.requests[first.request_id] = first
scheduler.max_num_active_reqs = 4
scheduler.max_num_running_reqs = 4
scheduler._native_data_plane = True
scheduler.chunk_transfer_adapter = None
@@ -9,7 +9,7 @@ handing control to upstream, because upstream ``Scheduler.schedule()`` raises
admits in a finished state -- which kills the stage's engine core, not just
the request.
Regression for the sweep missing ``skipped_waiting``, the third queue
Regression for the sweep missing ``kv_holding_waiting``, the third queue
upstream admits from. An aborted duplex session parked there was re-selected
by ``_select_waiting_queue_for_scheduling`` on a later tick and crashed the
stage.
@@ -49,28 +49,30 @@ class _StubRequest:
return (self.priority, self.arrival_time) < (other.priority, other.arrival_time)
def _make_scheduler(policy: SchedulingPolicy, *, waiting=(), skipped_waiting=(), running=()):
def _make_scheduler(policy: SchedulingPolicy, *, waiting=(), kv_holding_waiting=(), running=()):
"""Bare scheduler with just the surface the sweep reads."""
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler.waiting = create_request_queue(policy)
for req in waiting:
scheduler.waiting.add_request(req)
scheduler.skipped_waiting = create_request_queue(policy)
for req in skipped_waiting:
scheduler.skipped_waiting.add_request(req)
scheduler.kv_holding_waiting = create_request_queue(policy)
scheduler.deferred_waiting = set()
for req in kv_holding_waiting:
scheduler.kv_holding_waiting.add_request(req)
scheduler.running = list(running)
return scheduler
@pytest.mark.parametrize("policy", _POLICIES)
def test_schedule_sweeps_skipped_waiting_before_upstream_selection(
def test_schedule_sweeps_kv_holding_waiting_before_upstream_selection(
monkeypatch: pytest.MonkeyPatch,
policy: SchedulingPolicy,
) -> None:
"""The public schedule path must remove aborted skipped requests first."""
aborted = _StubRequest("req-aborted", RequestStatus.FINISHED_ABORTED)
scheduler = _make_scheduler(policy, skipped_waiting=[aborted])
scheduler = _make_scheduler(policy, kv_holding_waiting=[aborted])
scheduler.chunk_transfer_adapter = None
scheduler.max_num_active_reqs = 8
scheduler.max_num_running_reqs = 8
scheduler.input_coordinator = None
scheduler.num_waiting_for_streaming_input = 0
@@ -85,7 +87,7 @@ def test_schedule_sweeps_skipped_waiting_before_upstream_selection(
sentinel = object()
def upstream_schedule(_self, _throttle_prefills: bool = False):
assert list(scheduler.skipped_waiting) == []
assert list(scheduler.kv_holding_waiting) == []
return sentinel
monkeypatch.setattr(ar_sched_mod.VLLMScheduler, "schedule", upstream_schedule)
@@ -185,10 +185,15 @@ def test_stage_zero_final_skips_inter_stage_output(mocker):
def test_streaming_session_update_invalidates_omits_kv_transfer_cache():
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler._omits_kv_transfer_cache = {}
scheduler.skipped_waiting = []
scheduler.deferred_waiting = set()
scheduler.kv_holding_waiting = []
scheduler.log_stats = False
tagged = apply_omni_final_stage_metadata(_make_engine_request(), final_stage_id=0)
session = SimpleNamespace(
class Session(SimpleNamespace):
__hash__ = object.__hash__
session = Session(
request_id="req",
additional_information=tagged.additional_information,
status=RequestStatus.WAITING,
@@ -11,12 +11,14 @@ from types import MethodType, SimpleNamespace
import numpy as np
import pytest
import torch
from vllm.v1.engine import FinishReason
from vllm.v1.outputs import LogprobsLists
from vllm.v1.request import RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler, _slice_sampled_logprobs
from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin
from vllm_omni.outputs import OmniModelRunnerOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -132,11 +134,13 @@ def _make_scheduler_stub(requests: list[_Request]) -> SimpleNamespace:
scheduler = SimpleNamespace(
perf_metrics=None,
connector=None,
aux_output_connector=None,
chunk_transfer_adapter=None,
requests={request.request_id: request for request in requests},
running=list(requests),
waiting=_RequestQueue(),
skipped_waiting=_RequestQueue(),
kv_holding_waiting=_RequestQueue(),
deferred_waiting=set(),
structured_output_manager=SimpleNamespace(accept_tokens=lambda _request, _token_ids: True),
transfer_triggered_requests=set(),
active_kv_transfers=set(),
@@ -292,6 +296,36 @@ def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None:
np.testing.assert_array_equal(output_by_id["good"].new_logprobs.logprob_token_ids[:, 0], [8])
@pytest.mark.parametrize("scored", [False, True])
def test_fixed_token_scores_reach_engine_output(scored) -> None:
request = _Request("req")
request.sampling_params.num_logprobs = None
scheduler = _make_scheduler_stub([request])
_bind_request_lifecycle(scheduler, update_request=lambda req, tokens: (tokens, False))
scheduler_output = SimpleNamespace(
num_scheduled_tokens={"req": 3},
scheduled_spec_decode_tokens={},
num_invalid_spec_tokens=0,
)
scores = torch.tensor([[-0.5, -1.5], [-0.2, -2.2]])
model_runner_output = OmniModelRunnerOutput(
req_ids=["req"],
req_id_to_index={"req": 0},
sampled_token_ids=[[7]],
prompt_logprobs_dict={},
prompt_token_id_logprobs_dict={"req": scores} if scored else {},
)
outputs = OmniARScheduler.update_from_output(scheduler, scheduler_output, model_runner_output)
(output,) = outputs[0].outputs
assert output.new_token_ids == [7]
if scored:
torch.testing.assert_close(output.prompt_token_id_logprobs, scores)
else:
assert output.prompt_token_id_logprobs is None
def _pooling_model_runner_output(pooler_tensor) -> SimpleNamespace:
return SimpleNamespace(
sampled_token_ids=[[]],
@@ -73,7 +73,8 @@ def _replace_streaming_session(session: Request) -> None:
sched.num_waiting_for_streaming_input = 0
sched.log_stats = False
sched.chunk_transfer_adapter = None
sched.skipped_waiting = set()
sched.kv_holding_waiting = set()
sched.deferred_waiting = set()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
update = StreamingUpdate(
mm_features=None,
@@ -43,7 +43,8 @@ def _make_scheduler(*, stage_id: int = 0, session_mode: str = "turn") -> OmniARS
sched.num_waiting_for_streaming_input = 0
sched.log_stats = False
sched.chunk_transfer_adapter = None
sched.skipped_waiting = set()
sched.kv_holding_waiting = set()
sched.deferred_waiting = set()
sched._free_request_blocks = MagicMock()
sched.encoder_cache_manager = MagicMock()
sched._inflight_prefills = set()
@@ -1185,7 +1186,8 @@ def test_chunk_segment_cleanup_keeps_requeued_resumable_receiver() -> None:
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(session_mode="duplex"))
sched.running = []
sched.waiting = queue()
sched.skipped_waiting = queue(session)
sched.kv_holding_waiting = queue(session)
sched.deferred_waiting = set()
sched.num_waiting_for_streaming_input = 1
sched.chunk_transfer_adapter = SimpleNamespace(
receives_chunks=True,
@@ -1197,7 +1199,7 @@ def test_chunk_segment_cleanup_keeps_requeued_resumable_receiver() -> None:
assert session.status == RequestStatus.WAITING
assert session in sched.waiting.requests
assert session not in sched.skipped_waiting.requests
assert session not in sched.kv_holding_waiting.requests
assert sched.num_waiting_for_streaming_input == 0
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
@@ -1224,7 +1226,8 @@ def test_chunk_segment_cleanup_keeps_explicit_update_stage_parked(
receives_chunks=receives_chunks,
segment_finished_requests={session.request_id},
)
sched.skipped_waiting = MagicMock()
sched.kv_holding_waiting = MagicMock()
sched.deferred_waiting = set()
sched._enqueue_waiting_request = MagicMock()
sched._resume_downstream_chunk_receiver(session)
@@ -1232,7 +1235,7 @@ def test_chunk_segment_cleanup_keeps_explicit_update_stage_parked(
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
assert sched.num_waiting_for_streaming_input == 1
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
sched.skipped_waiting.remove_requests.assert_not_called()
sched.kv_holding_waiting.remove_requests.assert_not_called()
sched._enqueue_waiting_request.assert_not_called()
@@ -1288,7 +1291,8 @@ def _park_session(sched: OmniARScheduler, session: Request) -> None:
session.num_computed_tokens = 6
session.num_output_placeholders = 0
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
sched.skipped_waiting.add_request(session)
sched.kv_holding_waiting.add_request(session)
sched.deferred_waiting.add(session)
sched.num_waiting_for_streaming_input = 1
@@ -1311,7 +1315,7 @@ def test_stage0_streaming_update_that_overflows_max_model_len_finishes_the_sessi
assert session.status == RequestStatus.FINISHED_ERROR
assert session.request_id not in sched.requests
assert len(sched.waiting) == 0
assert len(sched.skipped_waiting) == 0
assert len(sched.kv_holding_waiting) == 0
assert sched.num_waiting_for_streaming_input == 0
sched._free_request_blocks.assert_called_once_with(session)
assert sched.finished_req_ids == {session.request_id}
@@ -1419,7 +1423,8 @@ def _make_admission_scheduler(*, max_model_len: int) -> OmniARScheduler:
sched.max_model_len = max_model_len
sched.policy = SchedulingPolicy.FCFS
sched.waiting = create_request_queue(sched.policy)
sched.skipped_waiting = create_request_queue(sched.policy)
sched.kv_holding_waiting = create_request_queue(sched.policy)
sched.deferred_waiting = set()
sched.running = []
sched.requests = {}
sched._free_request = MagicMock() # needs a KV manager; not what this covers
@@ -1448,7 +1453,7 @@ def test_queued_streaming_update_that_overflows_does_not_return_to_admission() -
assert finished is True
assert len(sched.waiting) == 0
assert len(sched.skipped_waiting) == 0
assert len(sched.kv_holding_waiting) == 0
assert session.status == RequestStatus.FINISHED_ERROR
assert session.resumable is False
assert "context_length_exceeded: " in sched._streaming_context_overflow[session.request_id][1]
@@ -1467,7 +1472,8 @@ def test_queued_streaming_update_that_fits_still_resumes_the_session() -> None:
finished = sched._handle_stopped_request(session)
assert finished is False
assert len(sched.waiting) == 1
assert list(sched.kv_holding_waiting) == [session]
assert session not in sched.deferred_waiting
assert session.status == RequestStatus.WAITING
assert not getattr(sched, "_streaming_context_overflow", {})
sched._free_request.assert_not_called()
@@ -1489,7 +1495,7 @@ def test_queued_stop_with_a_recorded_overflow_still_leaves_admission() -> None:
assert finished is True
assert len(sched.waiting) == 0
assert len(sched.skipped_waiting) == 0
assert len(sched.kv_holding_waiting) == 0
assert session.status == RequestStatus.FINISHED_ERROR
assert session.resumable is False
sched._free_request.assert_not_called()
@@ -1512,7 +1518,7 @@ def test_parked_stop_with_a_recorded_overflow_keeps_the_streaming_counter_balanc
finished = sched._handle_stopped_request(session)
assert finished is True
assert len(sched.skipped_waiting) == 0
assert len(sched.kv_holding_waiting) == 0
assert sched.num_waiting_for_streaming_input == 0
assert session.status == RequestStatus.FINISHED_ERROR
sched._free_request.assert_not_called()
@@ -1545,6 +1551,7 @@ def test_async_chunk_reserves_parked_slots_during_ar_admission(monkeypatch, nati
sched.running = []
sched._native_data_plane = native
sched.use_v2_model_runner = native
sched.max_num_active_reqs = 8
sched.max_num_running_reqs = 8
sched.input_coordinator = (
SimpleNamespace(_async_chunk=True, _waiting_for_chunk_running=[parked], restore_queues=lambda _w, _r: None)
@@ -1571,7 +1578,7 @@ def test_async_chunk_reserves_parked_slots_during_ar_admission(monkeypatch, nati
observed_limits: list[int] = []
def fake_schedule(self, _throttle_prefills=False):
observed_limits.append(self.max_num_running_reqs)
observed_limits.append(self.max_num_active_reqs)
return SimpleNamespace(scheduled_new_reqs=[])
monkeypatch.setattr(VLLMScheduler, "schedule", fake_schedule)
@@ -1579,4 +1586,4 @@ def test_async_chunk_reserves_parked_slots_during_ar_admission(monkeypatch, nati
sched.schedule()
assert observed_limits == [7 if native else 8]
assert sched.max_num_running_reqs == 8
assert sched.max_num_active_reqs == 8
@@ -79,16 +79,19 @@ def _make_scheduler(
else:
hf_config.voxcpm2_runtime_config = runtime_config
sched.vllm_config = SimpleNamespace(model_config=model_config)
# Upstream ``Scheduler.__init__`` always creates ``skipped_waiting``; the
# Upstream ``Scheduler.__init__`` always creates ``kv_holding_waiting``; the
# abort sweep at the top of ``schedule()`` reads it.
sched.skipped_waiting = _MockQueue()
sched.kv_holding_waiting = _MockQueue()
sched.deferred_waiting = set()
return sched
def test_voxcpm2_unified_decode_graph_defers_waiting_when_decode_ready() -> None:
@pytest.mark.parametrize("queue_name", ["waiting", "kv_holding_waiting"])
def test_voxcpm2_unified_decode_graph_defers_waiting_when_decode_ready(queue_name) -> None:
scheduler = _make_scheduler()
scheduler.running = [_MockRequest("decode")]
scheduler.waiting = _MockQueue([_MockRequest("prefill", status=RequestStatus.WAITING)])
scheduler.waiting = _MockQueue()
setattr(scheduler, queue_name, _MockQueue([_MockRequest("prefill", status=RequestStatus.WAITING)]))
assert scheduler._should_defer_waiting_for_unified_decode_graph()
@@ -144,12 +147,17 @@ def test_voxcpm2_unified_decode_graph_does_not_defer_without_cuda_graph(monkeypa
def test_unified_decode_graph_deferral_restores_waiting_queue(monkeypatch) -> None:
scheduler = _make_scheduler()
scheduler.max_num_active_reqs = 8
scheduler.max_num_running_reqs = 8
scheduler.running = [_MockRequest("decode")]
original_waiting_req = _MockRequest("waiting", status=RequestStatus.WAITING)
deferred_by_upstream = _MockRequest("deferred-by-upstream", status=RequestStatus.WAITING)
original_waiting = _MockQueue([original_waiting_req])
scheduler.waiting = original_waiting
kv_holder = _MockRequest("kv-holder", status=RequestStatus.WAITING)
original_kv_holding = _MockQueue([kv_holder])
scheduler.kv_holding_waiting = original_kv_holding
preempted_by_upstream = _MockRequest("preempted-by-upstream", status=RequestStatus.PREEMPTED)
scheduler.policy = "fcfs"
scheduler.chunk_transfer_adapter = None
scheduler.input_coordinator = None
@@ -160,7 +168,10 @@ def test_unified_decode_graph_deferral_restores_waiting_queue(monkeypatch) -> No
def fake_upstream_schedule(self, throttle_prefills: bool = False):
assert self.waiting is not original_waiting
assert not self.waiting
assert self.kv_holding_waiting is not original_kv_holding
assert not self.kv_holding_waiting
self.waiting.add_request(deferred_by_upstream)
self.kv_holding_waiting.add_request(preempted_by_upstream)
raise RuntimeError("stop before output wrapping")
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "schedule", fake_upstream_schedule)
@@ -169,3 +180,6 @@ def test_unified_decode_graph_deferral_restores_waiting_queue(monkeypatch) -> No
scheduler.schedule()
assert scheduler.waiting._items == [deferred_by_upstream, original_waiting_req]
assert scheduler.waiting is original_waiting
assert scheduler.kv_holding_waiting is original_kv_holding
assert scheduler.kv_holding_waiting._items == [preempted_by_upstream, kv_holder]
@@ -23,10 +23,12 @@ def _scheduler(monkeypatch, waiting, running=()):
scheduler.waiting = create_request_queue(scheduler.policy)
for request in waiting:
scheduler.waiting.add_request(request)
scheduler.skipped_waiting = create_request_queue(scheduler.policy)
scheduler.kv_holding_waiting = create_request_queue(scheduler.policy)
scheduler.deferred_waiting = set()
scheduler.running = list(running)
scheduler.requests = {request.request_id: request for request in [*waiting, *running]}
scheduler.max_model_len = 128
scheduler.max_num_active_reqs = 2
scheduler.max_num_running_reqs = 2
scheduler.max_num_scheduled_tokens = 256
scheduler.block_size = 16
@@ -52,10 +52,11 @@ class _SchedulerStub(OmniGenerationScheduler):
self.num_waiting_for_streaming_input = 0
self.log_stats = log_stats
self.chunk_transfer_adapter = _ChunkTransferAdapterStub()
self.skipped_waiting = _SkippedWaitingStub()
self.kv_holding_waiting = _SkippedWaitingStub()
self.deferred_waiting: set[Request] = set()
def _enqueue_waiting_request(self, session: Request) -> None:
raise AssertionError("unexpected enqueue for skipped_waiting miss")
raise AssertionError("unexpected enqueue for kv_holding_waiting miss")
class _AsyncChunkStopSchedulerStub(OmniGenerationScheduler):
@@ -134,7 +135,8 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None:
sched._handle_stopped_request.side_effect = rearm_resumable_request
sched.running = [session]
sched.waiting = MagicMock()
sched.skipped_waiting = MagicMock()
sched.kv_holding_waiting = MagicMock()
sched.deferred_waiting = set()
sched.structured_output_manager.accept_tokens.return_value = True
# Async scheduling can observe the segment boundary in the next schedule()
# before this model output is applied and defer the same request for finish.
@@ -206,7 +208,8 @@ def test_rejected_grammar_finishes_and_frees_generation_request(prompt_complete:
OmniSchedulerMixin._remove_stopped_requests_from_queues(sched, running, preempted)
)
sched.waiting = MagicMock()
sched.skipped_waiting = MagicMock()
sched.kv_holding_waiting = MagicMock()
sched.deferred_waiting = set()
sched.structured_output_manager.accept_tokens.return_value = False
sched._pending_finish_reqs = []
sched.recompute_kv_load_failures = False
@@ -350,6 +353,8 @@ class _RealignSchedulerStub(OmniSchedulerMixin):
self.requests = requests
self.running = running
self.waiting = waiting
self.kv_holding_waiting: list[Request] = []
self.deferred_waiting: set[Request] = set()
class TestRealignRequestStatusToQueues:
@@ -123,6 +123,7 @@ def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request
sched = MagicMock()
sched.max_num_scheduled_tokens = 8
sched.max_num_active_reqs = 8
sched.max_num_running_reqs = 8
sched._native_data_plane = False
sched._retains_state_across_chunks = False
@@ -135,6 +136,8 @@ def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request
sched.requests = {request.request_id: request}
sched.running = [request]
sched.waiting = [] # falsy: the waiting loop never runs
sched.kv_holding_waiting = []
sched.deferred_waiting = set()
sched.policy = SchedulingPolicy.FCFS
sched.chunk_transfer_adapter = None
sched.scheduler_config = MagicMock(enable_chunked_prefill=True)
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Integration tests for the post-finish defensive purge of
``self.running`` placed in
``OmniARScheduler.finish_requests`` /
@@ -52,6 +55,8 @@ def _make_scheduler(scheduler_cls, *, requests, running, waiting):
scheduler.requests = requests
scheduler.running = running
scheduler.waiting = waiting
scheduler.kv_holding_waiting = []
scheduler.deferred_waiting = set()
return scheduler
@@ -52,6 +52,8 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
scheduler.requests = {}
scheduler.running = []
scheduler.waiting = []
scheduler.kv_holding_waiting = []
scheduler.deferred_waiting = set()
def fake_finish_requests(self, request_ids, finished_status):
assert request_ids == ["req-a", "req-b"]
@@ -7,7 +7,10 @@ from typing import Any
import pytest
from vllm.config import SchedulerConfig, VllmConfig
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.engine import FinishReason
from vllm.v1.request import Request
from vllm_omni.config.model import OmniModelConfig
from vllm_omni.core.sched import omni_scheduler_mixin
@@ -18,7 +21,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _Scheduler(OmniSchedulerMixin):
pass
_is_blocked_waiting_status = staticmethod(Scheduler._is_blocked_waiting_status)
def test_async_chunk_adapter_initializes_for_stage_zero_sender_and_stage_one_receiver(monkeypatch):
@@ -120,7 +123,11 @@ def test_native_data_plane_uses_the_selected_input_protocol(async_chunk, stage_i
def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
calls: list[tuple[Any, ...]] = []
scheduler = _Scheduler()
scheduler.waiting = ["waiting"]
waiting = Request("waiting", [1], SamplingParams(max_tokens=1), pooling_params=None)
holding = Request("holding", [1, 2], SamplingParams(max_tokens=1), pooling_params=None)
holding.num_computed_tokens = 1
scheduler.waiting = [waiting]
scheduler.kv_holding_waiting = [holding]
scheduler.running = ["running"]
scheduler.requests = {"request": object()}
scheduler._consume_pending_connector_output = lambda mode: calls.append(("consume", mode))
@@ -137,16 +144,16 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
scheduler.chunk_transfer_adapter = SimpleNamespace(
receives_chunks=True,
process_pending_chunks=lambda waiting, running, scheduler_requests: calls.append(
("process", waiting, running, scheduler_requests)
("process", list(waiting), running, scheduler_requests)
),
restore_queues=lambda waiting, running, scheduler_requests: calls.append(
("restore-chunks", waiting, running, scheduler_requests)
("restore-chunks", list(waiting), running, scheduler_requests)
),
collect_timed_out_request_ids=_collect_timed_out,
collect_failed_send_request_ids=_collect_failed_sends,
)
scheduler.input_coordinator = SimpleNamespace(
restore_queues=lambda waiting, running: calls.append(("restore-full", waiting, running))
restore_queues=lambda waiting, running: calls.append(("restore-full", list(waiting), running))
)
scheduler._process_pending_omni_inputs("ar")
@@ -155,13 +162,13 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
assert calls == [
("consume", "ar"),
("timeouts",),
("process", scheduler.waiting, scheduler.running, scheduler.requests),
("process", [holding, waiting], scheduler.running, scheduler.requests),
# The chunk deadline runs after chunks are applied, so a chunk that
# arrived this cycle resets the clock before it is measured (R1.1).
("chunk-timeouts", omni_scheduler_mixin.DEFAULT_INPUT_WAIT_TIMEOUT_S),
("failed-sends",),
("restore-chunks", scheduler.waiting, scheduler.running, scheduler.requests),
("restore-full", scheduler.waiting, scheduler.running),
("restore-chunks", [holding, waiting], scheduler.running, scheduler.requests),
("restore-full", [holding, waiting], scheduler.running),
]
@@ -5,8 +5,13 @@
from types import SimpleNamespace
import pytest
from vllm import SamplingParams
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
from vllm.v1.request import Request, RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin
from vllm_omni.core.sched.omni_scheduling_coordinator import OmniSchedulingCoordinator
from vllm_omni.outputs import OmniConnectorOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -15,6 +20,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_ready_inbox_coalesces_live_events_and_drops_cancelled(mocker):
scheduler = OmniSchedulerMixin()
scheduler.requests, scheduler.waiting, scheduler.running = {"r": object()}, [], []
scheduler.kv_holding_waiting = []
scheduler._is_blocked_waiting_status = OmniARScheduler._is_blocked_waiting_status
coordinator = SimpleNamespace(
_async_chunk=True, update_request_metadata=mocker.Mock(), process_pending_chunks=mocker.Mock()
)
@@ -32,7 +39,92 @@ def test_ready_inbox_coalesces_live_events_and_drops_cancelled(mocker):
coordinator.update_request_metadata.assert_called_once_with(
scheduler.requests, {"r": {"decode_token_end": 2}}, model_mode="ar"
)
coordinator.process_pending_chunks.assert_called_once_with([], [], {"r"}, {"r"})
coordinator.process_pending_chunks.assert_called_once_with(mocker.ANY, [], {"r"}, {"r"})
assert list(coordinator.process_pending_chunks.call_args.args[0]) == []
scheduler._consume_pending_connector_output(model_mode="ar")
assert coordinator.update_request_metadata.call_count == 1
coordinator.process_pending_chunks.assert_called_with([], [], set(), set())
coordinator.process_pending_chunks.assert_called_with(mocker.ANY, [], set(), set())
assert list(coordinator.process_pending_chunks.call_args.args[0]) == []
@pytest.mark.parametrize("policy", [SchedulingPolicy.FCFS, SchedulingPolicy.PRIORITY])
@pytest.mark.parametrize("async_chunk", [False, True])
def test_native_input_gate_parks_and_restores_kv_holders(policy, async_chunk):
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler.waiting = create_request_queue(policy)
scheduler.kv_holding_waiting = create_request_queue(policy)
scheduler.deferred_waiting = set()
scheduler.running = []
scheduler.chunk_transfer_adapter = None
scheduler.input_coordinator = OmniSchedulingCoordinator(
stage_id=1, scheduler_max_num_seqs=2, async_chunk=async_chunk
)
scheduler._init_omni_connector_output_inbox()
holder = Request("holder", [1, 2, 3], SamplingParams(max_tokens=1), pooling_params=None)
holder.num_computed_tokens = 2
fresh = Request("fresh", [1], SamplingParams(max_tokens=1), pooling_params=None)
scheduler.requests = {request.request_id: request for request in (holder, fresh)}
scheduler._enqueue_waiting_request(fresh)
scheduler._enqueue_waiting_request(holder)
scheduler._consume_pending_connector_output(model_mode="ar")
assert not scheduler.waiting and not scheduler.kv_holding_waiting
expected_status = RequestStatus.WAITING_FOR_CHUNK if async_chunk else RequestStatus.WAITING_FOR_INPUT
assert holder.status == fresh.status == expected_status
coordinator = scheduler.input_coordinator
registrations = coordinator.pending_chunk_registrations if async_chunk else coordinator.pending_input_registrations
assert [entry.request_id for entry in registrations] == ["holder", "fresh"]
scheduler._restore_omni_wait_queues()
assert list(scheduler.kv_holding_waiting) == [holder]
assert list(scheduler.waiting) == [fresh]
ready_output = (
OmniConnectorOutput(chunk_ready_req_ids={"holder"})
if async_chunk
else OmniConnectorOutput(stage_recv_req_ids={"holder"})
)
scheduler.enqueue_omni_connector_output(ready_output)
scheduler._consume_pending_connector_output(model_mode="ar")
assert holder.status == RequestStatus.WAITING
assert fresh.status == expected_status
assert list(scheduler.kv_holding_waiting) == [holder]
assert not scheduler.waiting
scheduler._restore_omni_wait_queues()
assert list(scheduler.kv_holding_waiting) == [holder]
assert list(scheduler.waiting) == [fresh]
@pytest.mark.parametrize("async_chunk", [False, True])
@pytest.mark.parametrize(
"status",
[
RequestStatus.WAITING_FOR_STREAMING_REQ,
RequestStatus.WAITING_FOR_REMOTE_KVS,
RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR,
],
)
def test_native_input_gate_preserves_upstream_blocked_waits(async_chunk, status):
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler.waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.kv_holding_waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.deferred_waiting = set()
scheduler.running = []
request = Request("blocked", [1, 2, 3], SamplingParams(max_tokens=1), pooling_params=None)
request.status = status
request.num_computed_tokens = 2
scheduler.requests = {request.request_id: request}
scheduler._enqueue_waiting_request(request)
scheduler.input_coordinator = OmniSchedulingCoordinator(
stage_id=1, scheduler_max_num_seqs=1, async_chunk=async_chunk
)
scheduler._init_omni_connector_output_inbox()
scheduler._consume_pending_connector_output(model_mode="ar")
assert request.status == status
assert list(scheduler.kv_holding_waiting) == [request]
assert scheduler.deferred_waiting == {request}
assert not scheduler.input_coordinator.pending_chunk_registrations
assert not scheduler.input_coordinator.pending_input_registrations
@@ -59,7 +59,8 @@ def _make_scheduler(scheduler_cls, *, requests, running, waiting, counter, skipp
scheduler.requests = requests
scheduler.running = running
scheduler.waiting = waiting
scheduler.skipped_waiting = skipped if skipped is not None else []
scheduler.kv_holding_waiting = skipped if skipped is not None else []
scheduler.deferred_waiting = set()
scheduler.num_waiting_for_streaming_input = counter
return scheduler
@@ -97,8 +98,8 @@ def test_finish_requests_clears_counter_left_by_a_stomped_status(
assert scheduler.num_waiting_for_streaming_input == 0
def test_resync_counts_parked_requests_in_skipped_waiting() -> None:
"""``get_num_unfinished_requests`` sums ``waiting`` and ``skipped_waiting``,
def test_resync_counts_parked_requests_in_kv_holding_waiting() -> None:
"""``get_num_unfinished_requests`` sums ``waiting`` and ``kv_holding_waiting``,
so the resync has to look at both or it under-counts and reintroduces the
spin the counter exists to prevent."""
parked = _StubRequest("req-skipped", RequestStatus.WAITING_FOR_STREAMING_REQ)
@@ -128,9 +129,11 @@ def test_ar_schedule_resyncs_before_delegating_upstream(monkeypatch: pytest.Monk
scheduler.requests = {parked.request_id: parked}
scheduler.running = [parked]
scheduler.waiting = []
scheduler.skipped_waiting = []
scheduler.kv_holding_waiting = []
scheduler.deferred_waiting = set()
scheduler.num_waiting_for_streaming_input = 1 # leaked by the status rewrite
scheduler.policy = "fcfs"
scheduler.max_num_active_reqs = 8
scheduler.max_num_running_reqs = 8
seen: dict[str, int] = {}
@@ -161,12 +164,13 @@ def test_generation_scheduler_sweep_removes_aborted_requests_from_all_queues() -
scheduler = OmniGenerationScheduler.__new__(OmniGenerationScheduler)
scheduler.waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.waiting.add_request(aborted)
scheduler.skipped_waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.skipped_waiting.add_request(aborted)
scheduler.kv_holding_waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.deferred_waiting = set()
scheduler.kv_holding_waiting.add_request(aborted)
scheduler.running = [aborted]
scheduler._drop_aborted_queued_requests()
assert list(scheduler.waiting) == []
assert list(scheduler.skipped_waiting) == []
assert list(scheduler.kv_holding_waiting) == []
assert scheduler.running == []