[Rebase] Rebase to vllm 0.28.0 (#6606)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zhou TaichangandClaude Opus 5 authored and GitHub committed 2026-08-27 06:12:55 +00:00
1 parent da4a08b65b
commit 651b03225b
42 files changed
+931 -121

No files matched your search

@@ -100,6 +100,7 @@ def _make_drain_sched(session: Request) -> MagicMock:
sched.pending_stop_after_extraction = set()
sched.connector = None
sched.kv_cache_manager.take_events.return_value = None
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
sched.finished_req_ids_dict = {}
sched.make_stats.return_value = None
return sched
@@ -93,6 +93,7 @@ def _run_resumable_segment_stop(
sched.pending_stop_after_extraction = set()
sched.connector = None
sched.kv_cache_manager.take_events.return_value = None
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
sched.finished_req_ids_dict = {}
sched.make_stats.return_value = None
@@ -227,6 +228,7 @@ def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> Non
sched.pending_stop_after_extraction = set()
sched.connector = None
sched.kv_cache_manager.take_events.return_value = None
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
sched.finished_req_ids_dict = {}
sched.make_stats.return_value = None
@@ -138,6 +138,7 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None:
sched.recompute_kv_load_failures = False
sched.connector = None
sched.kv_cache_manager.take_events.return_value = None
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
sched.finished_req_ids_dict = {}
sched.make_stats.return_value = None
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""v0.28 deferred KV-block free fence in the Omni schedulers (#6606 review).
Upstream v0.28 (defer_block_free) advances ``sched_step_seq`` for every
non-empty scheduled step and, at the top of ``update_from_output()``,
advances ``processed_step_seq`` and drains the deferred frees. The Omni
schedulers each carried only half of that contract:
- ``OmniARScheduler.schedule()`` delegates to ``super().schedule()`` (fence
advances) but its reimplemented ``update_from_output()`` never advanced
``processed_step_seq`` nor drained -> deferred blocks accumulate without
bound under async/PP KV-consumer configurations.
- ``OmniGenerationScheduler`` reimplements both: the fast-path
``schedule()`` never advanced ``sched_step_seq`` (so finished requests
freed blocks an in-flight GPU step could still be writing) and its
``update_from_output()`` never drained.
These tests pin the restored contract at the call boundary; the drain and
free logic itself is upstream-tested.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
# before Request / RequestStatus are bound in this module.
# isort: off
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.request_queue import SchedulingPolicy
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
# isort: on
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_update_sched(scheduler_cls) -> MagicMock:
"""MagicMock scheduler driving the real ``update_from_output`` with an
empty batch; only the fence state carries real values."""
sched = MagicMock()
sched.requests = {}
sched.perf_metrics = None
sched.chunk_transfer_adapter = None
sched.defer_block_free = True
sched.processed_step_seq = 0
return sched
def _empty_frame(*, total_num_scheduled_tokens: int):
scheduler_output = MagicMock(spec=SchedulerOutput)
scheduler_output.num_scheduled_tokens = {}
scheduler_output.total_num_scheduled_tokens = total_num_scheduled_tokens
model_runner_output = MagicMock(spec=ModelRunnerOutput)
model_runner_output.sampled_token_ids = []
model_runner_output.logprobs = None
model_runner_output.prompt_logprobs_dict = {}
model_runner_output.pooler_output = None
model_runner_output.num_nans_in_logits = None
model_runner_output.kv_connector_output = None
model_runner_output.cudagraph_stats = None
return scheduler_output, model_runner_output
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
def test_update_from_output_advances_fence_and_drains(scheduler_cls) -> None:
sched = _make_update_sched(scheduler_cls)
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=3)
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
assert sched.processed_step_seq == 1
sched._drain_deferred_frees.assert_called_once_with()
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
def test_update_from_output_zero_token_step_leaves_fence(scheduler_cls) -> None:
"""0-token steps do not advance the fence upstream (their seq was never
advanced in schedule()); draining on them would run ahead of the GPU."""
sched = _make_update_sched(scheduler_cls)
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=0)
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
assert sched.processed_step_seq == 0
sched._drain_deferred_frees.assert_not_called()
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
def test_update_from_output_defer_disabled_leaves_fence(scheduler_cls) -> None:
sched = _make_update_sched(scheduler_cls)
sched.defer_block_free = False
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=3)
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
assert sched.processed_step_seq == 0
sched._drain_deferred_frees.assert_not_called()
def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request]:
"""MagicMock scheduler that drives the real generation fast-path
``schedule()`` over one running one-shot request."""
request = Request(
request_id="req-fence",
prompt_token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=4),
pooling_params=None,
arrival_time=100.0,
block_hasher=None,
)
request.status = RequestStatus.RUNNING
sched = MagicMock()
sched.max_num_scheduled_tokens = 8
sched._pause_state = PauseState.UNPAUSED
sched.requests = {request.request_id: request}
sched.running = [request]
sched.waiting = [] # falsy: the waiting loop never runs
sched.policy = SchedulingPolicy.FCFS
sched.chunk_transfer_adapter = None
sched.scheduler_config = MagicMock(enable_chunked_prefill=True)
sched.num_lookahead_tokens = 0
sched.log_stats = False
sched.kv_cache_config = MagicMock(kv_cache_groups=[])
sched.use_v2_model_runner = False
sched.prev_step_scheduled_req_ids = set()
sched.needs_kv_cache_zeroing = False
sched.connector = None
sched.ec_connector = None
sched.defer_block_free = defer_block_free
sched.sched_step_seq = 0
# last_sched_seq stamping order: _update_after_schedule must observe the
# already-advanced fence value, as upstream schedule() guarantees.
sched.seq_at_update_after_schedule = None
def _capture(_scheduler_output):
sched.seq_at_update_after_schedule = sched.sched_step_seq
sched._update_after_schedule.side_effect = _capture
return sched, request
def test_fast_path_schedule_advances_fence_before_update_after_schedule() -> None:
sched, request = _make_fast_path_sched(defer_block_free=True)
OmniGenerationScheduler.schedule(sched)
assert sched.sched_step_seq == 1
assert sched.seq_at_update_after_schedule == 1
def test_fast_path_schedule_defer_disabled_leaves_fence() -> None:
sched, request = _make_fast_path_sched(defer_block_free=False)
OmniGenerationScheduler.schedule(sched)
assert sched.sched_step_seq == 0
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""v0.28 EC request-finish contract in the Omni schedulers (#6606 review).
Upstream v0.28 ``_free_request()`` fires ``ec_connector.request_finished()``
BEFORE the encoder cache is freed (so the connector can inspect per-request
state), honors its delayed-free result, and returns the generated
``ec_transfer_params``; ``update_from_output()`` then carries them into the
``EngineCoreOutput``. ``OmniARScheduler`` previously skipped the hook and
always returned ``None``; both Omni schedulers discarded the second tuple
item, so a remote encoder-cache handle could never reach the frontend.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
# before Request / RequestStatus are bound in this module.
# isort: off
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
# isort: on
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
EC_PARAMS = {"remote_encoder_cache_handle": "ec-1"}
def _make_free_request_sched(*, ec_connector) -> tuple[OmniARScheduler, MagicMock]:
"""Minimal real-`_free_request` harness; `order` records the EC hook
firing relative to the encoder-cache free."""
order = MagicMock()
sched = OmniARScheduler.__new__(OmniARScheduler)
sched._omits_kv_transfer_cache = {}
sched._connector_finished = lambda request: (False, None)
sched.ec_connector = ec_connector
if ec_connector is not None:
order.attach_mock(ec_connector.request_finished, "request_finished")
sched.encoder_cache_manager = MagicMock()
order.attach_mock(sched.encoder_cache_manager.free, "encoder_cache_free")
sched.finished_req_ids = set()
sched._new_prompt_len_snapshot = {}
sched.finished_req_ids_dict = None
sched._should_transfer_kv_for_request = lambda req_id: False
sched._free_blocks = MagicMock()
sched._free_input_coordinator_request = MagicMock()
sched.chunk_transfer_adapter = None
return sched, order
class _FakeFinishedRequest:
def __init__(self, request_id: str) -> None:
self.request_id = request_id
def is_finished(self) -> bool:
return True
def test_free_request_fires_ec_hook_before_encoder_cache_free() -> None:
ec_connector = MagicMock()
ec_connector.request_finished.return_value = (False, EC_PARAMS)
sched, order = _make_free_request_sched(ec_connector=ec_connector)
request = _FakeFinishedRequest("req-ec")
kv_params, ec_params = sched._free_request(request)
assert ec_params == EC_PARAMS
ec_connector.request_finished.assert_called_once_with(request)
hook_calls = [name for name, *_ in order.mock_calls]
assert hook_calls.index("request_finished") < hook_calls.index("encoder_cache_free")
sched._free_blocks.assert_called_once_with(request)
def test_free_request_honors_ec_delayed_free() -> None:
ec_connector = MagicMock()
ec_connector.request_finished.return_value = (True, EC_PARAMS)
sched, _ = _make_free_request_sched(ec_connector=ec_connector)
request = _FakeFinishedRequest("req-ec-delay")
kv_params, ec_params = sched._free_request(request)
assert ec_params == EC_PARAMS
sched._free_blocks.assert_not_called()
def test_free_request_without_ec_connector_returns_none() -> None:
sched, _ = _make_free_request_sched(ec_connector=None)
request = _FakeFinishedRequest("req-no-ec")
kv_params, ec_params = sched._free_request(request)
assert ec_params is None
sched._free_blocks.assert_called_once_with(request)
def _make_finishing_request() -> Request:
request = Request(
request_id="req-ec-finish",
prompt_token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=4),
pooling_params=None,
arrival_time=100.0,
block_hasher=None,
)
request.status = RequestStatus.RUNNING
return request
def _make_finish_sched(request: Request) -> MagicMock:
"""MagicMock scheduler driving the real ``update_from_output`` through a
finishing request; ``_free_request`` reports EC transfer params."""
sched = MagicMock()
sched.requests = {request.request_id: request}
sched.perf_metrics = None
sched.defer_block_free = False
sched.structured_output_manager.should_advance.return_value = False
sched._process_kv_transfer_trigger.return_value = False
sched._maybe_decode_pooling_output.return_value = None
sched._handle_stopped_request.return_value = True
sched._free_request.return_value = (None, EC_PARAMS)
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
sched.chunk_transfer_adapter = None
sched.finished_req_ids_dict = None
sched._new_prompt_len_snapshot = {}
return sched
def test_ar_update_from_output_carries_ec_params_to_engine_output() -> None:
request = _make_finishing_request()
sched = _make_finish_sched(request)
sched._update_request_with_output.return_value = ([42], True)
engine_core_outputs = OmniARScheduler.update_from_output(sched, *_finish_frame_parts(request))
(eco,) = engine_core_outputs[request.client_index].outputs
assert eco.ec_transfer_params == EC_PARAMS
sched._free_request.assert_called_once_with(request)
def _finish_frame_parts(request: Request):
scheduler_output = MagicMock(spec=SchedulerOutput)
scheduler_output.num_scheduled_tokens = {request.request_id: 1}
scheduler_output.total_num_scheduled_tokens = 1
scheduler_output.scheduled_spec_decode_tokens = {}
scheduler_output.num_invalid_spec_tokens = 0
model_runner_output = MagicMock(spec=ModelRunnerOutput)
model_runner_output.sampled_token_ids = [[42]]
model_runner_output.logprobs = None
model_runner_output.prompt_logprobs_dict = {}
model_runner_output.pooler_output = None
model_runner_output.num_nans_in_logits = None
model_runner_output.kv_connector_output = None
model_runner_output.cudagraph_stats = None
model_runner_output.req_id_to_index = {request.request_id: 0}
model_runner_output.routed_experts = None
return scheduler_output, model_runner_output
def test_generation_update_from_output_carries_ec_params_to_engine_output() -> None:
request = _make_finishing_request()
# One-shot generation finish: the whole prompt is computed.
request.num_computed_tokens = len(request.prompt_token_ids)
sched = _make_finish_sched(request)
engine_core_outputs = OmniGenerationScheduler.update_from_output(sched, *_finish_frame_parts(request))
(eco,) = engine_core_outputs[request.client_index].outputs
assert eco.ec_transfer_params == EC_PARAMS
sched._free_request.assert_called_once_with(request)
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""v0.28 prefill-stats finalization in the Omni schedulers (#6606 review).
Upstream v0.28 finalizes prefill statistics with
``PrefillStats.finalize(kv_cache_manager.estimate_cached_tokens(request))``
before emitting them, which computes ``num_cache_creation_tokens`` (tokens
computed and written to the prefix cache). Both Omni schedulers previously
emitted ``take_prefill_stats()`` raw, so creation usage was always 0.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
# before Request / RequestStatus are bound in this module.
# isort: off
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
# isort: on
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_request() -> Request:
request = Request(
request_id="req-prefill-stats",
prompt_token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=4),
pooling_params=None,
arrival_time=100.0,
block_hasher=None,
)
request.status = RequestStatus.RUNNING
# A scheduled prefill: 3 prompt tokens, 1 from the local prefix cache.
request.prefill_stats.set(
num_prompt_tokens=3,
num_local_cached_tokens=1,
num_external_cached_tokens=0,
)
return request
def _make_sched(request: Request) -> MagicMock:
sched = MagicMock()
sched.requests = {request.request_id: request}
sched.perf_metrics = None
sched.defer_block_free = False
sched.structured_output_manager.should_advance.return_value = False
sched._process_kv_transfer_trigger.return_value = False
sched._maybe_decode_pooling_output.return_value = None
sched._handle_stopped_request.return_value = True
# The whole 3-token prompt ends up cached: 3 - 1 pre-cached = 2 created.
sched.kv_cache_manager.estimate_cached_tokens.return_value = 3
def _free_and_invalidate(req):
# Lifecycle guard: freeing releases the KV blocks, after which the
# manager reports 0 cached tokens. If finalize ran after
# _free_request (the reviewed bug), creation count would compute
# against 0 and the assertions below would see 0, not 2.
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
return (None, None)
sched._free_request.side_effect = _free_and_invalidate
sched.chunk_transfer_adapter = None
sched.finished_req_ids_dict = None
sched._new_prompt_len_snapshot = {}
return sched
def _frame(request: Request):
scheduler_output = MagicMock(spec=SchedulerOutput)
scheduler_output.num_scheduled_tokens = {request.request_id: 1}
scheduler_output.total_num_scheduled_tokens = 1
scheduler_output.scheduled_spec_decode_tokens = {}
scheduler_output.num_invalid_spec_tokens = 0
model_runner_output = MagicMock(spec=ModelRunnerOutput)
model_runner_output.sampled_token_ids = [[42]]
model_runner_output.logprobs = None
model_runner_output.prompt_logprobs_dict = {}
model_runner_output.pooler_output = None
model_runner_output.num_nans_in_logits = None
model_runner_output.kv_connector_output = None
model_runner_output.cudagraph_stats = None
model_runner_output.req_id_to_index = {request.request_id: 0}
model_runner_output.routed_experts = None
return scheduler_output, model_runner_output
def test_ar_scheduler_finalizes_prefill_stats_before_emitting() -> None:
request = _make_request()
sched = _make_sched(request)
sched._update_request_with_output.return_value = ([42], True)
engine_core_outputs = OmniARScheduler.update_from_output(sched, *_frame(request))
(eco,) = engine_core_outputs[request.client_index].outputs
assert eco.prefill_stats is not None
assert eco.prefill_stats.num_cached_tokens == 1
assert eco.prefill_stats.num_cache_creation_tokens == 2
sched.kv_cache_manager.estimate_cached_tokens.assert_called_once_with(request)
def test_generation_scheduler_finalizes_prefill_stats_before_emitting() -> None:
request = _make_request()
request.num_computed_tokens = len(request.prompt_token_ids)
sched = _make_sched(request)
engine_core_outputs = OmniGenerationScheduler.update_from_output(sched, *_frame(request))
(eco,) = engine_core_outputs[request.client_index].outputs
assert eco.prefill_stats is not None
assert eco.prefill_stats.num_cached_tokens == 1
assert eco.prefill_stats.num_cache_creation_tokens == 2
sched.kv_cache_manager.estimate_cached_tokens.assert_called_once_with(request)