[Rebase] Rebase to vllm 0.28.0 (#6606)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
da4a08b65b
commit
651b03225b
42 files changed
+931
-121
No files matched your search
@@ -100,6 +100,7 @@ def _make_drain_sched(session: Request) -> MagicMock:
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
return sched
|
||||
|
||||
@@ -93,6 +93,7 @@ def _run_resumable_segment_stop(
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
|
||||
@@ -227,6 +228,7 @@ def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> Non
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
|
||||
|
||||
@@ -138,6 +138,7 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None:
|
||||
sched.recompute_kv_load_failures = False
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""v0.28 deferred KV-block free fence in the Omni schedulers (#6606 review).
|
||||
|
||||
Upstream v0.28 (defer_block_free) advances ``sched_step_seq`` for every
|
||||
non-empty scheduled step and, at the top of ``update_from_output()``,
|
||||
advances ``processed_step_seq`` and drains the deferred frees. The Omni
|
||||
schedulers each carried only half of that contract:
|
||||
|
||||
- ``OmniARScheduler.schedule()`` delegates to ``super().schedule()`` (fence
|
||||
advances) but its reimplemented ``update_from_output()`` never advanced
|
||||
``processed_step_seq`` nor drained -> deferred blocks accumulate without
|
||||
bound under async/PP KV-consumer configurations.
|
||||
- ``OmniGenerationScheduler`` reimplements both: the fast-path
|
||||
``schedule()`` never advanced ``sched_step_seq`` (so finished requests
|
||||
freed blocks an in-flight GPU step could still be writing) and its
|
||||
``update_from_output()`` never drained.
|
||||
|
||||
These tests pin the restored contract at the call boundary; the drain and
|
||||
free logic itself is upstream-tested.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
|
||||
# before Request / RequestStatus are bound in this module.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.request_queue import SchedulingPolicy
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_update_sched(scheduler_cls) -> MagicMock:
|
||||
"""MagicMock scheduler driving the real ``update_from_output`` with an
|
||||
empty batch; only the fence state carries real values."""
|
||||
sched = MagicMock()
|
||||
sched.requests = {}
|
||||
sched.perf_metrics = None
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.defer_block_free = True
|
||||
sched.processed_step_seq = 0
|
||||
return sched
|
||||
|
||||
|
||||
def _empty_frame(*, total_num_scheduled_tokens: int):
|
||||
scheduler_output = MagicMock(spec=SchedulerOutput)
|
||||
scheduler_output.num_scheduled_tokens = {}
|
||||
scheduler_output.total_num_scheduled_tokens = total_num_scheduled_tokens
|
||||
model_runner_output = MagicMock(spec=ModelRunnerOutput)
|
||||
model_runner_output.sampled_token_ids = []
|
||||
model_runner_output.logprobs = None
|
||||
model_runner_output.prompt_logprobs_dict = {}
|
||||
model_runner_output.pooler_output = None
|
||||
model_runner_output.num_nans_in_logits = None
|
||||
model_runner_output.kv_connector_output = None
|
||||
model_runner_output.cudagraph_stats = None
|
||||
return scheduler_output, model_runner_output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
|
||||
def test_update_from_output_advances_fence_and_drains(scheduler_cls) -> None:
|
||||
sched = _make_update_sched(scheduler_cls)
|
||||
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=3)
|
||||
|
||||
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
assert sched.processed_step_seq == 1
|
||||
sched._drain_deferred_frees.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
|
||||
def test_update_from_output_zero_token_step_leaves_fence(scheduler_cls) -> None:
|
||||
"""0-token steps do not advance the fence upstream (their seq was never
|
||||
advanced in schedule()); draining on them would run ahead of the GPU."""
|
||||
sched = _make_update_sched(scheduler_cls)
|
||||
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=0)
|
||||
|
||||
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
assert sched.processed_step_seq == 0
|
||||
sched._drain_deferred_frees.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
|
||||
def test_update_from_output_defer_disabled_leaves_fence(scheduler_cls) -> None:
|
||||
sched = _make_update_sched(scheduler_cls)
|
||||
sched.defer_block_free = False
|
||||
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=3)
|
||||
|
||||
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
assert sched.processed_step_seq == 0
|
||||
sched._drain_deferred_frees.assert_not_called()
|
||||
|
||||
|
||||
def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request]:
|
||||
"""MagicMock scheduler that drives the real generation fast-path
|
||||
``schedule()`` over one running one-shot request."""
|
||||
request = Request(
|
||||
request_id="req-fence",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=4),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
request.status = RequestStatus.RUNNING
|
||||
|
||||
sched = MagicMock()
|
||||
sched.max_num_scheduled_tokens = 8
|
||||
sched._pause_state = PauseState.UNPAUSED
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.running = [request]
|
||||
sched.waiting = [] # falsy: the waiting loop never runs
|
||||
sched.policy = SchedulingPolicy.FCFS
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.scheduler_config = MagicMock(enable_chunked_prefill=True)
|
||||
sched.num_lookahead_tokens = 0
|
||||
sched.log_stats = False
|
||||
sched.kv_cache_config = MagicMock(kv_cache_groups=[])
|
||||
sched.use_v2_model_runner = False
|
||||
sched.prev_step_scheduled_req_ids = set()
|
||||
sched.needs_kv_cache_zeroing = False
|
||||
sched.connector = None
|
||||
sched.ec_connector = None
|
||||
sched.defer_block_free = defer_block_free
|
||||
sched.sched_step_seq = 0
|
||||
# last_sched_seq stamping order: _update_after_schedule must observe the
|
||||
# already-advanced fence value, as upstream schedule() guarantees.
|
||||
sched.seq_at_update_after_schedule = None
|
||||
|
||||
def _capture(_scheduler_output):
|
||||
sched.seq_at_update_after_schedule = sched.sched_step_seq
|
||||
|
||||
sched._update_after_schedule.side_effect = _capture
|
||||
return sched, request
|
||||
|
||||
|
||||
def test_fast_path_schedule_advances_fence_before_update_after_schedule() -> None:
|
||||
sched, request = _make_fast_path_sched(defer_block_free=True)
|
||||
|
||||
OmniGenerationScheduler.schedule(sched)
|
||||
|
||||
assert sched.sched_step_seq == 1
|
||||
assert sched.seq_at_update_after_schedule == 1
|
||||
|
||||
|
||||
def test_fast_path_schedule_defer_disabled_leaves_fence() -> None:
|
||||
sched, request = _make_fast_path_sched(defer_block_free=False)
|
||||
|
||||
OmniGenerationScheduler.schedule(sched)
|
||||
|
||||
assert sched.sched_step_seq == 0
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""v0.28 EC request-finish contract in the Omni schedulers (#6606 review).
|
||||
|
||||
Upstream v0.28 ``_free_request()`` fires ``ec_connector.request_finished()``
|
||||
BEFORE the encoder cache is freed (so the connector can inspect per-request
|
||||
state), honors its delayed-free result, and returns the generated
|
||||
``ec_transfer_params``; ``update_from_output()`` then carries them into the
|
||||
``EngineCoreOutput``. ``OmniARScheduler`` previously skipped the hook and
|
||||
always returned ``None``; both Omni schedulers discarded the second tuple
|
||||
item, so a remote encoder-cache handle could never reach the frontend.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
|
||||
# before Request / RequestStatus are bound in this module.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
EC_PARAMS = {"remote_encoder_cache_handle": "ec-1"}
|
||||
|
||||
|
||||
def _make_free_request_sched(*, ec_connector) -> tuple[OmniARScheduler, MagicMock]:
|
||||
"""Minimal real-`_free_request` harness; `order` records the EC hook
|
||||
firing relative to the encoder-cache free."""
|
||||
order = MagicMock()
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched._omits_kv_transfer_cache = {}
|
||||
sched._connector_finished = lambda request: (False, None)
|
||||
sched.ec_connector = ec_connector
|
||||
if ec_connector is not None:
|
||||
order.attach_mock(ec_connector.request_finished, "request_finished")
|
||||
sched.encoder_cache_manager = MagicMock()
|
||||
order.attach_mock(sched.encoder_cache_manager.free, "encoder_cache_free")
|
||||
sched.finished_req_ids = set()
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
sched.finished_req_ids_dict = None
|
||||
sched._should_transfer_kv_for_request = lambda req_id: False
|
||||
sched._free_blocks = MagicMock()
|
||||
sched._free_input_coordinator_request = MagicMock()
|
||||
sched.chunk_transfer_adapter = None
|
||||
return sched, order
|
||||
|
||||
|
||||
class _FakeFinishedRequest:
|
||||
def __init__(self, request_id: str) -> None:
|
||||
self.request_id = request_id
|
||||
|
||||
def is_finished(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def test_free_request_fires_ec_hook_before_encoder_cache_free() -> None:
|
||||
ec_connector = MagicMock()
|
||||
ec_connector.request_finished.return_value = (False, EC_PARAMS)
|
||||
sched, order = _make_free_request_sched(ec_connector=ec_connector)
|
||||
request = _FakeFinishedRequest("req-ec")
|
||||
|
||||
kv_params, ec_params = sched._free_request(request)
|
||||
|
||||
assert ec_params == EC_PARAMS
|
||||
ec_connector.request_finished.assert_called_once_with(request)
|
||||
hook_calls = [name for name, *_ in order.mock_calls]
|
||||
assert hook_calls.index("request_finished") < hook_calls.index("encoder_cache_free")
|
||||
sched._free_blocks.assert_called_once_with(request)
|
||||
|
||||
|
||||
def test_free_request_honors_ec_delayed_free() -> None:
|
||||
ec_connector = MagicMock()
|
||||
ec_connector.request_finished.return_value = (True, EC_PARAMS)
|
||||
sched, _ = _make_free_request_sched(ec_connector=ec_connector)
|
||||
request = _FakeFinishedRequest("req-ec-delay")
|
||||
|
||||
kv_params, ec_params = sched._free_request(request)
|
||||
|
||||
assert ec_params == EC_PARAMS
|
||||
sched._free_blocks.assert_not_called()
|
||||
|
||||
|
||||
def test_free_request_without_ec_connector_returns_none() -> None:
|
||||
sched, _ = _make_free_request_sched(ec_connector=None)
|
||||
request = _FakeFinishedRequest("req-no-ec")
|
||||
|
||||
kv_params, ec_params = sched._free_request(request)
|
||||
|
||||
assert ec_params is None
|
||||
sched._free_blocks.assert_called_once_with(request)
|
||||
|
||||
|
||||
def _make_finishing_request() -> Request:
|
||||
request = Request(
|
||||
request_id="req-ec-finish",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=4),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
request.status = RequestStatus.RUNNING
|
||||
return request
|
||||
|
||||
|
||||
def _make_finish_sched(request: Request) -> MagicMock:
|
||||
"""MagicMock scheduler driving the real ``update_from_output`` through a
|
||||
finishing request; ``_free_request`` reports EC transfer params."""
|
||||
sched = MagicMock()
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.perf_metrics = None
|
||||
sched.defer_block_free = False
|
||||
sched.structured_output_manager.should_advance.return_value = False
|
||||
sched._process_kv_transfer_trigger.return_value = False
|
||||
sched._maybe_decode_pooling_output.return_value = None
|
||||
sched._handle_stopped_request.return_value = True
|
||||
sched._free_request.return_value = (None, EC_PARAMS)
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.finished_req_ids_dict = None
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
return sched
|
||||
|
||||
|
||||
def test_ar_update_from_output_carries_ec_params_to_engine_output() -> None:
|
||||
request = _make_finishing_request()
|
||||
sched = _make_finish_sched(request)
|
||||
sched._update_request_with_output.return_value = ([42], True)
|
||||
|
||||
engine_core_outputs = OmniARScheduler.update_from_output(sched, *_finish_frame_parts(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.ec_transfer_params == EC_PARAMS
|
||||
sched._free_request.assert_called_once_with(request)
|
||||
|
||||
|
||||
def _finish_frame_parts(request: Request):
|
||||
scheduler_output = MagicMock(spec=SchedulerOutput)
|
||||
scheduler_output.num_scheduled_tokens = {request.request_id: 1}
|
||||
scheduler_output.total_num_scheduled_tokens = 1
|
||||
scheduler_output.scheduled_spec_decode_tokens = {}
|
||||
scheduler_output.num_invalid_spec_tokens = 0
|
||||
|
||||
model_runner_output = MagicMock(spec=ModelRunnerOutput)
|
||||
model_runner_output.sampled_token_ids = [[42]]
|
||||
model_runner_output.logprobs = None
|
||||
model_runner_output.prompt_logprobs_dict = {}
|
||||
model_runner_output.pooler_output = None
|
||||
model_runner_output.num_nans_in_logits = None
|
||||
model_runner_output.kv_connector_output = None
|
||||
model_runner_output.cudagraph_stats = None
|
||||
model_runner_output.req_id_to_index = {request.request_id: 0}
|
||||
model_runner_output.routed_experts = None
|
||||
return scheduler_output, model_runner_output
|
||||
|
||||
|
||||
def test_generation_update_from_output_carries_ec_params_to_engine_output() -> None:
|
||||
request = _make_finishing_request()
|
||||
# One-shot generation finish: the whole prompt is computed.
|
||||
request.num_computed_tokens = len(request.prompt_token_ids)
|
||||
sched = _make_finish_sched(request)
|
||||
|
||||
engine_core_outputs = OmniGenerationScheduler.update_from_output(sched, *_finish_frame_parts(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.ec_transfer_params == EC_PARAMS
|
||||
sched._free_request.assert_called_once_with(request)
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""v0.28 prefill-stats finalization in the Omni schedulers (#6606 review).
|
||||
|
||||
Upstream v0.28 finalizes prefill statistics with
|
||||
``PrefillStats.finalize(kv_cache_manager.estimate_cached_tokens(request))``
|
||||
before emitting them, which computes ``num_cache_creation_tokens`` (tokens
|
||||
computed and written to the prefix cache). Both Omni schedulers previously
|
||||
emitted ``take_prefill_stats()`` raw, so creation usage was always 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
|
||||
# before Request / RequestStatus are bound in this module.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_request() -> Request:
|
||||
request = Request(
|
||||
request_id="req-prefill-stats",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=4),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
request.status = RequestStatus.RUNNING
|
||||
# A scheduled prefill: 3 prompt tokens, 1 from the local prefix cache.
|
||||
request.prefill_stats.set(
|
||||
num_prompt_tokens=3,
|
||||
num_local_cached_tokens=1,
|
||||
num_external_cached_tokens=0,
|
||||
)
|
||||
return request
|
||||
|
||||
|
||||
def _make_sched(request: Request) -> MagicMock:
|
||||
sched = MagicMock()
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.perf_metrics = None
|
||||
sched.defer_block_free = False
|
||||
sched.structured_output_manager.should_advance.return_value = False
|
||||
sched._process_kv_transfer_trigger.return_value = False
|
||||
sched._maybe_decode_pooling_output.return_value = None
|
||||
sched._handle_stopped_request.return_value = True
|
||||
# The whole 3-token prompt ends up cached: 3 - 1 pre-cached = 2 created.
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 3
|
||||
|
||||
def _free_and_invalidate(req):
|
||||
# Lifecycle guard: freeing releases the KV blocks, after which the
|
||||
# manager reports 0 cached tokens. If finalize ran after
|
||||
# _free_request (the reviewed bug), creation count would compute
|
||||
# against 0 and the assertions below would see 0, not 2.
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
return (None, None)
|
||||
|
||||
sched._free_request.side_effect = _free_and_invalidate
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.finished_req_ids_dict = None
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
return sched
|
||||
|
||||
|
||||
def _frame(request: Request):
|
||||
scheduler_output = MagicMock(spec=SchedulerOutput)
|
||||
scheduler_output.num_scheduled_tokens = {request.request_id: 1}
|
||||
scheduler_output.total_num_scheduled_tokens = 1
|
||||
scheduler_output.scheduled_spec_decode_tokens = {}
|
||||
scheduler_output.num_invalid_spec_tokens = 0
|
||||
|
||||
model_runner_output = MagicMock(spec=ModelRunnerOutput)
|
||||
model_runner_output.sampled_token_ids = [[42]]
|
||||
model_runner_output.logprobs = None
|
||||
model_runner_output.prompt_logprobs_dict = {}
|
||||
model_runner_output.pooler_output = None
|
||||
model_runner_output.num_nans_in_logits = None
|
||||
model_runner_output.kv_connector_output = None
|
||||
model_runner_output.cudagraph_stats = None
|
||||
model_runner_output.req_id_to_index = {request.request_id: 0}
|
||||
model_runner_output.routed_experts = None
|
||||
return scheduler_output, model_runner_output
|
||||
|
||||
|
||||
def test_ar_scheduler_finalizes_prefill_stats_before_emitting() -> None:
|
||||
request = _make_request()
|
||||
sched = _make_sched(request)
|
||||
sched._update_request_with_output.return_value = ([42], True)
|
||||
|
||||
engine_core_outputs = OmniARScheduler.update_from_output(sched, *_frame(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.prefill_stats is not None
|
||||
assert eco.prefill_stats.num_cached_tokens == 1
|
||||
assert eco.prefill_stats.num_cache_creation_tokens == 2
|
||||
sched.kv_cache_manager.estimate_cached_tokens.assert_called_once_with(request)
|
||||
|
||||
|
||||
def test_generation_scheduler_finalizes_prefill_stats_before_emitting() -> None:
|
||||
request = _make_request()
|
||||
request.num_computed_tokens = len(request.prompt_token_ids)
|
||||
sched = _make_sched(request)
|
||||
|
||||
engine_core_outputs = OmniGenerationScheduler.update_from_output(sched, *_frame(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.prefill_stats is not None
|
||||
assert eco.prefill_stats.num_cached_tokens == 1
|
||||
assert eco.prefill_stats.num_cache_creation_tokens == 2
|
||||
sched.kv_cache_manager.estimate_cached_tokens.assert_called_once_with(request)
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# ruff: noqa: N803, E741
|
||||
"""Layer-level correctness & performance test for MoT parallel linear layers.
|
||||
|
||||
@@ -237,6 +240,53 @@ def _reference_forward(x, text_indices, vae_indices, text_linear, vae_linear):
|
||||
return output
|
||||
|
||||
|
||||
def _assert_gen_expert_parallel_metadata(layer):
|
||||
"""The secondary expert must satisfy vLLM's parallel-linear contract."""
|
||||
for attr in (
|
||||
"tp_rank",
|
||||
"tp_size",
|
||||
"input_size",
|
||||
"input_size_per_partition",
|
||||
"output_size",
|
||||
"output_size_per_partition",
|
||||
):
|
||||
assert getattr(layer.gen_exp, attr) == getattr(layer, attr)
|
||||
|
||||
|
||||
def test_mot_gen_expert_parallel_metadata():
|
||||
"""Keep secondary experts compatible with vLLM online quantizers."""
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
_is_tp_sharded,
|
||||
)
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
layers = (
|
||||
MoTQKVParallelLinear(
|
||||
hidden_size=16,
|
||||
head_size=4,
|
||||
total_num_heads=4,
|
||||
total_num_kv_heads=2,
|
||||
bias=False,
|
||||
vae_bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
disable_tp=True,
|
||||
),
|
||||
MoTRowParallelLinear(
|
||||
16,
|
||||
8,
|
||||
bias=False,
|
||||
vae_bias=False,
|
||||
input_is_parallel=True,
|
||||
params_dtype=torch.bfloat16,
|
||||
disable_tp=True,
|
||||
),
|
||||
)
|
||||
|
||||
for layer in layers:
|
||||
_assert_gen_expert_parallel_metadata(layer)
|
||||
assert _is_tp_sharded(layer.gen_exp) is False
|
||||
|
||||
|
||||
def _check_and_report(ref: torch.Tensor, mot: torch.Tensor, tag: str):
|
||||
"""Compare outputs, print metrics, and assert correctness.
|
||||
|
||||
@@ -335,6 +385,8 @@ def test_mot_qkv_parallel(image_num: int, K: int, N: int, dtype: str, bias: bool
|
||||
disable_tp=True,
|
||||
).cuda()
|
||||
|
||||
_assert_gen_expert_parallel_metadata(mot_linear)
|
||||
|
||||
assert text_linear.output_size_per_partition == N, (
|
||||
f"Expected output_size_per_partition={N}, "
|
||||
f"got {text_linear.output_size_per_partition}. "
|
||||
@@ -423,6 +475,8 @@ def test_mot_o_proj(
|
||||
disable_tp=True,
|
||||
).cuda()
|
||||
|
||||
_assert_gen_expert_parallel_metadata(mot_linear)
|
||||
|
||||
_sync_weights(text_linear, vae_linear, mot_linear)
|
||||
|
||||
text_idx, vae_idx, M = _make_indices(image_num, _VAE_CHUNK_SIZE)
|
||||
@@ -501,6 +555,13 @@ def test_mot_qkv_und_mode(K: int, N: int, dtype: str, bias: bool):
|
||||
).cuda()
|
||||
|
||||
with torch.no_grad():
|
||||
# vLLM linear layers leave weights uninitialized (torch.empty), so
|
||||
# a freshly zeroed GPU page would make both outputs zero and the
|
||||
# cosine-similarity gate degenerate. Seed deterministic values so
|
||||
# und-mode is compared against a meaningful reference.
|
||||
ref_linear.weight.normal_(mean=0.0, std=0.02)
|
||||
if ref_linear.bias is not None:
|
||||
ref_linear.bias.normal_(mean=0.0, std=0.02)
|
||||
mot_linear.weight.copy_(ref_linear.weight)
|
||||
if bias and ref_linear.bias is not None:
|
||||
mot_linear.bias.copy_(ref_linear.bias)
|
||||
@@ -549,6 +610,13 @@ def test_mot_row_und_mode(K: int, N: int, dtype: str, bias: bool):
|
||||
).cuda()
|
||||
|
||||
with torch.no_grad():
|
||||
# vLLM linear layers leave weights uninitialized (torch.empty), so
|
||||
# a freshly zeroed GPU page would make both outputs zero and the
|
||||
# cosine-similarity gate degenerate. Seed deterministic values so
|
||||
# und-mode is compared against a meaningful reference.
|
||||
ref_linear.weight.normal_(mean=0.0, std=0.02)
|
||||
if ref_linear.bias is not None:
|
||||
ref_linear.bias.normal_(mean=0.0, std=0.02)
|
||||
mot_linear.weight.copy_(ref_linear.weight)
|
||||
if bias and ref_linear.bias is not None:
|
||||
mot_linear.bias.copy_(ref_linear.bias)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -245,7 +245,7 @@ else:
|
||||
[sys.executable, "-c", script, str(_MODULE_PATH), str(tmp_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -1568,6 +1568,7 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc
|
||||
scheduler.structured_output_manager.should_advance.return_value = False
|
||||
scheduler.finished_req_ids_dict = {}
|
||||
scheduler.kv_cache_manager.take_events.return_value = None
|
||||
scheduler.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
scheduler.kv_event_publisher = mocker.MagicMock()
|
||||
|
||||
request = _HashableRequest(
|
||||
@@ -1648,6 +1649,7 @@ def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerF
|
||||
scheduler.finished_req_ids_dict = {}
|
||||
scheduler.kv_cache_manager = mocker.MagicMock()
|
||||
scheduler.kv_cache_manager.take_events.return_value = None
|
||||
scheduler.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
scheduler.kv_event_publisher = mocker.MagicMock()
|
||||
scheduler.waiting_for_transfer_free = set()
|
||||
scheduler.transfer_triggered_requests = set()
|
||||
@@ -1803,6 +1805,7 @@ def _build_deferred_finish_scheduler(mocker, *, running, pending_finish_reqs):
|
||||
scheduler.structured_output_manager.should_advance.return_value = False
|
||||
scheduler.finished_req_ids_dict = {}
|
||||
scheduler.kv_cache_manager.take_events.return_value = None
|
||||
scheduler.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
scheduler.kv_event_publisher = mocker.MagicMock()
|
||||
scheduler._pending_finish_reqs = list(pending_finish_reqs)
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ def _send_inpaint_with_generator(
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_deterministic(offline_client: OfflineOmniClient):
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
@@ -97,6 +98,7 @@ def test_flux2_klein_inpaint_deterministic(offline_client: OfflineOmniClient):
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_different_seeds_different_output(offline_client: OfflineOmniClient):
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
|
||||
@@ -452,6 +452,10 @@ def test_initialize_local_llm_replica_scopes_runtime_env(monkeypatch):
|
||||
"env": {runtime_env_var: "stage-value"},
|
||||
}
|
||||
monkeypatch.setenv(runtime_env_var, "parent-value")
|
||||
# Pin the logical->physical device mapping so the assertion below is
|
||||
# deterministic regardless of the ambient CUDA_VISIBLE_DEVICES (which on a
|
||||
# rebase host can be e.g. "3,4", mapping logical device 0 -> physical 3).
|
||||
runtime._init_visible_devices_baseline = "0"
|
||||
|
||||
captured: dict[str, str | None] = {}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for OmniRequestState multimodal DELTA drain and consolidation guard."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -581,18 +581,18 @@ def test_mm_only_terminal_finish_removes_request_state(monkeypatch):
|
||||
|
||||
def test_no_detokenizer_make_request_output_with_routed_experts():
|
||||
"""make_request_output accepts the routed_experts arg that the multimodal
|
||||
output channel (_process_mm_only_outputs) passes positionally, and attaches
|
||||
it to the completion output.
|
||||
output channel (_process_mm_only_outputs) passes, and attaches it to the
|
||||
completion output.
|
||||
|
||||
Regression: the call site passes 6 positional args
|
||||
(..., kv_transfer_params, routed_experts) but the override previously took
|
||||
only 5, raising TypeError on every generation-stage output.
|
||||
routed_experts is omni-specific keyword-only: the 6th positional now
|
||||
matches upstream's ec_transfer_params so that super().process_outputs()
|
||||
calls do not misroute it.
|
||||
"""
|
||||
s = _make_no_detok_state(RequestOutputKind.CUMULATIVE)
|
||||
s.add_multimodal_tensor(torch.randn(10), mm_type=AUDIO)
|
||||
routed_experts = np.zeros((2, 3), dtype=np.int32)
|
||||
# Mirror the exact call shape of _process_mm_only_outputs.
|
||||
result = s.make_request_output([], None, FinishReason.STOP, None, None, routed_experts)
|
||||
result = s.make_request_output([], None, FinishReason.STOP, None, None, routed_experts=routed_experts)
|
||||
assert result is not None
|
||||
assert not isinstance(result, PoolingRequestOutput)
|
||||
assert result.outputs[0].routed_experts is routed_experts
|
||||
@@ -663,3 +663,40 @@ def test_mm_only_outputs_update_iteration_stats():
|
||||
assert finished.finish_reason == FinishReason.STOP
|
||||
assert finished.num_prompt_tokens == state.prompt_len
|
||||
assert finished.num_generation_tokens == 2
|
||||
|
||||
|
||||
def test_ec_transfer_params_survive_both_construction_paths():
|
||||
"""v0.28 contract: encoder-cache transfer metadata must reach
|
||||
RequestOutput.ec_transfer_params through BOTH omni construction paths —
|
||||
the upstream-delegating one (logprobs processor present) and the
|
||||
no-detokenizer generation-stage one (direct RequestOutput build).
|
||||
Regression: the override accepted the argument but dropped it."""
|
||||
ec = {"remote_handle": "enc-cache-1"}
|
||||
|
||||
state = _make_state(RequestOutputKind.CUMULATIVE)
|
||||
out = state.make_request_output([1], None, None, None, {"kv": 1}, ec)
|
||||
assert out is not None and out.ec_transfer_params == ec
|
||||
|
||||
kwargs = dict(_DEFAULT_STATE_KWARGS)
|
||||
kwargs.update(logprobs_processor=None, detokenizer=None)
|
||||
gen_state = OmniRequestState(**kwargs, output_kind=RequestOutputKind.CUMULATIVE)
|
||||
completion = gen_state._new_completion_output([1], None, None)
|
||||
direct = gen_state._new_request_output("r", [completion], False, {"kv": 1}, ec)
|
||||
assert direct.ec_transfer_params == ec
|
||||
|
||||
|
||||
def test_num_cache_creation_tokens_reaches_direct_request_output():
|
||||
"""v0.28 contract: prefix-cache creation usage must reach
|
||||
RequestOutput.num_cache_creation_tokens through the no-detokenizer
|
||||
direct build, which previously passed only num_cached_tokens."""
|
||||
kwargs = dict(_DEFAULT_STATE_KWARGS)
|
||||
kwargs.update(logprobs_processor=None, detokenizer=None)
|
||||
gen_state = OmniRequestState(**kwargs, output_kind=RequestOutputKind.CUMULATIVE)
|
||||
gen_state.num_cached_tokens = 3
|
||||
gen_state.num_cache_creation_tokens = 2
|
||||
|
||||
completion = gen_state._new_completion_output([1], None, None)
|
||||
direct = gen_state._new_request_output("r", [completion], False, None, None)
|
||||
|
||||
assert direct.num_cached_tokens == 3
|
||||
assert direct.num_cache_creation_tokens == 2
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Unit tests for AsyncOmniEngine single-stage mode and OmniMasterServer."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -1374,7 +1377,7 @@ class TestConnectRemoteEngineCoresCoordinator:
|
||||
omni_master_server.get_zmq_addresses.assert_called_once_with(7, replica_id=2)
|
||||
omni_master_server.get_allocation.assert_called_once_with(7, replica_id=2)
|
||||
mock_wait.assert_called_once()
|
||||
_, _, core_engines, parallel_config, *_ = mock_wait.call_args.args
|
||||
_, core_engines, parallel_config, *_ = mock_wait.call_args.args
|
||||
assert core_engines[0].local is False
|
||||
assert parallel_config.data_parallel_size_local == 0
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""``omni_snapshot_download`` must honor vLLM's ``VLLM_USE_MODELSCOPE`` semantics.
|
||||
|
||||
vLLM treats the flag as enabled only for the literal string ``"true"``
|
||||
(case-insensitive). Reading ``os.environ`` directly made every non-empty value
|
||||
truthy, so an explicit opt-out such as ``VLLM_USE_MODELSCOPE=0`` still took the
|
||||
ModelScope path.
|
||||
vLLM treats the flag as enabled for the literal strings ``"1"`` or ``"true"``
|
||||
(case-insensitive; see vllm.envs). Reading ``os.environ`` directly made every
|
||||
non-empty value truthy, so an explicit opt-out such as ``VLLM_USE_MODELSCOPE=0``
|
||||
still took the ModelScope path.
|
||||
"""
|
||||
|
||||
import sys
|
||||
@@ -127,7 +127,7 @@ def download_backend(monkeypatch: pytest.MonkeyPatch):
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["0", "1", "False", "false", "no", "off"])
|
||||
@pytest.mark.parametrize("value", ["0", "False", "false", "no", "off"])
|
||||
def test_non_true_values_do_not_enable_modelscope(monkeypatch, download_backend, value):
|
||||
monkeypatch.setenv("VLLM_USE_MODELSCOPE", value)
|
||||
|
||||
@@ -135,7 +135,7 @@ def test_non_true_values_do_not_enable_modelscope(monkeypatch, download_backend,
|
||||
assert download_backend() == "huggingface"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["true", "True", "TRUE"])
|
||||
@pytest.mark.parametrize("value", ["1", "true", "True", "TRUE"])
|
||||
def test_true_values_enable_modelscope(monkeypatch, download_backend, value):
|
||||
monkeypatch.setenv("VLLM_USE_MODELSCOPE", value)
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import re
|
||||
@@ -894,6 +897,7 @@ def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monk
|
||||
lambda *args, **kwargs: (
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
[],
|
||||
None,
|
||||
["r1", "r2"],
|
||||
|
||||
Reference in new issue
Block a user