[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
Zhou Taichang authored and GitHub committed 2026-10-05 14:11:01 +08:00
1 parent 1f47fa2f29
commit 0b0d2d7009
141 files changed
+2885 -721

No files matched your search

@@ -44,8 +44,8 @@ def _check_prefill():
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=128, dtype=torch.bfloat16)
kvconfig = KVCacheConfig(num_blocks=1024, kv_cache_tensors=[], kv_cache_groups=[KVCacheGroupSpec(["attn"], spec)])
state = OmniModelState.__new__(OmniModelState)
state.model_config = SimpleNamespace(max_model_len=256)
state.vllm_config = c
state.max_model_len = 256
state.supports_mm_inputs = False
with set_current_vllm_config(c):
group = AttentionGroup(FlashAttentionBackend, ["attn"], spec, 0)
@@ -54,7 +54,7 @@ def test_full_decode_graph_matches_fa3_aot_schedule(graph_mode, tmp_path):
kvconfig = KVCacheConfig(num_blocks=1024, kv_cache_tensors=[], kv_cache_groups=[KVCacheGroupSpec(["attn"], spec)])
state = OmniModelState.__new__(OmniModelState)
state.vllm_config = c
state.max_model_len = 2048
state.model_config = c.model_config
state.supports_mm_inputs = False
with set_current_vllm_config(c):
group = AttentionGroup(FlashAttentionBackend, ["attn"], spec, 0)
+112 -16
View File
@@ -11,10 +11,13 @@ from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.v1.outputs import RoutedExpertsTensors
from vllm.distributed.aux_output_connector.connector import AuxRequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
import vllm_omni.worker_v2.omni_ar_model_runner as omni_ar_model_runner
from tests.helpers.mark import hardware_test
from vllm_omni.model_executor.output_snapshot import PackedOutputSnapshot
from vllm_omni.worker_v2.omni_ar_model_runner import OmniARModelRunner, OmniAsyncOutput
from vllm_omni.worker_v2.output_snapshot import pack_output_snapshot
@@ -41,7 +44,7 @@ class _FakeEvent:
def _async_output(req_ids=("req-0",), **overrides) -> OmniAsyncOutput:
rid2idx = {rid: i for i, rid in enumerate(req_ids)}
mro = omni_ar_model_runner.OmniModelRunnerOutput(list(req_ids), rid2idx, None, prompt_logprobs_dict={})
sampler_output = SamplerOutput(torch.tensor([[123]]), None, None, None)
sampler_output = SamplerOutput(torch.tensor([[123]]), None, None, torch.tensor([1]), torch.tensor([0]))
kwargs = dict(model_runner_output=mro, sampler_output=sampler_output)
kwargs.update(num_sampled_tokens=torch.tensor([1] * len(req_ids)), copy_event=_FakeEvent())
kwargs.update(main_stream=_FakeStream(), copy_stream=_FakeStream())
@@ -64,23 +67,38 @@ def test_async_output_blocking_event_and_routing_masks(monkeypatch, compact_widt
counts=torch.tensor([2, 0]),
vocab_size=4,
)
sampler_output = SamplerOutput(torch.tensor([[2], [0]]), None, None, torch.tensor([1, 0]), None, masks)
routed = RoutedExpertsTensors(torch.tensor([[[2, 3]], [[4, 5]]]), torch.tensor([7, 9]))
sampler_output = SamplerOutput(
torch.tensor([[2], [0]]), None, None, torch.tensor([1, 0]), torch.tensor([0, 0]), masks
)
copied: list[tuple[np.ndarray, np.ndarray]] = []
class _FakePendingAuxOutput:
def enqueue_cpu_copy(self, *, num_sampled, num_rejected) -> None:
copied.append((num_sampled, num_rejected))
def process_output(self) -> dict[str, AuxRequestOutput]:
return {
"decode": AuxRequestOutput(token_start=0, rows=np.array([[2, 3]], dtype=np.uint8)),
"prefill": AuxRequestOutput(token_start=0, rows=np.array([[4, 5]], dtype=np.uint8)),
}
pending = _FakePendingAuxOutput()
output = _async_output(
req_ids=["decode", "prefill"],
sampler_output=sampler_output,
num_sampled_tokens=torch.tensor([1, 0]),
copy_event=None, # None → constructor builds the default blocking event
routed_experts=routed,
pending_aux_output=pending,
).get_output()
assert event_kwargs == [{"blocking": True}] # blocking event by default
assert output.sampled_token_ids == [[2], []]
assert output.sampling_masks.to_nested_list() == [[0, 2], []]
np.testing.assert_array_equal(output.sampling_masks.offsets, [0, 2, 2])
np.testing.assert_array_equal(output.routed_experts.routing_data, [[[2, 3]], [[4, 5]]])
np.testing.assert_array_equal(output.routed_experts.slot_mapping, [7, 9])
np.testing.assert_array_equal(output.aux_output_connector_output["decode"].rows, [[2, 3]])
np.testing.assert_array_equal(output.aux_output_connector_output["prefill"].rows, [[4, 5]])
assert output.aux_output_connector_output["decode"].token_start == 0
np.testing.assert_array_equal(output.sampling_masks.token_ids, [0, 2])
assert len(copied) == 1 and copied[0][0].tolist() == [1, 0]
@pytest.mark.parametrize("needs_history", [False, True])
@@ -88,19 +106,26 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
runner = OmniARModelRunner.__new__(OmniARModelRunner)
input_batch = SimpleNamespace(req_ids=["req"], num_reqs=1, seq_lens=torch.tensor([3]))
input_batch.idx_mapping, input_batch.query_start_loc = torch.tensor([0]), torch.tensor([0, 1])
input_batch.idx_mapping_np = np.array([0])
input_batch.num_computed_prefill_tokens_np = np.array([0])
input_batch.num_scheduled_tokens = np.array([3])
input_batch.prefill_len_np = np.array([3])
input_batch.query_start_loc_np = np.array([0, 3])
state = SimpleNamespace(input_batch=input_batch, hidden_states=torch.zeros(1, 2))
state.finished_req_ids, state.ec_connector_output, state.routed_experts = {"finished"}, None, None
state.finished_req_ids, state.ec_connector_output = {"finished"}, None
runner.execute_model_state = state
runner._kv_extracted_req_ids = runner._last_aux_output = runner._last_multimodal_outputs = None
runner.is_last_pp_rank, runner.pp_handler, runner.check_ep_fault = True, None, False
runner.aux_output_connector = None
runner.model_config = SimpleNamespace(async_chunk=False)
runner.vllm_config = SimpleNamespace(model_config=SimpleNamespace(engine_output_type="text"))
runner.model_state = SimpleNamespace(postprocess_model_output=MagicMock(return_value=(torch.zeros(1, 2), None)))
text_hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
runner.model_state = SimpleNamespace(postprocess_model_output=MagicMock(return_value=(text_hidden, None)))
runner.model_state.intermediate_buffer = SimpleNamespace(buffers={0: {"global_request_id": "global-req"}})
runner.req_states = SimpleNamespace(req_id_to_index={"req": 0})
runner.req_states.all_token_ids = SimpleNamespace(gpu=torch.tensor([[1]]))
runner.req_states.num_computed_tokens = SimpleNamespace(gpu=torch.tensor([0]))
runner.req_states.prompt_len = SimpleNamespace(np=np.array([1]), gpu=torch.tensor([1]))
runner.req_states.prompt_len = SimpleNamespace(np=np.array([3]), gpu=torch.tensor([3]))
runner.main_stream = runner.output_copy_stream = MagicMock()
runner.eplb = runner._finalize_native_data_plane_output = runner._reserve_native_data_plane_outputs = MagicMock()
sampler_out = (SimpleNamespace(sampled_token_ids=torch.tensor([[2]])), MagicMock(), MagicMock())
@@ -114,13 +139,17 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
yield
sampling_active = False
runner.model = SimpleNamespace(compute_logits=None, logitsprocs_need_output_token_ids=needs_history)
runner.model = SimpleNamespace(
compute_logits=lambda hidden: hidden, logitsprocs_need_output_token_ids=needs_history
)
runner.model.mrv2_sampling_context = sampling_context
runner.sampler = None
runner.sample = MagicMock(return_value=sampler_out)
runner.sample.side_effect = lambda *_: sampler_out if sampling_active is needs_history else pytest.fail()
logprobs_mock = MagicMock(side_effect=lambda *_: pytest.fail("inside ctx") if sampling_active else {})
runner.prompt_logprobs_worker = SimpleNamespace(compute_prompt_logprobs=logprobs_mock)
runner.prompt_logprobs_worker = PromptLogprobsWorker(1, torch.device("cpu"), logprobs_mode="raw_logits")
runner.prompt_logprobs_worker.add_request("req", 0, SamplingParams(prompt_logprob_token_ids=[1, 0]))
runner.prompt_logprobs_worker.compute_prompt_logprobs = logprobs_mock
runner.postprocess_sampled, connector_output = MagicMock(), object()
def post_forward(finished_req_ids):
@@ -135,10 +164,67 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
assert runner.sample_tokens(None) is mock_out
built = omni_ar_model_runner.OmniAsyncOutput.call_args.kwargs["model_runner_output"]
assert built.kv_connector_output is connector_output
torch.testing.assert_close(built.prompt_token_id_logprobs_dict["req"], torch.tensor([[2.0, 1.0], [4.0, 3.0]]))
assert runner._resolve_global_request_id("req") == "global-req" # from the intermediate buffer
assert runner._resolve_global_request_id("unknown") == "unknown" # fallback to the local id
def test_async_output_preserves_fixed_token_scores_on_host(monkeypatch) -> None:
monkeypatch.setattr(torch.cuda, "set_stream", lambda _stream: None)
scores = torch.tensor([[-0.5, -1.5], [-0.2, -2.2]])
mro = omni_ar_model_runner.OmniModelRunnerOutput(
req_ids=["req-0"],
req_id_to_index={"req-0": 0},
sampled_token_ids=None,
prompt_logprobs_dict={},
prompt_token_id_logprobs_dict={"req-0": scores},
)
output = _async_output(model_runner_output=mro).get_output()
assert output.prompt_token_id_logprobs_dict["req-0"].device.type == "cpu"
torch.testing.assert_close(output.prompt_token_id_logprobs_dict["req-0"], scores)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_chunked_fixed_token_scores_are_copied_from_cuda() -> None:
device = torch.device("cuda")
hidden = torch.tensor([[1.0, 2.0, 3.0], [3.0, 1.0, 2.0], [2.0, 3.0, 1.0], [4.0, 5.0, 6.0]], device=device)
worker = PromptLogprobsWorker(1, device)
worker.add_request("req", 0, SamplingParams(prompt_logprob_token_ids=[2, 0], prompt_logprob_start=1))
batch = SimpleNamespace(
req_ids=["req"],
idx_mapping_np=np.array([0]),
num_computed_prefill_tokens_np=np.array([0]),
num_scheduled_tokens=np.array([2]),
prefill_len_np=np.array([4]),
query_start_loc_np=np.array([0, 2]),
)
assert worker.compute_prompt_token_id_logprobs(lambda hidden: hidden, hidden[:2], batch, np.array([4])) == {}
batch.num_computed_prefill_tokens_np = np.array([2])
scores = worker.compute_prompt_token_id_logprobs(lambda hidden: hidden, hidden[2:], batch, np.array([4]))
mro = omni_ar_model_runner.OmniModelRunnerOutput(
req_ids=["req"],
req_id_to_index={"req": 0},
sampled_token_ids=None,
prompt_logprobs_dict={},
prompt_token_id_logprobs_dict=scores,
)
num_sampled = torch.tensor([1], device=device, dtype=torch.int32)
sampler_output = SamplerOutput(
torch.tensor([[1]], device=device), None, None, num_sampled, torch.zeros_like(num_sampled)
)
output = OmniAsyncOutput(
model_runner_output=mro,
sampler_output=sampler_output,
num_sampled_tokens=num_sampled,
main_stream=torch.cuda.current_stream(),
copy_stream=torch.cuda.Stream(),
).get_output()
expected = hidden[1:3].log_softmax(dim=-1)[:, [2, 0]].cpu()
assert output.prompt_token_id_logprobs_dict["req"].device.type == "cpu"
torch.testing.assert_close(output.prompt_token_id_logprobs_dict["req"], expected)
def test_async_mm_snapshot_owns_output_until_copy_finishes() -> None:
runner = OmniARModelRunner.__new__(OmniARModelRunner)
runner.model_config = SimpleNamespace(async_chunk=True)
@@ -284,7 +370,13 @@ def test_async_output_slices_request_payloads_with_graph_padding(
batch.num_tokens_after_padding = padded_total
output = _async_output(
req_ids=[f"req-{i}" for i in range(len(lengths))],
sampler_output=SamplerOutput(torch.ones(len(lengths), 1, dtype=torch.long), None, None, None),
sampler_output=SamplerOutput(
torch.ones(len(lengths), 1, dtype=torch.long),
None,
None,
torch.ones(len(lengths), dtype=torch.long),
torch.zeros(len(lengths), dtype=torch.long),
),
text_hidden=hidden,
multimodal_outputs={"codes": {"audio": codes, "ref": refs}},
input_batch=batch,
@@ -585,7 +677,9 @@ def test_model_owned_audio_finalizer_runs_after_copy_without_hidden(monkeypatch,
result = _async_output(
req_ids=["audio", "partial"],
sampler_output=SamplerOutput(torch.tensor([[99], [100]]), None, None, None),
sampler_output=SamplerOutput(
torch.tensor([[99], [100]]), None, None, torch.tensor([1, 1]), torch.tensor([0, 0])
),
num_sampled_tokens=torch.tensor([1, 0]),
multimodal_outputs={"snapshot": snapshot},
input_batch=batch,
@@ -710,7 +804,7 @@ def test_stream_audio_snapshot_skips_discarded_code_partition(monkeypatch, mocke
extra = ({"extra": torch.tensor([7])}, _FakeEvent()) if consumer == "extra" else None
result = _async_output(
req_ids=["frame", "eos", "prefill"],
sampler_output=SamplerOutput(torch.tensor([[1], [2], [3]]), None, None, None),
sampler_output=SamplerOutput(torch.tensor([[1], [2], [3]]), None, None, None, None),
num_sampled_tokens=torch.tensor([1, 1, 1]),
multimodal_outputs={"codes": {"audio": torch.ones(4, 2)}},
input_batch=batch,
@@ -753,7 +847,9 @@ def test_request_owned_snapshot_skips_generic_partition(monkeypatch, streaming):
result = _async_output(
req_ids=["audio", "partial"],
sampler_output=SamplerOutput(torch.tensor([[99], [100]]), None, None, None),
sampler_output=SamplerOutput(
torch.tensor([[99], [100]]), None, None, torch.tensor([1, 1]), torch.tensor([0, 0])
),
num_sampled_tokens=torch.tensor([1, 0]),
multimodal_outputs={"snapshot": snapshot},
input_batch=batch,
@@ -4,12 +4,14 @@
"""OmniGenerationModelRunner contracts: empty-step lifecycle, output partition,
CPU-sync vs CUDA-async dispatch, and async-chunk slot recycling."""
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.distributed.aux_output_connector.connector import AuxRequestOutput
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.outputs import OmniModelRunnerOutput
@@ -18,6 +20,17 @@ from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModel
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize("randomize_inputs", [False, True])
def test_profile_run_accepts_release_flag_without_running_codec(monkeypatch, randomize_inputs):
runner = object.__new__(OmniGenerationModelRunner)
runner.model = MagicMock(side_effect=AssertionError("profiling ran codec model"))
synchronize = MagicMock()
monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)
runner.profile_run(randomize_inputs=randomize_inputs)
synchronize.assert_called_once_with()
runner.model.assert_not_called()
class _FakeStagedField:
def __init__(self, data):
self.np = data
@@ -45,6 +58,7 @@ def _make_runner(model_output, num_reqs=1, prompt_len=10):
runner.execute_model_state = SimpleNamespace(finished_req_ids={"finished"}, ec_connector_output=None)
runner.kv_connector = SimpleNamespace(post_forward=MagicMock(return_value=None))
runner.check_ep_fault = False
runner.aux_output_connector = None
runner.req_states = MagicMock(
prompt_len=SimpleNamespace(np=np.full(num_reqs, prompt_len, dtype=np.int32)),
num_computed_tokens=_FakeStagedField(np.zeros(num_reqs, dtype=np.int32)),
@@ -52,7 +66,8 @@ def _make_runner(model_output, num_reqs=1, prompt_len=10):
return runner
def test_control_only_step_keeps_lifecycle_and_skips_input_construction():
@pytest.mark.parametrize("aux_enabled", [False, True])
def test_control_only_step_keeps_lifecycle_and_skips_input_construction(aux_enabled):
runner = object.__new__(OmniGenerationModelRunner)
order = []
for name in (
@@ -68,8 +83,13 @@ def test_control_only_step_keeps_lifecycle_and_skips_input_construction():
runner.kv_connector = SimpleNamespace(no_forward=lambda _s: output)
runner._merge_ec_connector_no_forward = lambda _s, value: value
runner._attach_native_data_plane_signals = lambda value: value
metadata = object()
runner.aux_output_connector = SimpleNamespace(begin_step=MagicMock()) if aux_enabled else None
scheduler_output = SimpleNamespace(
total_num_scheduled_tokens=0, scheduled_new_reqs=[], scheduled_cached_reqs=SimpleNamespace(req_ids=[])
total_num_scheduled_tokens=0,
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(req_ids=[]),
aux_output_connector_metadata=metadata,
)
assert runner.execute_model(scheduler_output) is output
assert order == [
@@ -78,6 +98,9 @@ def test_control_only_step_keeps_lifecycle_and_skips_input_construction():
"free_states",
"_apply_block_table_staged_writes_if_available",
]
if aux_enabled:
assert runner.aux_output_connector is not None
runner.aux_output_connector.begin_step.assert_called_once_with(metadata)
def test_released_chunk_reuses_scheduler_output_and_slot_recycle_clears_state():
@@ -184,6 +207,72 @@ def test_sample_tokens_uses_async_output_for_cuda_and_snapshots_req_ids(monkeypa
assert output_req_ids == ["req-0"]
def test_generation_aux_output_copies_and_materializes_without_sampling(monkeypatch):
from vllm_omni.worker_v2 import omni_generation_model_runner as module
events = []
class PendingOutput:
def enqueue_cpu_copy(self, *, num_sampled, num_rejected):
np.testing.assert_array_equal(num_sampled, [0])
np.testing.assert_array_equal(num_rejected, [0])
events.append("copy")
def process_output(self):
assert events == ["copy", "record", "synchronize"]
return {"req-0": AuxRequestOutput(0, np.array([[2, 3]], dtype=np.uint8))}
class CopyEvent:
def record(self, stream):
events.append("record")
def synchronize(self):
events.append("synchronize")
monkeypatch.setattr(torch.cuda, "Event", lambda **kw: CopyEvent())
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
monkeypatch.setattr(module, "_async_copy_mm", lambda *a, **kw: {})
output = OmniModelRunnerOutput(["req-0"], {"req-0": 0}, [[]])
result = module.OmniGenerationAsyncOutput(
model_runner_output=output,
multimodal_outputs={},
num_reqs=1,
main_stream=object(),
copy_stream=SimpleNamespace(wait_stream=lambda stream: None),
pending_aux_output=PendingOutput(),
).get_output()
assert result.sampled_token_ids == [[]]
np.testing.assert_array_equal(result.aux_output_connector_output["req-0"].rows, [[2, 3]])
def test_pending_aux_output_forces_async_delivery_without_cuda_payload(monkeypatch):
from vllm_omni.worker_v2 import omni_generation_model_runner as module
runner = _make_runner(OmniOutput(text_hidden_states=torch.empty(0), multimodal_outputs={}))
runner.device = SimpleNamespace(type="cuda")
runner.main_stream = object()
runner.output_copy_stream = object()
pending = object()
runner.aux_output_connector = SimpleNamespace(prepare_output=MagicMock(return_value=pending))
runner._reserve_native_data_plane_outputs = MagicMock()
runner._release_generation_slots = MagicMock()
runner._finalize_native_data_plane_output = MagicMock()
captured = {}
class AsyncOutput:
def __init__(self, **kwargs):
captured.update(kwargs)
monkeypatch.setattr(module, "OmniGenerationAsyncOutput", AsyncOutput)
input_batch = runner._gen_input_batch
result = runner.sample_tokens()
assert isinstance(result, AsyncOutput)
runner.aux_output_connector.prepare_output.assert_called_once_with(input_batch)
assert captured["pending_aux_output"] is pending
@pytest.mark.parametrize("exact", [True, False])
def test_exact_input_shape_models_reject_padded_batches(exact):
# Qwen3-Omni Code2Wav splits input_ids by seq_token_counts (#6712).
+61 -8
View File
@@ -13,6 +13,7 @@ from vllm import SamplingParams
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm_omni.config.model import OmniModelConfig
@@ -95,7 +96,8 @@ def test_full_payload_receive_is_polled_without_scheduled_tokens(mocker):
@pytest.mark.parametrize("output_form", ["tuple", "omni"])
def test_capture_model_unwraps_exclude_full_and_capture_mtp(output_form):
@pytest.mark.parametrize("profile_only", [False, True])
def test_capture_model_unwraps_exclude_full_and_capture_mtp(output_form, profile_only):
runner = object.__new__(OmniGPUModelRunner)
hidden = torch.ones(1, 2)
@@ -117,16 +119,21 @@ def test_capture_model_unwraps_exclude_full_and_capture_mtp(output_form):
runner.model_state = SimpleNamespace(capture_mtp_graphs=MagicMock())
runner._dispatch_mtp_batch_descriptor = MagicMock(return_value="desc")
def assert_unwrapped(_self):
def assert_unwrapped(_self, *, profile_only: bool = False):
assert torch.equal(runner.model.forward(), hidden) # forward unwrapped during capture
assert profile_only is expected_profile_only
return 3
expected_profile_only = profile_only
with patch.object(GPUModelRunner, "capture_model", assert_unwrapped):
assert runner.capture_model() == 3
assert runner.capture_model(profile_only=profile_only) == 3
assert runner.model.forward is original_forward # restored after capture
assert runner.cudagraph_manager._capture_descs == {CUDAGraphMode.PIECEWISE: [piecewise]}
runner.model_state.capture_mtp_graphs.assert_called_once_with(runner._dispatch_mtp_batch_descriptor)
if profile_only:
runner.model_state.capture_mtp_graphs.assert_not_called()
else:
runner.model_state.capture_mtp_graphs.assert_called_once_with(runner._dispatch_mtp_batch_descriptor)
@pytest.mark.parametrize(
@@ -163,6 +170,28 @@ def test_init_model_state_factory_dispatches_omni_only(monkeypatch, flag, expect
assert state is upstream.return_value
@pytest.mark.parametrize("omni_stage", [False, True])
@pytest.mark.parametrize("specialized", [False, True])
def test_auxiliary_default_state_has_omni_lifecycle_but_preserves_upstream_states(monkeypatch, omni_stage, specialized):
class SpecializedState(DefaultModelState):
pass
state_cls = SpecializedState if specialized else DefaultModelState
upstream_state = object.__new__(state_cls)
monkeypatch.setattr("vllm_omni.worker_v2.model_states._upstream_init_model_state", lambda *_args: upstream_state)
monkeypatch.setattr(OmniModelState, "__init__", lambda *_args: None)
model_config = object.__new__(OmniModelConfig) if omni_stage else SimpleNamespace()
state = init_omni_model_state(
SimpleNamespace(model_config=model_config), SimpleNamespace(), None, torch.device("cpu")
)
if omni_stage and not specialized:
assert isinstance(state, OmniModelState)
assert callable(state.run_preprocess)
assert callable(state.postprocess_model_output)
else:
assert state is upstream_state
def test_finish_requests_notifies_model_and_cleans_only_known_slots(monkeypatch):
runner = _make_runner()
calls = []
@@ -209,7 +238,7 @@ def test_full_graph_aux_contract_keeps_full_and_round_trips_leaves():
runner.model_state = SimpleNamespace()
captured = {}
def capture(_self):
def capture(_self, *, profile_only=False):
# The graph manager stores (hidden, leaves) in its aux buffers.
assert runner.use_aux_hidden_state_outputs is True
hidden, leaves = runner.model.forward(4)
@@ -262,7 +291,9 @@ def test_full_graph_output_without_contract_is_hidden_only():
@pytest.mark.parametrize("runner_kind", ["gpu", "ar", "generation"])
def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
@pytest.mark.parametrize("randomize_inputs", [False, True])
@pytest.mark.parametrize("is_profile", [False, True])
def test_dummy_forward_uses_upstream_execution_state(runner_kind, randomize_inputs, is_profile, monkeypatch):
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_runner import ExecuteModelState
@@ -288,6 +319,7 @@ def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
runner.model_state.prepare_inputs.return_value = {}
runner._omni_data_plane = object()
runner.supports_mm_inputs = False
runner.vocab_size = 32
runner.lora_config = None
runner.is_encoder_decoder = False
runner.eplb = MagicMock()
@@ -308,7 +340,15 @@ def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
cg_mode=CUDAGraphMode.NONE, num_reqs=1, num_tokens=1, num_active_loras=0, max_query_len=1
)
runner._dispatch_batch_descriptor = MagicMock(return_value=(batch_desc, None))
monkeypatch.setattr(InputBatch, "make_dummy", lambda *args, **kwargs: input_batch)
make_dummy = MagicMock(return_value=input_batch)
monkeypatch.setattr(InputBatch, "make_dummy", make_dummy)
randomized = []
def randomize(tensor, low, high):
randomized.append((low, high))
return tensor.fill_(7)
monkeypatch.setattr(torch.Tensor, "random_", randomize)
monkeypatch.setattr("vllm_omni.worker_v2.omni_model_runner.build_slot_mappings_by_layer", lambda *args: {})
for module in ("omni_model_runner", "omni_generation_model_runner"):
monkeypatch.setattr(f"vllm_omni.worker_v2.{module}.set_forward_context", lambda *args, **kwargs: nullcontext())
@@ -318,7 +358,20 @@ def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
# Upstream _dummy_run always supplies valid_dummy_state_slots, and the
# result must use the real upstream constructor rather than a mocked state.
assert runner.execute_model(scheduled, dummy_run=True, valid_dummy_state_slots=True) is None
assert (
runner.execute_model(
scheduled,
dummy_run=True,
valid_dummy_state_slots=True,
randomize_inputs=randomize_inputs,
is_profile=is_profile,
)
is None
)
assert make_dummy.call_args.kwargs["is_padding"] is not is_profile
should_randomize = randomize_inputs and runner_kind != "generation"
assert randomized == ([(0, 32)] if should_randomize else [])
assert input_batch.input_ids.tolist() == ([7] if should_randomize else [1])
assert isinstance(runner.execute_model_state, ExecuteModelState)
assert runner.execute_model_state.input_batch is input_batch
assert runner.execute_model_state.cudagraph_stats is None
+10
View File
@@ -22,6 +22,16 @@ from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState, _m
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_prepare_attn_forwards_release_model_specific_metadata():
state = object.__new__(OmniModelState)
metadata = object()
output = object()
batch = SimpleNamespace()
with patch.object(DefaultModelState, "prepare_attn", return_value=output) as prepare:
assert state.prepare_attn(batch, None, (), None, [], None, model_specific_attn_metadata=metadata) is output
assert prepare.call_args.kwargs["model_specific_attn_metadata"] is metadata
class _DummyInputBatch:
is_prefilling_np: np.ndarray
req_ids: list[str]