[MR V2] Migrate to Model Runner V2

Squashed migration rebased onto current main. Collapses the 46-commit
dev/migrate-MR-v2 history (10 merge commits + superseded pre-rebase work)
into a single commit, so the diff reflects only the genuine MR-V2 feature
(worker_v2/ runners + model_states, omni schedulers, stage engine pipeline,
pooling_output serialization bridge). Content already merged to main
separately (step_audio2, tts_adapters, npu/_310p, etc.) drops out.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Taichang ZhouandClaude Opus 4.8 committed 2026-06-20 07:19:40 +00:00
1 parent cead615288
commit 9678ae58cf
74 files changed
+8244 -95

No files matched your search

View File
Whitespace-only changes.
+83
View File
@@ -0,0 +1,83 @@
"""Unit tests for init_omni_model_state factory dispatch."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm_omni.worker_v2.model_states import (
_OMNI_ARCHITECTURES,
init_omni_model_state,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_vllm_config(architectures):
model_config = SimpleNamespace(architectures=architectures)
return SimpleNamespace(model_config=model_config)
@patch("vllm_omni.worker_v2.model_states.omni_model_state.OmniModelState.__init__", return_value=None)
def test_qwen3_omni_dispatches_to_omni_model_state(mock_init):
cfg = _make_vllm_config(["Qwen3OmniMoeForConditionalGeneration"])
model = MagicMock()
device = torch.device("cpu")
state = init_omni_model_state(cfg, model, None, device)
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
assert isinstance(state, OmniModelState)
@patch("vllm_omni.worker_v2.model_states.omni_model_state.OmniModelState.__init__", return_value=None)
def test_mammoth_dispatches_to_omni_model_state(mock_init):
cfg = _make_vllm_config(["MammothModa2ForConditionalGeneration"])
model = MagicMock()
device = torch.device("cpu")
state = init_omni_model_state(cfg, model, None, device)
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
assert isinstance(state, OmniModelState)
@patch("vllm_omni.worker_v2.model_states._upstream_init_model_state")
def test_unknown_arch_delegates_to_upstream(mock_upstream):
mock_upstream.return_value = MagicMock()
cfg = _make_vllm_config(["LlamaForCausalLM"])
model = MagicMock()
device = torch.device("cpu")
state = init_omni_model_state(cfg, model, None, device)
mock_upstream.assert_called_once_with(cfg, model, None, device)
assert state is mock_upstream.return_value
def test_omni_architectures_set_contains_expected():
expected = {
"Qwen3OmniMoeForConditionalGeneration",
"Qwen2_5OmniForConditionalGeneration",
"MammothModa2ForConditionalGeneration",
"MiMoAudioForConditionalGeneration",
"MammothModa2ARForConditionalGeneration",
"Qwen3TTSTalkerForConditionalGeneration",
"Qwen3TTSCode2Wav",
}
assert _OMNI_ARCHITECTURES == expected
@patch("vllm_omni.worker_v2.model_states._upstream_init_model_state")
def test_none_architectures_delegates_to_upstream(mock_upstream):
mock_upstream.return_value = MagicMock()
cfg = _make_vllm_config(None)
model = MagicMock()
device = torch.device("cpu")
init_omni_model_state(cfg, model, None, device)
mock_upstream.assert_called_once()
+168
View File
@@ -0,0 +1,168 @@
"""Unit tests for OmniIntermediateBuffer lifecycle."""
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.worker_v2.model_states.intermediate_buffer import (
OmniIntermediateBuffer,
_resolve_additional_information,
_resolve_prompt_embeds,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _DummyInputBatch:
def __init__(self, idx_mapping):
self.idx_mapping_np = idx_mapping
def _make_new_req_data(req_id="r1", prompt_embeds=None, mm_features=None, additional_information=None):
ns = SimpleNamespace(
req_id=req_id,
mm_features=mm_features or [],
)
if prompt_embeds is not None:
ns.prompt_embeds = prompt_embeds
if additional_information is not None:
ns.additional_information = additional_information
return ns
# ---------------------------------------------------------------
# _resolve_prompt_embeds
# ---------------------------------------------------------------
def test_resolve_prompt_embeds_tensor():
t = torch.randn(3, 4, device="cpu")
result = _resolve_prompt_embeds(t)
assert isinstance(result, torch.Tensor)
assert result.shape == (3, 4)
assert result.is_contiguous()
def test_resolve_prompt_embeds_none():
assert _resolve_prompt_embeds(None) is None
# ---------------------------------------------------------------
# _resolve_additional_information
# ---------------------------------------------------------------
def test_resolve_additional_information_dict_passthrough():
d = {"key": "val"}
assert _resolve_additional_information(d) == d
def test_resolve_additional_information_none():
assert _resolve_additional_information(None) == {}
# ---------------------------------------------------------------
# OmniIntermediateBuffer core lifecycle
# ---------------------------------------------------------------
def test_add_and_remove():
buf = OmniIntermediateBuffer(max_num_reqs=4)
req = _make_new_req_data(req_id="r1")
buf.add_request(0, req)
assert buf.buffers[0]["req_id"] == "r1"
buf.remove_request(0)
assert buf.buffers[0] == {}
def test_add_with_prompt_embeds():
pe = torch.randn(2, 8)
req = _make_new_req_data(req_id="r1", prompt_embeds=pe)
buf = OmniIntermediateBuffer(max_num_reqs=4)
buf.add_request(0, req)
assert "prompt_embeds_cpu" in buf.buffers[0]
assert buf.buffers[0]["prompt_embeds_cpu"].shape == (2, 8)
def test_add_with_mm_features():
features = [torch.randn(3, 4)]
req = _make_new_req_data(req_id="r1", mm_features=features)
buf = OmniIntermediateBuffer(max_num_reqs=4)
buf.add_request(1, req)
assert buf.buffers[1]["mm_features"] is features
def test_add_with_additional_information():
info = {"audio_embeds": torch.randn(5)}
req = _make_new_req_data(req_id="r1", additional_information=info)
buf = OmniIntermediateBuffer(max_num_reqs=4)
buf.add_request(0, req)
assert torch.allclose(buf.buffers[0]["audio_embeds"], info["audio_embeds"])
# ---------------------------------------------------------------
# gather
# ---------------------------------------------------------------
def test_gather_returns_batch_order():
buf = OmniIntermediateBuffer(max_num_reqs=4)
buf.add_request(0, _make_new_req_data(req_id="r0"))
buf.add_request(2, _make_new_req_data(req_id="r2"))
buf.add_request(3, _make_new_req_data(req_id="r3"))
batch = _DummyInputBatch(idx_mapping=[2, 0, 3])
gathered = buf.gather(batch)
assert [g["req_id"] for g in gathered] == ["r2", "r0", "r3"]
# ---------------------------------------------------------------
# update
# ---------------------------------------------------------------
def test_update_merges_tensors_to_cpu():
buf = OmniIntermediateBuffer(max_num_reqs=2)
buf.add_request(0, _make_new_req_data(req_id="r0"))
buf.update(0, {"hidden": torch.randn(4)})
assert "hidden" in buf.buffers[0]
assert buf.buffers[0]["hidden"].device == torch.device("cpu")
def test_update_gpu_resident_keys():
buf = OmniIntermediateBuffer(max_num_reqs=2)
buf.add_request(0, _make_new_req_data(req_id="r0"))
t = torch.randn(4)
buf.update(0, {"kv": t}, gpu_resident_keys={"kv"})
assert "kv" in buf.buffers[0]
def test_update_list_values():
buf = OmniIntermediateBuffer(max_num_reqs=2)
buf.add_request(0, _make_new_req_data(req_id="r0"))
items = [torch.randn(2), "text"]
buf.update(0, {"items": items})
stored = buf.buffers[0]["items"]
assert isinstance(stored[0], torch.Tensor)
assert stored[1] == "text"
def test_update_empty_is_noop():
buf = OmniIntermediateBuffer(max_num_reqs=2)
buf.add_request(0, _make_new_req_data(req_id="r0"))
before = dict(buf.buffers[0])
buf.update(0, {})
assert buf.buffers[0] == before
def test_update_accumulates():
buf = OmniIntermediateBuffer(max_num_reqs=2)
buf.add_request(0, _make_new_req_data(req_id="r0"))
buf.update(0, {"a": torch.tensor([1.0])})
buf.update(0, {"b": torch.tensor([2.0])})
assert "a" in buf.buffers[0] and "b" in buf.buffers[0]
@@ -0,0 +1,279 @@
"""Unit tests for OmniARModelRunner v2."""
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from vllm_omni.data_entry_keys import unflatten_payload
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.worker_v2.omni_ar_model_runner import (
OmniARModelRunner,
_async_copy_mm,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# ---------------------------------------------------------------
# _build_pooler_output_from_cpu (was _build_pooler_output)
# ---------------------------------------------------------------
def test_reconstruct_raw_model_output_preserves_omni_multimodal_outputs():
hidden = torch.randn(3, 4)
latent = torch.randn(3, 4)
raw = OmniARModelRunner._reconstruct_raw_model_output(
hidden_states=hidden,
multimodal_outputs={"latent": latent},
aux=None,
)
assert isinstance(raw, OmniOutput)
assert raw.text_hidden_states is hidden
assert raw.multimodal_outputs["latent"] is latent
def test_reconstruct_raw_model_output_keeps_aux_tuple_without_multimodal_outputs():
hidden = torch.randn(3, 4)
aux = {"layers": torch.randn(3, 2)}
raw = OmniARModelRunner._reconstruct_raw_model_output(
hidden_states=hidden,
multimodal_outputs=None,
aux=aux,
)
assert raw == (hidden, aux)
def test_reconstruct_raw_model_output_ignores_empty_multimodal_outputs():
hidden = torch.randn(3, 4)
raw = OmniARModelRunner._reconstruct_raw_model_output(
hidden_states=hidden,
multimodal_outputs={},
aux=None,
)
assert raw is hidden
def test_build_pooler_output_basic():
"""Verify _build_pooler_output_from_cpu slices per-request hidden + mm."""
hidden = torch.randn(6, 8)
mm = {"audio": torch.randn(6, 2)}
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
mm,
query_start_loc_np=np.array([0, 3]),
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
num_reqs=2,
)
assert len(pooler) == 2
assert pooler[0]["hidden"].shape == (3, 8)
assert pooler[1]["hidden"].shape == (3, 8)
assert pooler[0]["audio"].shape == (3, 2)
def test_build_pooler_output_empty_mm():
hidden = torch.randn(4, 8)
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
{},
query_start_loc_np=np.array([0]),
num_scheduled_tokens=np.array([4], dtype=np.int32),
num_reqs=1,
)
assert len(pooler) == 1
assert "hidden" in pooler[0]
assert len(pooler[0]) == 1
# ---------------------------------------------------------------
# _async_copy_mm (was _copy_mm_to_cpu)
# ---------------------------------------------------------------
def test_copy_mm_to_cpu_tensor():
total = 10
t = torch.randn(10, 4)
result = _async_copy_mm({"feat": t}, total)
assert "feat" in result
assert result["feat"].shape == (10, 4)
assert result["feat"].device == torch.device("cpu")
def test_copy_mm_to_cpu_dict():
total = 10
d = {"inner": torch.randn(10, 2)}
result = _async_copy_mm({"nested": d}, total)
assert "nested" in result
assert "inner" in result["nested"]
def test_copy_mm_to_cpu_list():
result = _async_copy_mm({"items": [torch.randn(3), "text"]}, 10)
assert "items" in result
assert isinstance(result["items"][0], torch.Tensor)
assert result["items"][1] == "text"
def test_copy_mm_to_cpu_empty():
assert _async_copy_mm({}, 10) == {}
# ---------------------------------------------------------------
# Slicing via _build_pooler_output_from_cpu (was _slice_mm_payload)
# ---------------------------------------------------------------
def test_slice_mm_payload_tensor():
hidden = torch.randn(6, 4)
mm_cpu = {"feat": torch.randn(6, 2)}
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
mm_cpu,
query_start_loc_np=np.array([0, 3]),
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
num_reqs=2,
)
assert pooler[0]["feat"].shape == (3, 2)
assert pooler[1]["feat"].shape == (3, 2)
def test_slice_mm_payload_list():
hidden = torch.randn(6, 4)
mm_cpu = {"items": [torch.randn(2), torch.randn(3)]}
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
mm_cpu,
query_start_loc_np=np.array([0, 3]),
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
num_reqs=2,
)
assert isinstance(pooler[0]["items"], torch.Tensor)
assert isinstance(pooler[1]["items"], torch.Tensor)
def test_slice_mm_payload_dict():
hidden = torch.randn(6, 4)
mm_cpu = {"nested": {"a": torch.randn(6, 2)}}
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
mm_cpu,
query_start_loc_np=np.array([0, 3]),
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
num_reqs=2,
)
assert pooler[1]["nested"]["a"].shape == (3, 2)
def test_build_pooler_output_flattens_nested_payload_for_msgspec():
hidden = torch.randn(4, 4)
mm_cpu = {"codes": {"audio": torch.randn(4, 16)}}
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
mm_cpu,
query_start_loc_np=np.array([0, 2]),
num_scheduled_tokens=np.array([2, 2], dtype=np.int32),
num_reqs=2,
)
assert "codes" not in pooler[0]
assert pooler[0]["codes.audio"].shape == (2, 16)
assert pooler[1]["codes.audio"].shape == (2, 16)
def test_build_pooler_output_preserves_qwen3_nested_payload():
hidden = torch.randn(4, 4)
mm = {
"hidden_states": {
"layers": {
0: torch.randn(4, 4),
24: torch.randn(4, 4),
},
},
"embed": {
"tts_bos": [torch.randn(1, 1, 4)],
"tts_eos": [torch.randn(1, 1, 4)],
"tts_pad": [torch.randn(1, 1, 4)],
},
}
mm_cpu = _async_copy_mm(mm, total_tokens=4)
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
hidden,
mm_cpu,
query_start_loc_np=np.array([0, 2]),
num_scheduled_tokens=np.array([2, 2], dtype=np.int32),
num_reqs=2,
)
payload = unflatten_payload(pooler[0])
assert payload["hidden_states"]["layers"][0].shape == (2, 4)
assert payload["hidden_states"]["layers"][24].shape == (2, 4)
assert payload["embed"]["tts_bos"].shape == (1, 1, 4)
def test_clamp_sampling_prompt_token_ids_to_logits_vocab():
prompt_token_ids = torch.tensor([[3, 99, 152064]])
input_batch = SimpleNamespace(
vocab_size=152064,
sampling_metadata=SimpleNamespace(
no_penalties=False,
prompt_token_ids=prompt_token_ids,
),
)
OmniARModelRunner._clamp_sampling_prompt_token_ids(
input_batch,
logits_vocab_size=10,
)
assert prompt_token_ids.tolist() == [[3, 9, 9]]
def test_sample_clamps_prompt_token_ids_from_actual_logits_vocab():
runner = object.__new__(OmniARModelRunner)
class Model:
def compute_logits(self, *_args, **_kwargs):
return torch.empty(1, 7)
runner.model = Model()
prompt_token_ids = torch.tensor([[3, 99, 152064]])
input_batch = SimpleNamespace(
vocab_size=152064,
sampling_metadata=SimpleNamespace(
no_penalties=False,
prompt_token_ids=prompt_token_ids,
),
)
sample_seen_prompt_ids = []
def sample(text_hidden, input_batch_arg, grammar_output):
assert grammar_output is None
runner.model.compute_logits(text_hidden)
sample_seen_prompt_ids.append(input_batch_arg.sampling_metadata.prompt_token_ids.clone())
return "sampler", "num_sampled", "num_rejected"
runner.sample = sample
result = runner._sample_with_prompt_token_compat(
torch.empty(1, 4),
input_batch,
None,
)
assert result == ("sampler", "num_sampled", "num_rejected")
assert prompt_token_ids.tolist() == [[3, 6, 6]]
assert sample_seen_prompt_ids[0].tolist() == [[3, 6, 6]]
assert runner.model.compute_logits.__func__ is Model.compute_logits
assert "compute_logits" not in runner.model.__dict__
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
from pathlib import Path
from types import SimpleNamespace
def _load_add_forward_compat_kwargs():
root = Path(__file__).resolve().parents[2]
module_path = root / "vllm_omni" / "worker_v2" / "forward_compat.py"
spec = importlib.util.spec_from_file_location("omni_forward_compat", module_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.add_forward_compat_kwargs
def test_forward_compat_kwargs_include_sampling_metadata_and_sampler():
add_forward_compat_kwargs = _load_add_forward_compat_kwargs()
model_inputs = {"input_ids": object()}
sampling_metadata = object()
sampler = object()
input_batch = SimpleNamespace(
sampling_metadata=sampling_metadata,
logits_indices=[3, 7],
)
add_forward_compat_kwargs(model_inputs, input_batch, sampler)
assert model_inputs["sampling_metadata"] is sampling_metadata
assert model_inputs["logits_index"] == [3, 7]
assert model_inputs["sampler"] is sampler
def test_forward_compat_kwargs_do_not_clobber_explicit_values():
add_forward_compat_kwargs = _load_add_forward_compat_kwargs()
model_inputs = {
"sampling_metadata": "explicit-metadata",
"logits_index": "explicit-logits",
"sampler": "explicit-sampler",
}
input_batch = SimpleNamespace(
sampling_metadata="batch-metadata",
logits_indices="batch-logits",
)
add_forward_compat_kwargs(model_inputs, input_batch, sampler="runner-sampler")
assert model_inputs["sampling_metadata"] == "explicit-metadata"
assert model_inputs["logits_index"] == "explicit-logits"
assert model_inputs["sampler"] == "explicit-sampler"
if __name__ == "__main__":
test_forward_compat_kwargs_include_sampling_metadata_and_sampler()
test_forward_compat_kwargs_do_not_clobber_explicit_values()
@@ -0,0 +1,260 @@
"""Tests for OmniGenerationModelRunner.sample_tokens (V2).
Covers the core pooler_output construction paths via _build_pooler_output:
- OmniOutput with batched tensor multimodal_outputs → per-request slicing
- OmniOutput with list multimodal_outputs → direct mapping (including None)
- OmniOutput with dict scalar values → broadcast to all requests
- None model output → returns None
- Non-dict multimodal_outputs → [None] * num_reqs
- sampled_token_ids always emits empty lists per request (no token sampling)
- req_states.num_computed_tokens updated to prompt_len after sample_tokens
"""
import unittest
from unittest.mock import MagicMock
import numpy as np
import torch
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.outputs import OmniModelRunnerOutput
pytestmark = []
class _FakeInputBatch:
"""Minimal input batch for sample_tokens."""
def __init__(self, num_reqs: int = 1, req_ids: list[str] | None = None):
self.num_reqs = num_reqs
self.req_ids = req_ids or [f"req-{i}" for i in range(num_reqs)]
self.idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
class _FakeStagedField:
"""Minimal mock for req_states fields that support staged writes."""
def __init__(self, data: np.ndarray):
self.np = data
self._staged: list[tuple[int, int]] = []
def stage_write_elem(self, idx: int, value: int) -> None:
self._staged.append((idx, value))
def apply_write(self) -> None:
for idx, value in self._staged:
self.np[idx] = value
self._staged.clear()
class _FakeNpField:
"""Minimal mock for req_states fields with .np attribute."""
def __init__(self, data: np.ndarray):
self.np = data
def _make_omni_output(multimodal_outputs: dict | None = None) -> OmniOutput:
"""Create an OmniOutput with given multimodal_outputs."""
return OmniOutput(
text_hidden_states=torch.zeros(1),
multimodal_outputs=multimodal_outputs,
)
def _make_runner(
model_output,
num_reqs: int = 1,
prompt_len: int = 10,
):
"""Build a minimal OmniGenerationModelRunner for sample_tokens testing."""
from vllm_omni.worker_v2.omni_generation_model_runner import (
OmniGenerationModelRunner,
)
runner = object.__new__(OmniGenerationModelRunner)
runner.device = torch.device("cpu")
mc = MagicMock()
del mc.eos_token_id
mc.hf_text_config = None
runner.model_config = mc
runner.postprocess = lambda *a, **kw: None
input_batch = _FakeInputBatch(num_reqs)
runner._gen_model_output = model_output
runner._gen_input_batch = input_batch
runner._gen_kv_connector_output = None
runner.execute_model_state = None
req_states = MagicMock()
req_states.prompt_len = _FakeNpField(
np.full(num_reqs, prompt_len, dtype=np.int32),
)
req_states.num_computed_tokens = _FakeStagedField(
np.zeros(num_reqs, dtype=np.int32),
)
runner.req_states = req_states
return runner
class TestSampleTokensTensorOutput(unittest.TestCase):
def test_single_request(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"model_outputs": torch.randn(1, 4, 8)})
runner = _make_runner(output, num_reqs=1)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert isinstance(result, OmniModelRunnerOutput)
assert len(result.pooler_output) == 1
assert result.pooler_output[0]["model_outputs"].shape == (4, 8)
def test_multi_request(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"model_outputs": torch.randn(3, 2, 5)})
runner = _make_runner(output, num_reqs=3)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.pooler_output) == 3
for i in range(3):
assert result.pooler_output[i]["model_outputs"].shape == (2, 5)
class TestSampleTokensListOutput(unittest.TestCase):
def test_list_of_tensors(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"model_outputs": [torch.randn(3, 2)]})
runner = _make_runner(output, num_reqs=1)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.pooler_output) == 1
assert result.pooler_output[0]["model_outputs"].shape == (3, 2)
def test_list_with_none(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"model_outputs": [None]})
runner = _make_runner(output, num_reqs=1)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.pooler_output) == 1
assert result.pooler_output[0]["model_outputs"] is None
class TestSampleTokensDictOutput(unittest.TestCase):
def test_dict_with_batched_tensor(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"audio": torch.randn(2, 16000), "sr": 24000})
runner = _make_runner(output, num_reqs=2)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.pooler_output) == 2
assert result.pooler_output[0]["audio"].shape == (16000,)
assert result.pooler_output[1]["audio"].shape == (16000,)
assert result.pooler_output[0]["sr"] == 24000
def test_dict_with_list_values(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"chunks": [torch.randn(10), torch.randn(20)]})
runner = _make_runner(output, num_reqs=2)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.pooler_output) == 2
assert result.pooler_output[0]["chunks"].shape == (10,)
assert result.pooler_output[1]["chunks"].shape == (20,)
class TestSampleTokensNoneOutput(unittest.TestCase):
def test_none_model_output(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
runner = _make_runner(None, num_reqs=1)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert result is None
class TestNonDictMultimodalOutputs(unittest.TestCase):
"""When multimodal_outputs is None or non-dict, pooler_output is [None]*num_reqs."""
def test_none_multimodal_outputs(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output(multimodal_outputs=None)
runner = _make_runner(output, num_reqs=2)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.pooler_output) == 2
assert result.pooler_output[0] is None
assert result.pooler_output[1] is None
class TestSampledTokenIds(unittest.TestCase):
def test_empty_sampled_token_ids_per_request(self):
"""Generation models emit empty sampled_token_ids (no token sampling)."""
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output({"model_outputs": torch.randn(3, 2)})
runner = _make_runner(output, num_reqs=3)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert len(result.sampled_token_ids) == 3
for ids in result.sampled_token_ids:
assert ids == []
class TestReqStatesUpdate(unittest.TestCase):
"""Verify that sample_tokens marks all tokens as computed."""
def test_num_computed_tokens_set_to_prompt_len(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
prompt_len = 15
output = _make_omni_output({"model_outputs": torch.randn(2, 4)})
runner = _make_runner(output, num_reqs=2, prompt_len=prompt_len)
OmniGenerationModelRunner.sample_tokens(runner)
for i in range(2):
assert runner.req_states.num_computed_tokens.np[i] == prompt_len
class TestMultimodalOutputsPassthrough(unittest.TestCase):
"""multimodal_outputs is a per-request list (tensor-only) on OmniModelRunnerOutput.
OmniGenerationScheduler indexes it as mm_outputs[req_index], so it must be a
list (not the raw dict). Each entry mirrors the per-request pooler payload.
"""
def test_multimodal_outputs_on_result(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
mm = {"audio": [torch.randn(10)]}
output = _make_omni_output(mm)
runner = _make_runner(output, num_reqs=1)
result = OmniGenerationModelRunner.sample_tokens(runner)
assert isinstance(result.multimodal_outputs, list)
assert len(result.multimodal_outputs) == 1
assert "audio" in result.multimodal_outputs[0]
assert torch.is_tensor(result.multimodal_outputs[0]["audio"])
def test_none_multimodal_outputs_becomes_empty_dict(self):
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
output = _make_omni_output(multimodal_outputs=None)
runner = _make_runner(output, num_reqs=1)
result = OmniGenerationModelRunner.sample_tokens(runner)
# No multimodal data -> one empty dict per request.
assert result.multimodal_outputs == [{}]
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,137 @@
"""Unit tests for OmniGPUModelRunner v2 overrides."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.worker_v2.omni_model_runner import OmniGPUModelRunner, _needs_capture_tensor_unwrap
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _DummyInputBatch:
def __init__(self, idx_mapping=None):
self.idx_mapping_np = idx_mapping or []
def _make_runner():
"""Create an OmniGPUModelRunner without calling __init__."""
runner = object.__new__(OmniGPUModelRunner)
runner.model = MagicMock()
runner.req_states = SimpleNamespace(req_id_to_index={"r1": 0, "r2": 1})
runner.execute_model_state = None
return runner
def test_finish_requests_calls_remove_for_finished():
runner = _make_runner()
mock_state = MagicMock()
runner.model_state = mock_state
sched_output = SimpleNamespace(
finished_req_ids={"r1"},
preempted_req_ids=set(),
)
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
runner.finish_requests(sched_output)
mock_state.remove_request.assert_called_once_with(0)
def test_finish_requests_calls_remove_for_preempted():
runner = _make_runner()
mock_state = MagicMock()
runner.model_state = mock_state
sched_output = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids={"r2"},
)
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
runner.finish_requests(sched_output)
mock_state.remove_request.assert_called_once_with(1)
def test_finish_requests_ignores_unknown_req_ids():
runner = _make_runner()
mock_state = MagicMock()
runner.model_state = mock_state
sched_output = SimpleNamespace(
finished_req_ids={"unknown"},
preempted_req_ids=set(),
)
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
runner.finish_requests(sched_output)
mock_state.remove_request.assert_not_called()
def test_finish_requests_handles_both_finished_and_preempted():
runner = _make_runner()
mock_state = MagicMock()
runner.model_state = mock_state
sched_output = SimpleNamespace(
finished_req_ids={"r1"},
preempted_req_ids={"r2"},
)
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
runner.finish_requests(sched_output)
assert mock_state.remove_request.call_count == 2
def test_thinker_stage_needs_capture_tensor_unwrap():
assert _needs_capture_tensor_unwrap(SimpleNamespace(model_stage="thinker"))
assert not _needs_capture_tensor_unwrap(SimpleNamespace(model_stage="talker"))
def test_capture_model_unwraps_tuple_outputs():
runner = object.__new__(OmniGPUModelRunner)
hidden = torch.ones(1, 2)
def original_forward():
return hidden, {"layers": {}}
runner.model = SimpleNamespace(forward=original_forward)
runner._model_returns_tuple = True
runner._exclude_full_graph = False
def capture_model(_self):
assert torch.equal(runner.model.forward(), hidden)
return 3
with patch.object(type(runner).__bases__[0], "capture_model", capture_model):
assert runner.capture_model() == 3
assert runner.model.forward is original_forward
def test_capture_model_unwraps_omni_outputs():
runner = object.__new__(OmniGPUModelRunner)
hidden = torch.ones(1, 2)
def original_forward():
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
runner.model = SimpleNamespace(forward=original_forward)
runner._model_returns_tuple = True
runner._exclude_full_graph = False
def capture_model(_self):
assert torch.equal(runner.model.forward(), hidden)
return 5
with patch.object(type(runner).__bases__[0], "capture_model", capture_model):
assert runner.capture_model() == 5
assert runner.model.forward is original_forward
+334
View File
@@ -0,0 +1,334 @@
"""Unit tests for OmniModelState core methods and plugin dispatch."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm_omni.engine.serialization import serialize_additional_information
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
from vllm_omni.worker_v2.model_states.plugin import OmniModelStatePlugin
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# ---------------------------------------------------------------
# Stubs
# ---------------------------------------------------------------
class _DummyInputBatch:
def __init__(self, indices):
self.idx_mapping_np = indices
self.num_reqs = len(indices)
self.num_scheduled_tokens = [1] * len(indices)
self.query_start_loc_np = list(range(len(indices)))
class _DummyReqState:
pass
class _SpyPlugin(OmniModelStatePlugin):
"""Plugin that records calls for verification."""
def __init__(self):
self.add_calls: list = []
self.remove_calls: list = []
self.prepare_calls: list = []
self.postprocess_calls: list = []
def on_add_request(self, req_index, new_req_data):
self.add_calls.append((req_index, new_req_data))
def on_remove_request(self, req_index):
self.remove_calls.append(req_index)
def prepare_extra_inputs(self, input_batch, req_states):
self.prepare_calls.append(True)
return {"plugin_key": "plugin_value"}
def postprocess(self, text_hidden, multimodal_outputs, input_batch, req_states):
self.postprocess_calls.append(True)
return text_hidden, multimodal_outputs
def _make_state(
max_num_reqs=4, has_preprocess=False, has_postprocess=False, have_multimodal_outputs=False, plugins=None
):
"""Create an OmniModelState without calling real __init__."""
state = object.__new__(OmniModelState)
# Minimal model mock
model = MagicMock()
model.has_preprocess = has_preprocess
model.has_postprocess = has_postprocess
model.have_multimodal_outputs = have_multimodal_outputs
model.preprocess_in_forward = False
model.get_omni_plugins = MagicMock(return_value=[])
state.model = model
# scheduler_config mock
state.scheduler_config = SimpleNamespace(max_num_seqs=max_num_reqs)
# Skip DefaultModelState.__init__ side effects
state.has_preprocess = has_preprocess
state.has_postprocess = has_postprocess
state.have_multimodal_outputs = have_multimodal_outputs
state.plugins = plugins or []
from vllm_omni.worker_v2.model_states.intermediate_buffer import (
OmniIntermediateBuffer,
)
state.intermediate_buffer = OmniIntermediateBuffer(max_num_reqs)
state._static_inputs_embeds = None
return state
def _make_new_req_data(req_id="r1"):
return SimpleNamespace(
req_id=req_id,
mm_features=[],
)
# ---------------------------------------------------------------
# add_request / remove_request
# ---------------------------------------------------------------
def test_add_request_populates_buffer():
state = _make_state()
req = _make_new_req_data("r1")
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
state.add_request(0, req)
assert state.intermediate_buffer.buffers[0]["req_id"] == "r1"
def test_add_request_dispatches_to_plugins():
plugin = _SpyPlugin()
state = _make_state(plugins=[plugin])
req = _make_new_req_data("r1")
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
state.add_request(0, req)
assert len(plugin.add_calls) == 1
assert plugin.add_calls[0][0] == 0
def test_add_request_unflattens_serialized_additional_information():
state = _make_state()
hidden = torch.randn(2, 4)
embeds = torch.randn(3, 4)
req = SimpleNamespace(
req_id="r1",
mm_features=[],
additional_information=serialize_additional_information(
{
"hidden_states": {"output": hidden},
"embed": {"prefill": embeds},
"ids": {"prompt": [1, 2, 3], "output": [4]},
}
),
)
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
state.add_request(0, req)
buf = state.intermediate_buffer.buffers[0]
assert torch.equal(buf["hidden_states"]["output"], hidden)
assert torch.equal(buf["embed"]["prefill"], embeds)
assert buf["ids"]["prompt"] == [1, 2, 3]
assert buf["ids"]["output"] == [4]
def test_remove_request_clears_buffer():
state = _make_state()
req = _make_new_req_data("r1")
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
state.add_request(0, req)
state.remove_request(0)
assert state.intermediate_buffer.buffers[0] == {}
def test_remove_request_dispatches_to_plugins():
plugin = _SpyPlugin()
state = _make_state(plugins=[plugin])
state.remove_request(2)
assert plugin.remove_calls == [2]
def test_intermediate_buffer_update_merges_nested_and_tuple_keys():
state = _make_state()
trailing = torch.randn(2, 4)
last = torch.randn(4)
codes = torch.ones(1, 16, dtype=torch.long)
state.intermediate_buffer.buffers[0] = {
"req_id": "r1",
"hidden_states": {"trailing_text": trailing},
}
state.intermediate_buffer.update(
0,
{
("codes", "audio"): codes,
"hidden_states": {"last": last},
},
{("codes", "audio"), ("hidden_states", "last")},
)
buf = state.intermediate_buffer.buffers[0]
assert ("codes", "audio") not in buf
assert torch.equal(buf["codes"]["audio"], codes)
assert torch.equal(buf["hidden_states"]["trailing_text"], trailing)
assert torch.equal(buf["hidden_states"]["last"], last)
# ---------------------------------------------------------------
# prepare_inputs
# ---------------------------------------------------------------
def test_prepare_inputs_injects_buffer_and_runtime_info():
state = _make_state()
req = _make_new_req_data("r1")
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
state.add_request(0, req)
batch = _DummyInputBatch([0])
with patch.object(type(state).__bases__[0], "prepare_inputs", return_value={}):
result = state.prepare_inputs(batch, _DummyReqState())
assert "model_intermediate_buffer" in result
assert "runtime_additional_information" in result
assert len(result["model_intermediate_buffer"]) == 1
assert result["model_intermediate_buffer"][0]["req_id"] == "r1"
def test_prepare_inputs_merges_plugin_extra():
plugin = _SpyPlugin()
state = _make_state(plugins=[plugin])
batch = _DummyInputBatch([])
with patch.object(type(state).__bases__[0], "prepare_inputs", return_value={"base": True}):
result = state.prepare_inputs(batch, _DummyReqState())
assert result["base"] is True
assert result["plugin_key"] == "plugin_value"
# ---------------------------------------------------------------
# prepare_dummy_inputs
# ---------------------------------------------------------------
def test_prepare_dummy_inputs_has_buffer_keys():
state = _make_state()
with patch.object(type(state).__bases__[0], "prepare_dummy_inputs", return_value={}):
result = state.prepare_dummy_inputs(num_reqs=2, num_tokens=16)
assert "model_intermediate_buffer" in result
assert len(result["model_intermediate_buffer"]) == 2
# ---------------------------------------------------------------
# postprocess_model_output
# ---------------------------------------------------------------
def test_postprocess_omni_output_with_multimodal():
state = _make_state(have_multimodal_outputs=True)
hidden = torch.randn(4, 8)
mm = {"audio": torch.randn(4, 2)}
omni = OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
batch = _DummyInputBatch([0])
th, mo = state.postprocess_model_output(omni, batch, _DummyReqState())
assert th is hidden
assert mo is mm
def test_postprocess_tuple_output():
state = _make_state(have_multimodal_outputs=False)
del state.model.make_omni_output
hidden = torch.randn(4, 8)
batch = _DummyInputBatch([0])
th, mo = state.postprocess_model_output((hidden, {}), batch, _DummyReqState())
assert th is hidden
assert mo == {}
def test_postprocess_raw_tensor():
state = _make_state()
del state.model.make_omni_output
hidden = torch.randn(4, 8)
batch = _DummyInputBatch([0])
th, mo = state.postprocess_model_output(hidden, batch, _DummyReqState())
assert th is hidden
assert mo == {}
def test_postprocess_calls_make_omni_output_when_not_omni():
"""When model output is not OmniOutput but model has make_omni_output, it should be called."""
state = _make_state(have_multimodal_outputs=True)
hidden = torch.randn(4, 8)
mm = {"audio": torch.randn(4, 2)}
expected_omni = OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
state.model.make_omni_output = MagicMock(return_value=expected_omni)
batch = _DummyInputBatch([0])
th, mo = state.postprocess_model_output((hidden, {}), batch, _DummyReqState())
state.model.make_omni_output.assert_called_once()
assert th is hidden
def test_postprocess_dispatches_to_plugins():
plugin = _SpyPlugin()
state = _make_state(plugins=[plugin])
hidden = torch.randn(4, 8)
batch = _DummyInputBatch([0])
state.postprocess_model_output(hidden, batch, _DummyReqState())
assert len(plugin.postprocess_calls) == 1
def test_run_postprocess_does_not_duplicate_hidden_states_kwarg():
state = _make_state(has_postprocess=True)
state.model.gpu_resident_buffer_keys = set()
seen = {}
trailing = torch.ones(1, 2)
def postprocess(hidden_states, **info):
seen["hidden_states"] = hidden_states
seen["info"] = info
return {"hidden_states": {"last": hidden_states[-1].detach()}}
state.model.postprocess = postprocess
state.intermediate_buffer.buffers[0] = {
"req_id": "r1",
"hidden_states": {"trailing_text": trailing},
"meta": {"codec_streaming": True},
}
hidden = torch.randn(1, 2)
state.run_postprocess(hidden, _DummyInputBatch([0]))
assert torch.equal(seen["hidden_states"], hidden)
assert "hidden_states" not in seen["info"]
assert seen["info"]["meta"] == {"codec_streaming": True}
assert torch.equal(state.intermediate_buffer.buffers[0]["hidden_states"]["trailing_text"], trailing)
assert torch.equal(state.intermediate_buffer.buffers[0]["hidden_states"]["last"], hidden[-1])
@@ -0,0 +1,71 @@
"""Unit tests for OmniModelStatePlugin interface correctness."""
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from vllm_omni.worker_v2.model_states.plugin import OmniModelStatePlugin
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _ConcretePlugin(OmniModelStatePlugin):
"""Minimal concrete plugin for testing the ABC."""
def __init__(self):
self.added: list[tuple[int, Any]] = []
self.removed: list[int] = []
def on_add_request(self, req_index, new_req_data):
self.added.append((req_index, new_req_data))
def on_remove_request(self, req_index):
self.removed.append(req_index)
def postprocess(self, text_hidden, multimodal_outputs, input_batch, req_states):
return text_hidden, multimodal_outputs
def test_concrete_plugin_instantiates():
plugin = _ConcretePlugin()
assert isinstance(plugin, OmniModelStatePlugin)
def test_on_add_request_called():
plugin = _ConcretePlugin()
plugin.on_add_request(0, SimpleNamespace(req_id="r1"))
assert len(plugin.added) == 1
assert plugin.added[0][0] == 0
def test_on_remove_request_called():
plugin = _ConcretePlugin()
plugin.on_remove_request(3)
assert plugin.removed == [3]
def test_prepare_extra_inputs_default_empty():
plugin = _ConcretePlugin()
result = plugin.prepare_extra_inputs(SimpleNamespace(), SimpleNamespace())
assert result == {}
def test_dummy_run_default_noop():
plugin = _ConcretePlugin()
plugin.dummy_run(None, 128)
def test_postprocess_returns_inputs():
plugin = _ConcretePlugin()
hidden = torch.randn(4, 8)
mm = {"audio": torch.randn(4, 2)}
out_h, out_mm = plugin.postprocess(hidden, mm, None, None)
assert out_h is hidden
assert out_mm is mm
def test_abc_cannot_instantiate_directly():
with pytest.raises(TypeError):
OmniModelStatePlugin()