[MR V2] Migrate to Model Runner V2
Squashed migration rebased onto current main. Collapses the 46-commit dev/migrate-MR-v2 history (10 merge commits + superseded pre-rebase work) into a single commit, so the diff reflects only the genuine MR-V2 feature (worker_v2/ runners + model_states, omni schedulers, stage engine pipeline, pooling_output serialization bridge). Content already merged to main separately (step_audio2, tts_adapters, npu/_310p, etc.) drops out. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cead615288
commit
9678ae58cf
74 files changed
+8244
-95
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,83 @@
|
||||
"""Unit tests for init_omni_model_state factory dispatch."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.worker_v2.model_states import (
|
||||
_OMNI_ARCHITECTURES,
|
||||
init_omni_model_state,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_vllm_config(architectures):
|
||||
model_config = SimpleNamespace(architectures=architectures)
|
||||
return SimpleNamespace(model_config=model_config)
|
||||
|
||||
|
||||
@patch("vllm_omni.worker_v2.model_states.omni_model_state.OmniModelState.__init__", return_value=None)
|
||||
def test_qwen3_omni_dispatches_to_omni_model_state(mock_init):
|
||||
cfg = _make_vllm_config(["Qwen3OmniMoeForConditionalGeneration"])
|
||||
model = MagicMock()
|
||||
device = torch.device("cpu")
|
||||
|
||||
state = init_omni_model_state(cfg, model, None, device)
|
||||
|
||||
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
|
||||
|
||||
assert isinstance(state, OmniModelState)
|
||||
|
||||
|
||||
@patch("vllm_omni.worker_v2.model_states.omni_model_state.OmniModelState.__init__", return_value=None)
|
||||
def test_mammoth_dispatches_to_omni_model_state(mock_init):
|
||||
cfg = _make_vllm_config(["MammothModa2ForConditionalGeneration"])
|
||||
model = MagicMock()
|
||||
device = torch.device("cpu")
|
||||
|
||||
state = init_omni_model_state(cfg, model, None, device)
|
||||
|
||||
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
|
||||
|
||||
assert isinstance(state, OmniModelState)
|
||||
|
||||
|
||||
@patch("vllm_omni.worker_v2.model_states._upstream_init_model_state")
|
||||
def test_unknown_arch_delegates_to_upstream(mock_upstream):
|
||||
mock_upstream.return_value = MagicMock()
|
||||
cfg = _make_vllm_config(["LlamaForCausalLM"])
|
||||
model = MagicMock()
|
||||
device = torch.device("cpu")
|
||||
|
||||
state = init_omni_model_state(cfg, model, None, device)
|
||||
|
||||
mock_upstream.assert_called_once_with(cfg, model, None, device)
|
||||
assert state is mock_upstream.return_value
|
||||
|
||||
|
||||
def test_omni_architectures_set_contains_expected():
|
||||
expected = {
|
||||
"Qwen3OmniMoeForConditionalGeneration",
|
||||
"Qwen2_5OmniForConditionalGeneration",
|
||||
"MammothModa2ForConditionalGeneration",
|
||||
"MiMoAudioForConditionalGeneration",
|
||||
"MammothModa2ARForConditionalGeneration",
|
||||
"Qwen3TTSTalkerForConditionalGeneration",
|
||||
"Qwen3TTSCode2Wav",
|
||||
}
|
||||
assert _OMNI_ARCHITECTURES == expected
|
||||
|
||||
|
||||
@patch("vllm_omni.worker_v2.model_states._upstream_init_model_state")
|
||||
def test_none_architectures_delegates_to_upstream(mock_upstream):
|
||||
mock_upstream.return_value = MagicMock()
|
||||
cfg = _make_vllm_config(None)
|
||||
model = MagicMock()
|
||||
device = torch.device("cpu")
|
||||
|
||||
init_omni_model_state(cfg, model, None, device)
|
||||
|
||||
mock_upstream.assert_called_once()
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Unit tests for OmniIntermediateBuffer lifecycle."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.worker_v2.model_states.intermediate_buffer import (
|
||||
OmniIntermediateBuffer,
|
||||
_resolve_additional_information,
|
||||
_resolve_prompt_embeds,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _DummyInputBatch:
|
||||
def __init__(self, idx_mapping):
|
||||
self.idx_mapping_np = idx_mapping
|
||||
|
||||
|
||||
def _make_new_req_data(req_id="r1", prompt_embeds=None, mm_features=None, additional_information=None):
|
||||
ns = SimpleNamespace(
|
||||
req_id=req_id,
|
||||
mm_features=mm_features or [],
|
||||
)
|
||||
if prompt_embeds is not None:
|
||||
ns.prompt_embeds = prompt_embeds
|
||||
if additional_information is not None:
|
||||
ns.additional_information = additional_information
|
||||
return ns
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# _resolve_prompt_embeds
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_prompt_embeds_tensor():
|
||||
t = torch.randn(3, 4, device="cpu")
|
||||
result = _resolve_prompt_embeds(t)
|
||||
assert isinstance(result, torch.Tensor)
|
||||
assert result.shape == (3, 4)
|
||||
assert result.is_contiguous()
|
||||
|
||||
|
||||
def test_resolve_prompt_embeds_none():
|
||||
assert _resolve_prompt_embeds(None) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# _resolve_additional_information
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_additional_information_dict_passthrough():
|
||||
d = {"key": "val"}
|
||||
assert _resolve_additional_information(d) == d
|
||||
|
||||
|
||||
def test_resolve_additional_information_none():
|
||||
assert _resolve_additional_information(None) == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# OmniIntermediateBuffer core lifecycle
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_add_and_remove():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=4)
|
||||
req = _make_new_req_data(req_id="r1")
|
||||
buf.add_request(0, req)
|
||||
assert buf.buffers[0]["req_id"] == "r1"
|
||||
|
||||
buf.remove_request(0)
|
||||
assert buf.buffers[0] == {}
|
||||
|
||||
|
||||
def test_add_with_prompt_embeds():
|
||||
pe = torch.randn(2, 8)
|
||||
req = _make_new_req_data(req_id="r1", prompt_embeds=pe)
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=4)
|
||||
buf.add_request(0, req)
|
||||
assert "prompt_embeds_cpu" in buf.buffers[0]
|
||||
assert buf.buffers[0]["prompt_embeds_cpu"].shape == (2, 8)
|
||||
|
||||
|
||||
def test_add_with_mm_features():
|
||||
features = [torch.randn(3, 4)]
|
||||
req = _make_new_req_data(req_id="r1", mm_features=features)
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=4)
|
||||
buf.add_request(1, req)
|
||||
assert buf.buffers[1]["mm_features"] is features
|
||||
|
||||
|
||||
def test_add_with_additional_information():
|
||||
info = {"audio_embeds": torch.randn(5)}
|
||||
req = _make_new_req_data(req_id="r1", additional_information=info)
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=4)
|
||||
buf.add_request(0, req)
|
||||
assert torch.allclose(buf.buffers[0]["audio_embeds"], info["audio_embeds"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# gather
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_gather_returns_batch_order():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=4)
|
||||
buf.add_request(0, _make_new_req_data(req_id="r0"))
|
||||
buf.add_request(2, _make_new_req_data(req_id="r2"))
|
||||
buf.add_request(3, _make_new_req_data(req_id="r3"))
|
||||
|
||||
batch = _DummyInputBatch(idx_mapping=[2, 0, 3])
|
||||
gathered = buf.gather(batch)
|
||||
assert [g["req_id"] for g in gathered] == ["r2", "r0", "r3"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# update
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_merges_tensors_to_cpu():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=2)
|
||||
buf.add_request(0, _make_new_req_data(req_id="r0"))
|
||||
|
||||
buf.update(0, {"hidden": torch.randn(4)})
|
||||
assert "hidden" in buf.buffers[0]
|
||||
assert buf.buffers[0]["hidden"].device == torch.device("cpu")
|
||||
|
||||
|
||||
def test_update_gpu_resident_keys():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=2)
|
||||
buf.add_request(0, _make_new_req_data(req_id="r0"))
|
||||
|
||||
t = torch.randn(4)
|
||||
buf.update(0, {"kv": t}, gpu_resident_keys={"kv"})
|
||||
assert "kv" in buf.buffers[0]
|
||||
|
||||
|
||||
def test_update_list_values():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=2)
|
||||
buf.add_request(0, _make_new_req_data(req_id="r0"))
|
||||
|
||||
items = [torch.randn(2), "text"]
|
||||
buf.update(0, {"items": items})
|
||||
stored = buf.buffers[0]["items"]
|
||||
assert isinstance(stored[0], torch.Tensor)
|
||||
assert stored[1] == "text"
|
||||
|
||||
|
||||
def test_update_empty_is_noop():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=2)
|
||||
buf.add_request(0, _make_new_req_data(req_id="r0"))
|
||||
before = dict(buf.buffers[0])
|
||||
buf.update(0, {})
|
||||
assert buf.buffers[0] == before
|
||||
|
||||
|
||||
def test_update_accumulates():
|
||||
buf = OmniIntermediateBuffer(max_num_reqs=2)
|
||||
buf.add_request(0, _make_new_req_data(req_id="r0"))
|
||||
buf.update(0, {"a": torch.tensor([1.0])})
|
||||
buf.update(0, {"b": torch.tensor([2.0])})
|
||||
assert "a" in buf.buffers[0] and "b" in buf.buffers[0]
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Unit tests for OmniARModelRunner v2."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.data_entry_keys import unflatten_payload
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.worker_v2.omni_ar_model_runner import (
|
||||
OmniARModelRunner,
|
||||
_async_copy_mm,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# _build_pooler_output_from_cpu (was _build_pooler_output)
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_reconstruct_raw_model_output_preserves_omni_multimodal_outputs():
|
||||
hidden = torch.randn(3, 4)
|
||||
latent = torch.randn(3, 4)
|
||||
raw = OmniARModelRunner._reconstruct_raw_model_output(
|
||||
hidden_states=hidden,
|
||||
multimodal_outputs={"latent": latent},
|
||||
aux=None,
|
||||
)
|
||||
|
||||
assert isinstance(raw, OmniOutput)
|
||||
assert raw.text_hidden_states is hidden
|
||||
assert raw.multimodal_outputs["latent"] is latent
|
||||
|
||||
|
||||
def test_reconstruct_raw_model_output_keeps_aux_tuple_without_multimodal_outputs():
|
||||
hidden = torch.randn(3, 4)
|
||||
aux = {"layers": torch.randn(3, 2)}
|
||||
raw = OmniARModelRunner._reconstruct_raw_model_output(
|
||||
hidden_states=hidden,
|
||||
multimodal_outputs=None,
|
||||
aux=aux,
|
||||
)
|
||||
|
||||
assert raw == (hidden, aux)
|
||||
|
||||
|
||||
def test_reconstruct_raw_model_output_ignores_empty_multimodal_outputs():
|
||||
hidden = torch.randn(3, 4)
|
||||
raw = OmniARModelRunner._reconstruct_raw_model_output(
|
||||
hidden_states=hidden,
|
||||
multimodal_outputs={},
|
||||
aux=None,
|
||||
)
|
||||
|
||||
assert raw is hidden
|
||||
|
||||
|
||||
def test_build_pooler_output_basic():
|
||||
"""Verify _build_pooler_output_from_cpu slices per-request hidden + mm."""
|
||||
hidden = torch.randn(6, 8)
|
||||
mm = {"audio": torch.randn(6, 2)}
|
||||
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
mm,
|
||||
query_start_loc_np=np.array([0, 3]),
|
||||
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
|
||||
num_reqs=2,
|
||||
)
|
||||
|
||||
assert len(pooler) == 2
|
||||
assert pooler[0]["hidden"].shape == (3, 8)
|
||||
assert pooler[1]["hidden"].shape == (3, 8)
|
||||
assert pooler[0]["audio"].shape == (3, 2)
|
||||
|
||||
|
||||
def test_build_pooler_output_empty_mm():
|
||||
hidden = torch.randn(4, 8)
|
||||
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
{},
|
||||
query_start_loc_np=np.array([0]),
|
||||
num_scheduled_tokens=np.array([4], dtype=np.int32),
|
||||
num_reqs=1,
|
||||
)
|
||||
assert len(pooler) == 1
|
||||
assert "hidden" in pooler[0]
|
||||
assert len(pooler[0]) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# _async_copy_mm (was _copy_mm_to_cpu)
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_copy_mm_to_cpu_tensor():
|
||||
total = 10
|
||||
t = torch.randn(10, 4)
|
||||
result = _async_copy_mm({"feat": t}, total)
|
||||
assert "feat" in result
|
||||
assert result["feat"].shape == (10, 4)
|
||||
assert result["feat"].device == torch.device("cpu")
|
||||
|
||||
|
||||
def test_copy_mm_to_cpu_dict():
|
||||
total = 10
|
||||
d = {"inner": torch.randn(10, 2)}
|
||||
result = _async_copy_mm({"nested": d}, total)
|
||||
assert "nested" in result
|
||||
assert "inner" in result["nested"]
|
||||
|
||||
|
||||
def test_copy_mm_to_cpu_list():
|
||||
result = _async_copy_mm({"items": [torch.randn(3), "text"]}, 10)
|
||||
assert "items" in result
|
||||
assert isinstance(result["items"][0], torch.Tensor)
|
||||
assert result["items"][1] == "text"
|
||||
|
||||
|
||||
def test_copy_mm_to_cpu_empty():
|
||||
assert _async_copy_mm({}, 10) == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Slicing via _build_pooler_output_from_cpu (was _slice_mm_payload)
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_slice_mm_payload_tensor():
|
||||
hidden = torch.randn(6, 4)
|
||||
mm_cpu = {"feat": torch.randn(6, 2)}
|
||||
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
mm_cpu,
|
||||
query_start_loc_np=np.array([0, 3]),
|
||||
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
|
||||
num_reqs=2,
|
||||
)
|
||||
assert pooler[0]["feat"].shape == (3, 2)
|
||||
assert pooler[1]["feat"].shape == (3, 2)
|
||||
|
||||
|
||||
def test_slice_mm_payload_list():
|
||||
hidden = torch.randn(6, 4)
|
||||
mm_cpu = {"items": [torch.randn(2), torch.randn(3)]}
|
||||
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
mm_cpu,
|
||||
query_start_loc_np=np.array([0, 3]),
|
||||
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
|
||||
num_reqs=2,
|
||||
)
|
||||
assert isinstance(pooler[0]["items"], torch.Tensor)
|
||||
assert isinstance(pooler[1]["items"], torch.Tensor)
|
||||
|
||||
|
||||
def test_slice_mm_payload_dict():
|
||||
hidden = torch.randn(6, 4)
|
||||
mm_cpu = {"nested": {"a": torch.randn(6, 2)}}
|
||||
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
mm_cpu,
|
||||
query_start_loc_np=np.array([0, 3]),
|
||||
num_scheduled_tokens=np.array([3, 3], dtype=np.int32),
|
||||
num_reqs=2,
|
||||
)
|
||||
assert pooler[1]["nested"]["a"].shape == (3, 2)
|
||||
|
||||
|
||||
def test_build_pooler_output_flattens_nested_payload_for_msgspec():
|
||||
hidden = torch.randn(4, 4)
|
||||
mm_cpu = {"codes": {"audio": torch.randn(4, 16)}}
|
||||
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
mm_cpu,
|
||||
query_start_loc_np=np.array([0, 2]),
|
||||
num_scheduled_tokens=np.array([2, 2], dtype=np.int32),
|
||||
num_reqs=2,
|
||||
)
|
||||
|
||||
assert "codes" not in pooler[0]
|
||||
assert pooler[0]["codes.audio"].shape == (2, 16)
|
||||
assert pooler[1]["codes.audio"].shape == (2, 16)
|
||||
|
||||
|
||||
def test_build_pooler_output_preserves_qwen3_nested_payload():
|
||||
hidden = torch.randn(4, 4)
|
||||
mm = {
|
||||
"hidden_states": {
|
||||
"layers": {
|
||||
0: torch.randn(4, 4),
|
||||
24: torch.randn(4, 4),
|
||||
},
|
||||
},
|
||||
"embed": {
|
||||
"tts_bos": [torch.randn(1, 1, 4)],
|
||||
"tts_eos": [torch.randn(1, 1, 4)],
|
||||
"tts_pad": [torch.randn(1, 1, 4)],
|
||||
},
|
||||
}
|
||||
|
||||
mm_cpu = _async_copy_mm(mm, total_tokens=4)
|
||||
pooler = OmniARModelRunner._build_pooler_output_from_cpu(
|
||||
hidden,
|
||||
mm_cpu,
|
||||
query_start_loc_np=np.array([0, 2]),
|
||||
num_scheduled_tokens=np.array([2, 2], dtype=np.int32),
|
||||
num_reqs=2,
|
||||
)
|
||||
|
||||
payload = unflatten_payload(pooler[0])
|
||||
assert payload["hidden_states"]["layers"][0].shape == (2, 4)
|
||||
assert payload["hidden_states"]["layers"][24].shape == (2, 4)
|
||||
assert payload["embed"]["tts_bos"].shape == (1, 1, 4)
|
||||
|
||||
|
||||
def test_clamp_sampling_prompt_token_ids_to_logits_vocab():
|
||||
prompt_token_ids = torch.tensor([[3, 99, 152064]])
|
||||
input_batch = SimpleNamespace(
|
||||
vocab_size=152064,
|
||||
sampling_metadata=SimpleNamespace(
|
||||
no_penalties=False,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
),
|
||||
)
|
||||
|
||||
OmniARModelRunner._clamp_sampling_prompt_token_ids(
|
||||
input_batch,
|
||||
logits_vocab_size=10,
|
||||
)
|
||||
|
||||
assert prompt_token_ids.tolist() == [[3, 9, 9]]
|
||||
|
||||
|
||||
def test_sample_clamps_prompt_token_ids_from_actual_logits_vocab():
|
||||
runner = object.__new__(OmniARModelRunner)
|
||||
|
||||
class Model:
|
||||
def compute_logits(self, *_args, **_kwargs):
|
||||
return torch.empty(1, 7)
|
||||
|
||||
runner.model = Model()
|
||||
prompt_token_ids = torch.tensor([[3, 99, 152064]])
|
||||
input_batch = SimpleNamespace(
|
||||
vocab_size=152064,
|
||||
sampling_metadata=SimpleNamespace(
|
||||
no_penalties=False,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
),
|
||||
)
|
||||
sample_seen_prompt_ids = []
|
||||
|
||||
def sample(text_hidden, input_batch_arg, grammar_output):
|
||||
assert grammar_output is None
|
||||
runner.model.compute_logits(text_hidden)
|
||||
sample_seen_prompt_ids.append(input_batch_arg.sampling_metadata.prompt_token_ids.clone())
|
||||
return "sampler", "num_sampled", "num_rejected"
|
||||
|
||||
runner.sample = sample
|
||||
|
||||
result = runner._sample_with_prompt_token_compat(
|
||||
torch.empty(1, 4),
|
||||
input_batch,
|
||||
None,
|
||||
)
|
||||
|
||||
assert result == ("sampler", "num_sampled", "num_rejected")
|
||||
assert prompt_token_ids.tolist() == [[3, 6, 6]]
|
||||
assert sample_seen_prompt_ids[0].tolist() == [[3, 6, 6]]
|
||||
assert runner.model.compute_logits.__func__ is Model.compute_logits
|
||||
assert "compute_logits" not in runner.model.__dict__
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
def _load_add_forward_compat_kwargs():
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
module_path = root / "vllm_omni" / "worker_v2" / "forward_compat.py"
|
||||
spec = importlib.util.spec_from_file_location("omni_forward_compat", module_path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.add_forward_compat_kwargs
|
||||
|
||||
|
||||
def test_forward_compat_kwargs_include_sampling_metadata_and_sampler():
|
||||
add_forward_compat_kwargs = _load_add_forward_compat_kwargs()
|
||||
model_inputs = {"input_ids": object()}
|
||||
sampling_metadata = object()
|
||||
sampler = object()
|
||||
input_batch = SimpleNamespace(
|
||||
sampling_metadata=sampling_metadata,
|
||||
logits_indices=[3, 7],
|
||||
)
|
||||
|
||||
add_forward_compat_kwargs(model_inputs, input_batch, sampler)
|
||||
|
||||
assert model_inputs["sampling_metadata"] is sampling_metadata
|
||||
assert model_inputs["logits_index"] == [3, 7]
|
||||
assert model_inputs["sampler"] is sampler
|
||||
|
||||
|
||||
def test_forward_compat_kwargs_do_not_clobber_explicit_values():
|
||||
add_forward_compat_kwargs = _load_add_forward_compat_kwargs()
|
||||
model_inputs = {
|
||||
"sampling_metadata": "explicit-metadata",
|
||||
"logits_index": "explicit-logits",
|
||||
"sampler": "explicit-sampler",
|
||||
}
|
||||
input_batch = SimpleNamespace(
|
||||
sampling_metadata="batch-metadata",
|
||||
logits_indices="batch-logits",
|
||||
)
|
||||
|
||||
add_forward_compat_kwargs(model_inputs, input_batch, sampler="runner-sampler")
|
||||
|
||||
assert model_inputs["sampling_metadata"] == "explicit-metadata"
|
||||
assert model_inputs["logits_index"] == "explicit-logits"
|
||||
assert model_inputs["sampler"] == "explicit-sampler"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_forward_compat_kwargs_include_sampling_metadata_and_sampler()
|
||||
test_forward_compat_kwargs_do_not_clobber_explicit_values()
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Tests for OmniGenerationModelRunner.sample_tokens (V2).
|
||||
|
||||
Covers the core pooler_output construction paths via _build_pooler_output:
|
||||
- OmniOutput with batched tensor multimodal_outputs → per-request slicing
|
||||
- OmniOutput with list multimodal_outputs → direct mapping (including None)
|
||||
- OmniOutput with dict scalar values → broadcast to all requests
|
||||
- None model output → returns None
|
||||
- Non-dict multimodal_outputs → [None] * num_reqs
|
||||
- sampled_token_ids always emits empty lists per request (no token sampling)
|
||||
- req_states.num_computed_tokens updated to prompt_len after sample_tokens
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.outputs import OmniModelRunnerOutput
|
||||
|
||||
pytestmark = []
|
||||
|
||||
|
||||
class _FakeInputBatch:
|
||||
"""Minimal input batch for sample_tokens."""
|
||||
|
||||
def __init__(self, num_reqs: int = 1, req_ids: list[str] | None = None):
|
||||
self.num_reqs = num_reqs
|
||||
self.req_ids = req_ids or [f"req-{i}" for i in range(num_reqs)]
|
||||
self.idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
|
||||
|
||||
|
||||
class _FakeStagedField:
|
||||
"""Minimal mock for req_states fields that support staged writes."""
|
||||
|
||||
def __init__(self, data: np.ndarray):
|
||||
self.np = data
|
||||
self._staged: list[tuple[int, int]] = []
|
||||
|
||||
def stage_write_elem(self, idx: int, value: int) -> None:
|
||||
self._staged.append((idx, value))
|
||||
|
||||
def apply_write(self) -> None:
|
||||
for idx, value in self._staged:
|
||||
self.np[idx] = value
|
||||
self._staged.clear()
|
||||
|
||||
|
||||
class _FakeNpField:
|
||||
"""Minimal mock for req_states fields with .np attribute."""
|
||||
|
||||
def __init__(self, data: np.ndarray):
|
||||
self.np = data
|
||||
|
||||
|
||||
def _make_omni_output(multimodal_outputs: dict | None = None) -> OmniOutput:
|
||||
"""Create an OmniOutput with given multimodal_outputs."""
|
||||
return OmniOutput(
|
||||
text_hidden_states=torch.zeros(1),
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
)
|
||||
|
||||
|
||||
def _make_runner(
|
||||
model_output,
|
||||
num_reqs: int = 1,
|
||||
prompt_len: int = 10,
|
||||
):
|
||||
"""Build a minimal OmniGenerationModelRunner for sample_tokens testing."""
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import (
|
||||
OmniGenerationModelRunner,
|
||||
)
|
||||
|
||||
runner = object.__new__(OmniGenerationModelRunner)
|
||||
runner.device = torch.device("cpu")
|
||||
|
||||
mc = MagicMock()
|
||||
del mc.eos_token_id
|
||||
mc.hf_text_config = None
|
||||
runner.model_config = mc
|
||||
|
||||
runner.postprocess = lambda *a, **kw: None
|
||||
|
||||
input_batch = _FakeInputBatch(num_reqs)
|
||||
runner._gen_model_output = model_output
|
||||
runner._gen_input_batch = input_batch
|
||||
runner._gen_kv_connector_output = None
|
||||
runner.execute_model_state = None
|
||||
|
||||
req_states = MagicMock()
|
||||
req_states.prompt_len = _FakeNpField(
|
||||
np.full(num_reqs, prompt_len, dtype=np.int32),
|
||||
)
|
||||
req_states.num_computed_tokens = _FakeStagedField(
|
||||
np.zeros(num_reqs, dtype=np.int32),
|
||||
)
|
||||
runner.req_states = req_states
|
||||
|
||||
return runner
|
||||
|
||||
|
||||
class TestSampleTokensTensorOutput(unittest.TestCase):
|
||||
def test_single_request(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"model_outputs": torch.randn(1, 4, 8)})
|
||||
runner = _make_runner(output, num_reqs=1)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert isinstance(result, OmniModelRunnerOutput)
|
||||
assert len(result.pooler_output) == 1
|
||||
assert result.pooler_output[0]["model_outputs"].shape == (4, 8)
|
||||
|
||||
def test_multi_request(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"model_outputs": torch.randn(3, 2, 5)})
|
||||
runner = _make_runner(output, num_reqs=3)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.pooler_output) == 3
|
||||
for i in range(3):
|
||||
assert result.pooler_output[i]["model_outputs"].shape == (2, 5)
|
||||
|
||||
|
||||
class TestSampleTokensListOutput(unittest.TestCase):
|
||||
def test_list_of_tensors(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"model_outputs": [torch.randn(3, 2)]})
|
||||
runner = _make_runner(output, num_reqs=1)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.pooler_output) == 1
|
||||
assert result.pooler_output[0]["model_outputs"].shape == (3, 2)
|
||||
|
||||
def test_list_with_none(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"model_outputs": [None]})
|
||||
runner = _make_runner(output, num_reqs=1)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.pooler_output) == 1
|
||||
assert result.pooler_output[0]["model_outputs"] is None
|
||||
|
||||
|
||||
class TestSampleTokensDictOutput(unittest.TestCase):
|
||||
def test_dict_with_batched_tensor(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"audio": torch.randn(2, 16000), "sr": 24000})
|
||||
runner = _make_runner(output, num_reqs=2)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.pooler_output) == 2
|
||||
assert result.pooler_output[0]["audio"].shape == (16000,)
|
||||
assert result.pooler_output[1]["audio"].shape == (16000,)
|
||||
assert result.pooler_output[0]["sr"] == 24000
|
||||
|
||||
def test_dict_with_list_values(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"chunks": [torch.randn(10), torch.randn(20)]})
|
||||
runner = _make_runner(output, num_reqs=2)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.pooler_output) == 2
|
||||
assert result.pooler_output[0]["chunks"].shape == (10,)
|
||||
assert result.pooler_output[1]["chunks"].shape == (20,)
|
||||
|
||||
|
||||
class TestSampleTokensNoneOutput(unittest.TestCase):
|
||||
def test_none_model_output(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
runner = _make_runner(None, num_reqs=1)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestNonDictMultimodalOutputs(unittest.TestCase):
|
||||
"""When multimodal_outputs is None or non-dict, pooler_output is [None]*num_reqs."""
|
||||
|
||||
def test_none_multimodal_outputs(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output(multimodal_outputs=None)
|
||||
runner = _make_runner(output, num_reqs=2)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.pooler_output) == 2
|
||||
assert result.pooler_output[0] is None
|
||||
assert result.pooler_output[1] is None
|
||||
|
||||
|
||||
class TestSampledTokenIds(unittest.TestCase):
|
||||
def test_empty_sampled_token_ids_per_request(self):
|
||||
"""Generation models emit empty sampled_token_ids (no token sampling)."""
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output({"model_outputs": torch.randn(3, 2)})
|
||||
runner = _make_runner(output, num_reqs=3)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert len(result.sampled_token_ids) == 3
|
||||
for ids in result.sampled_token_ids:
|
||||
assert ids == []
|
||||
|
||||
|
||||
class TestReqStatesUpdate(unittest.TestCase):
|
||||
"""Verify that sample_tokens marks all tokens as computed."""
|
||||
|
||||
def test_num_computed_tokens_set_to_prompt_len(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
prompt_len = 15
|
||||
output = _make_omni_output({"model_outputs": torch.randn(2, 4)})
|
||||
runner = _make_runner(output, num_reqs=2, prompt_len=prompt_len)
|
||||
|
||||
OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
for i in range(2):
|
||||
assert runner.req_states.num_computed_tokens.np[i] == prompt_len
|
||||
|
||||
|
||||
class TestMultimodalOutputsPassthrough(unittest.TestCase):
|
||||
"""multimodal_outputs is a per-request list (tensor-only) on OmniModelRunnerOutput.
|
||||
|
||||
OmniGenerationScheduler indexes it as mm_outputs[req_index], so it must be a
|
||||
list (not the raw dict). Each entry mirrors the per-request pooler payload.
|
||||
"""
|
||||
|
||||
def test_multimodal_outputs_on_result(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
mm = {"audio": [torch.randn(10)]}
|
||||
output = _make_omni_output(mm)
|
||||
runner = _make_runner(output, num_reqs=1)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
assert isinstance(result.multimodal_outputs, list)
|
||||
assert len(result.multimodal_outputs) == 1
|
||||
assert "audio" in result.multimodal_outputs[0]
|
||||
assert torch.is_tensor(result.multimodal_outputs[0]["audio"])
|
||||
|
||||
def test_none_multimodal_outputs_becomes_empty_dict(self):
|
||||
from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModelRunner
|
||||
|
||||
output = _make_omni_output(multimodal_outputs=None)
|
||||
runner = _make_runner(output, num_reqs=1)
|
||||
result = OmniGenerationModelRunner.sample_tokens(runner)
|
||||
|
||||
# No multimodal data -> one empty dict per request.
|
||||
assert result.multimodal_outputs == [{}]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Unit tests for OmniGPUModelRunner v2 overrides."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.worker_v2.omni_model_runner import OmniGPUModelRunner, _needs_capture_tensor_unwrap
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _DummyInputBatch:
|
||||
def __init__(self, idx_mapping=None):
|
||||
self.idx_mapping_np = idx_mapping or []
|
||||
|
||||
|
||||
def _make_runner():
|
||||
"""Create an OmniGPUModelRunner without calling __init__."""
|
||||
runner = object.__new__(OmniGPUModelRunner)
|
||||
runner.model = MagicMock()
|
||||
runner.req_states = SimpleNamespace(req_id_to_index={"r1": 0, "r2": 1})
|
||||
runner.execute_model_state = None
|
||||
return runner
|
||||
|
||||
|
||||
def test_finish_requests_calls_remove_for_finished():
|
||||
runner = _make_runner()
|
||||
mock_state = MagicMock()
|
||||
runner.model_state = mock_state
|
||||
|
||||
sched_output = SimpleNamespace(
|
||||
finished_req_ids={"r1"},
|
||||
preempted_req_ids=set(),
|
||||
)
|
||||
|
||||
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
|
||||
runner.finish_requests(sched_output)
|
||||
|
||||
mock_state.remove_request.assert_called_once_with(0)
|
||||
|
||||
|
||||
def test_finish_requests_calls_remove_for_preempted():
|
||||
runner = _make_runner()
|
||||
mock_state = MagicMock()
|
||||
runner.model_state = mock_state
|
||||
|
||||
sched_output = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids={"r2"},
|
||||
)
|
||||
|
||||
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
|
||||
runner.finish_requests(sched_output)
|
||||
|
||||
mock_state.remove_request.assert_called_once_with(1)
|
||||
|
||||
|
||||
def test_finish_requests_ignores_unknown_req_ids():
|
||||
runner = _make_runner()
|
||||
mock_state = MagicMock()
|
||||
runner.model_state = mock_state
|
||||
|
||||
sched_output = SimpleNamespace(
|
||||
finished_req_ids={"unknown"},
|
||||
preempted_req_ids=set(),
|
||||
)
|
||||
|
||||
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
|
||||
runner.finish_requests(sched_output)
|
||||
|
||||
mock_state.remove_request.assert_not_called()
|
||||
|
||||
|
||||
def test_finish_requests_handles_both_finished_and_preempted():
|
||||
runner = _make_runner()
|
||||
mock_state = MagicMock()
|
||||
runner.model_state = mock_state
|
||||
|
||||
sched_output = SimpleNamespace(
|
||||
finished_req_ids={"r1"},
|
||||
preempted_req_ids={"r2"},
|
||||
)
|
||||
|
||||
with patch.object(type(runner).__bases__[0], "finish_requests", return_value=None):
|
||||
runner.finish_requests(sched_output)
|
||||
|
||||
assert mock_state.remove_request.call_count == 2
|
||||
|
||||
|
||||
def test_thinker_stage_needs_capture_tensor_unwrap():
|
||||
assert _needs_capture_tensor_unwrap(SimpleNamespace(model_stage="thinker"))
|
||||
assert not _needs_capture_tensor_unwrap(SimpleNamespace(model_stage="talker"))
|
||||
|
||||
|
||||
def test_capture_model_unwraps_tuple_outputs():
|
||||
runner = object.__new__(OmniGPUModelRunner)
|
||||
hidden = torch.ones(1, 2)
|
||||
|
||||
def original_forward():
|
||||
return hidden, {"layers": {}}
|
||||
|
||||
runner.model = SimpleNamespace(forward=original_forward)
|
||||
runner._model_returns_tuple = True
|
||||
runner._exclude_full_graph = False
|
||||
|
||||
def capture_model(_self):
|
||||
assert torch.equal(runner.model.forward(), hidden)
|
||||
return 3
|
||||
|
||||
with patch.object(type(runner).__bases__[0], "capture_model", capture_model):
|
||||
assert runner.capture_model() == 3
|
||||
|
||||
assert runner.model.forward is original_forward
|
||||
|
||||
|
||||
def test_capture_model_unwraps_omni_outputs():
|
||||
runner = object.__new__(OmniGPUModelRunner)
|
||||
hidden = torch.ones(1, 2)
|
||||
|
||||
def original_forward():
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
|
||||
|
||||
runner.model = SimpleNamespace(forward=original_forward)
|
||||
runner._model_returns_tuple = True
|
||||
runner._exclude_full_graph = False
|
||||
|
||||
def capture_model(_self):
|
||||
assert torch.equal(runner.model.forward(), hidden)
|
||||
return 5
|
||||
|
||||
with patch.object(type(runner).__bases__[0], "capture_model", capture_model):
|
||||
assert runner.capture_model() == 5
|
||||
|
||||
assert runner.model.forward is original_forward
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Unit tests for OmniModelState core methods and plugin dispatch."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.engine.serialization import serialize_additional_information
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
|
||||
from vllm_omni.worker_v2.model_states.plugin import OmniModelStatePlugin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Stubs
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
class _DummyInputBatch:
|
||||
def __init__(self, indices):
|
||||
self.idx_mapping_np = indices
|
||||
self.num_reqs = len(indices)
|
||||
self.num_scheduled_tokens = [1] * len(indices)
|
||||
self.query_start_loc_np = list(range(len(indices)))
|
||||
|
||||
|
||||
class _DummyReqState:
|
||||
pass
|
||||
|
||||
|
||||
class _SpyPlugin(OmniModelStatePlugin):
|
||||
"""Plugin that records calls for verification."""
|
||||
|
||||
def __init__(self):
|
||||
self.add_calls: list = []
|
||||
self.remove_calls: list = []
|
||||
self.prepare_calls: list = []
|
||||
self.postprocess_calls: list = []
|
||||
|
||||
def on_add_request(self, req_index, new_req_data):
|
||||
self.add_calls.append((req_index, new_req_data))
|
||||
|
||||
def on_remove_request(self, req_index):
|
||||
self.remove_calls.append(req_index)
|
||||
|
||||
def prepare_extra_inputs(self, input_batch, req_states):
|
||||
self.prepare_calls.append(True)
|
||||
return {"plugin_key": "plugin_value"}
|
||||
|
||||
def postprocess(self, text_hidden, multimodal_outputs, input_batch, req_states):
|
||||
self.postprocess_calls.append(True)
|
||||
return text_hidden, multimodal_outputs
|
||||
|
||||
|
||||
def _make_state(
|
||||
max_num_reqs=4, has_preprocess=False, has_postprocess=False, have_multimodal_outputs=False, plugins=None
|
||||
):
|
||||
"""Create an OmniModelState without calling real __init__."""
|
||||
state = object.__new__(OmniModelState)
|
||||
|
||||
# Minimal model mock
|
||||
model = MagicMock()
|
||||
model.has_preprocess = has_preprocess
|
||||
model.has_postprocess = has_postprocess
|
||||
model.have_multimodal_outputs = have_multimodal_outputs
|
||||
model.preprocess_in_forward = False
|
||||
model.get_omni_plugins = MagicMock(return_value=[])
|
||||
state.model = model
|
||||
|
||||
# scheduler_config mock
|
||||
state.scheduler_config = SimpleNamespace(max_num_seqs=max_num_reqs)
|
||||
|
||||
# Skip DefaultModelState.__init__ side effects
|
||||
state.has_preprocess = has_preprocess
|
||||
state.has_postprocess = has_postprocess
|
||||
state.have_multimodal_outputs = have_multimodal_outputs
|
||||
state.plugins = plugins or []
|
||||
|
||||
from vllm_omni.worker_v2.model_states.intermediate_buffer import (
|
||||
OmniIntermediateBuffer,
|
||||
)
|
||||
|
||||
state.intermediate_buffer = OmniIntermediateBuffer(max_num_reqs)
|
||||
state._static_inputs_embeds = None
|
||||
return state
|
||||
|
||||
|
||||
def _make_new_req_data(req_id="r1"):
|
||||
return SimpleNamespace(
|
||||
req_id=req_id,
|
||||
mm_features=[],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# add_request / remove_request
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_add_request_populates_buffer():
|
||||
state = _make_state()
|
||||
req = _make_new_req_data("r1")
|
||||
|
||||
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
|
||||
state.add_request(0, req)
|
||||
|
||||
assert state.intermediate_buffer.buffers[0]["req_id"] == "r1"
|
||||
|
||||
|
||||
def test_add_request_dispatches_to_plugins():
|
||||
plugin = _SpyPlugin()
|
||||
state = _make_state(plugins=[plugin])
|
||||
req = _make_new_req_data("r1")
|
||||
|
||||
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
|
||||
state.add_request(0, req)
|
||||
|
||||
assert len(plugin.add_calls) == 1
|
||||
assert plugin.add_calls[0][0] == 0
|
||||
|
||||
|
||||
def test_add_request_unflattens_serialized_additional_information():
|
||||
state = _make_state()
|
||||
hidden = torch.randn(2, 4)
|
||||
embeds = torch.randn(3, 4)
|
||||
req = SimpleNamespace(
|
||||
req_id="r1",
|
||||
mm_features=[],
|
||||
additional_information=serialize_additional_information(
|
||||
{
|
||||
"hidden_states": {"output": hidden},
|
||||
"embed": {"prefill": embeds},
|
||||
"ids": {"prompt": [1, 2, 3], "output": [4]},
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
|
||||
state.add_request(0, req)
|
||||
|
||||
buf = state.intermediate_buffer.buffers[0]
|
||||
assert torch.equal(buf["hidden_states"]["output"], hidden)
|
||||
assert torch.equal(buf["embed"]["prefill"], embeds)
|
||||
assert buf["ids"]["prompt"] == [1, 2, 3]
|
||||
assert buf["ids"]["output"] == [4]
|
||||
|
||||
|
||||
def test_remove_request_clears_buffer():
|
||||
state = _make_state()
|
||||
req = _make_new_req_data("r1")
|
||||
|
||||
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
|
||||
state.add_request(0, req)
|
||||
|
||||
state.remove_request(0)
|
||||
assert state.intermediate_buffer.buffers[0] == {}
|
||||
|
||||
|
||||
def test_remove_request_dispatches_to_plugins():
|
||||
plugin = _SpyPlugin()
|
||||
state = _make_state(plugins=[plugin])
|
||||
state.remove_request(2)
|
||||
assert plugin.remove_calls == [2]
|
||||
|
||||
|
||||
def test_intermediate_buffer_update_merges_nested_and_tuple_keys():
|
||||
state = _make_state()
|
||||
trailing = torch.randn(2, 4)
|
||||
last = torch.randn(4)
|
||||
codes = torch.ones(1, 16, dtype=torch.long)
|
||||
state.intermediate_buffer.buffers[0] = {
|
||||
"req_id": "r1",
|
||||
"hidden_states": {"trailing_text": trailing},
|
||||
}
|
||||
|
||||
state.intermediate_buffer.update(
|
||||
0,
|
||||
{
|
||||
("codes", "audio"): codes,
|
||||
"hidden_states": {"last": last},
|
||||
},
|
||||
{("codes", "audio"), ("hidden_states", "last")},
|
||||
)
|
||||
|
||||
buf = state.intermediate_buffer.buffers[0]
|
||||
assert ("codes", "audio") not in buf
|
||||
assert torch.equal(buf["codes"]["audio"], codes)
|
||||
assert torch.equal(buf["hidden_states"]["trailing_text"], trailing)
|
||||
assert torch.equal(buf["hidden_states"]["last"], last)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# prepare_inputs
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prepare_inputs_injects_buffer_and_runtime_info():
|
||||
state = _make_state()
|
||||
req = _make_new_req_data("r1")
|
||||
|
||||
with patch.object(type(state).__bases__[0], "add_request", return_value=None):
|
||||
state.add_request(0, req)
|
||||
|
||||
batch = _DummyInputBatch([0])
|
||||
|
||||
with patch.object(type(state).__bases__[0], "prepare_inputs", return_value={}):
|
||||
result = state.prepare_inputs(batch, _DummyReqState())
|
||||
|
||||
assert "model_intermediate_buffer" in result
|
||||
assert "runtime_additional_information" in result
|
||||
assert len(result["model_intermediate_buffer"]) == 1
|
||||
assert result["model_intermediate_buffer"][0]["req_id"] == "r1"
|
||||
|
||||
|
||||
def test_prepare_inputs_merges_plugin_extra():
|
||||
plugin = _SpyPlugin()
|
||||
state = _make_state(plugins=[plugin])
|
||||
|
||||
batch = _DummyInputBatch([])
|
||||
|
||||
with patch.object(type(state).__bases__[0], "prepare_inputs", return_value={"base": True}):
|
||||
result = state.prepare_inputs(batch, _DummyReqState())
|
||||
|
||||
assert result["base"] is True
|
||||
assert result["plugin_key"] == "plugin_value"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# prepare_dummy_inputs
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prepare_dummy_inputs_has_buffer_keys():
|
||||
state = _make_state()
|
||||
with patch.object(type(state).__bases__[0], "prepare_dummy_inputs", return_value={}):
|
||||
result = state.prepare_dummy_inputs(num_reqs=2, num_tokens=16)
|
||||
|
||||
assert "model_intermediate_buffer" in result
|
||||
assert len(result["model_intermediate_buffer"]) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# postprocess_model_output
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_postprocess_omni_output_with_multimodal():
|
||||
state = _make_state(have_multimodal_outputs=True)
|
||||
hidden = torch.randn(4, 8)
|
||||
mm = {"audio": torch.randn(4, 2)}
|
||||
omni = OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
|
||||
|
||||
batch = _DummyInputBatch([0])
|
||||
th, mo = state.postprocess_model_output(omni, batch, _DummyReqState())
|
||||
assert th is hidden
|
||||
assert mo is mm
|
||||
|
||||
|
||||
def test_postprocess_tuple_output():
|
||||
state = _make_state(have_multimodal_outputs=False)
|
||||
del state.model.make_omni_output
|
||||
hidden = torch.randn(4, 8)
|
||||
|
||||
batch = _DummyInputBatch([0])
|
||||
th, mo = state.postprocess_model_output((hidden, {}), batch, _DummyReqState())
|
||||
assert th is hidden
|
||||
assert mo == {}
|
||||
|
||||
|
||||
def test_postprocess_raw_tensor():
|
||||
state = _make_state()
|
||||
del state.model.make_omni_output
|
||||
hidden = torch.randn(4, 8)
|
||||
|
||||
batch = _DummyInputBatch([0])
|
||||
th, mo = state.postprocess_model_output(hidden, batch, _DummyReqState())
|
||||
assert th is hidden
|
||||
assert mo == {}
|
||||
|
||||
|
||||
def test_postprocess_calls_make_omni_output_when_not_omni():
|
||||
"""When model output is not OmniOutput but model has make_omni_output, it should be called."""
|
||||
state = _make_state(have_multimodal_outputs=True)
|
||||
hidden = torch.randn(4, 8)
|
||||
mm = {"audio": torch.randn(4, 2)}
|
||||
expected_omni = OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
|
||||
|
||||
state.model.make_omni_output = MagicMock(return_value=expected_omni)
|
||||
|
||||
batch = _DummyInputBatch([0])
|
||||
th, mo = state.postprocess_model_output((hidden, {}), batch, _DummyReqState())
|
||||
state.model.make_omni_output.assert_called_once()
|
||||
assert th is hidden
|
||||
|
||||
|
||||
def test_postprocess_dispatches_to_plugins():
|
||||
plugin = _SpyPlugin()
|
||||
state = _make_state(plugins=[plugin])
|
||||
hidden = torch.randn(4, 8)
|
||||
|
||||
batch = _DummyInputBatch([0])
|
||||
state.postprocess_model_output(hidden, batch, _DummyReqState())
|
||||
assert len(plugin.postprocess_calls) == 1
|
||||
|
||||
|
||||
def test_run_postprocess_does_not_duplicate_hidden_states_kwarg():
|
||||
state = _make_state(has_postprocess=True)
|
||||
state.model.gpu_resident_buffer_keys = set()
|
||||
seen = {}
|
||||
trailing = torch.ones(1, 2)
|
||||
|
||||
def postprocess(hidden_states, **info):
|
||||
seen["hidden_states"] = hidden_states
|
||||
seen["info"] = info
|
||||
return {"hidden_states": {"last": hidden_states[-1].detach()}}
|
||||
|
||||
state.model.postprocess = postprocess
|
||||
state.intermediate_buffer.buffers[0] = {
|
||||
"req_id": "r1",
|
||||
"hidden_states": {"trailing_text": trailing},
|
||||
"meta": {"codec_streaming": True},
|
||||
}
|
||||
|
||||
hidden = torch.randn(1, 2)
|
||||
state.run_postprocess(hidden, _DummyInputBatch([0]))
|
||||
|
||||
assert torch.equal(seen["hidden_states"], hidden)
|
||||
assert "hidden_states" not in seen["info"]
|
||||
assert seen["info"]["meta"] == {"codec_streaming": True}
|
||||
assert torch.equal(state.intermediate_buffer.buffers[0]["hidden_states"]["trailing_text"], trailing)
|
||||
assert torch.equal(state.intermediate_buffer.buffers[0]["hidden_states"]["last"], hidden[-1])
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Unit tests for OmniModelStatePlugin interface correctness."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.worker_v2.model_states.plugin import OmniModelStatePlugin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _ConcretePlugin(OmniModelStatePlugin):
|
||||
"""Minimal concrete plugin for testing the ABC."""
|
||||
|
||||
def __init__(self):
|
||||
self.added: list[tuple[int, Any]] = []
|
||||
self.removed: list[int] = []
|
||||
|
||||
def on_add_request(self, req_index, new_req_data):
|
||||
self.added.append((req_index, new_req_data))
|
||||
|
||||
def on_remove_request(self, req_index):
|
||||
self.removed.append(req_index)
|
||||
|
||||
def postprocess(self, text_hidden, multimodal_outputs, input_batch, req_states):
|
||||
return text_hidden, multimodal_outputs
|
||||
|
||||
|
||||
def test_concrete_plugin_instantiates():
|
||||
plugin = _ConcretePlugin()
|
||||
assert isinstance(plugin, OmniModelStatePlugin)
|
||||
|
||||
|
||||
def test_on_add_request_called():
|
||||
plugin = _ConcretePlugin()
|
||||
plugin.on_add_request(0, SimpleNamespace(req_id="r1"))
|
||||
assert len(plugin.added) == 1
|
||||
assert plugin.added[0][0] == 0
|
||||
|
||||
|
||||
def test_on_remove_request_called():
|
||||
plugin = _ConcretePlugin()
|
||||
plugin.on_remove_request(3)
|
||||
assert plugin.removed == [3]
|
||||
|
||||
|
||||
def test_prepare_extra_inputs_default_empty():
|
||||
plugin = _ConcretePlugin()
|
||||
result = plugin.prepare_extra_inputs(SimpleNamespace(), SimpleNamespace())
|
||||
assert result == {}
|
||||
|
||||
|
||||
def test_dummy_run_default_noop():
|
||||
plugin = _ConcretePlugin()
|
||||
plugin.dummy_run(None, 128)
|
||||
|
||||
|
||||
def test_postprocess_returns_inputs():
|
||||
plugin = _ConcretePlugin()
|
||||
hidden = torch.randn(4, 8)
|
||||
mm = {"audio": torch.randn(4, 2)}
|
||||
out_h, out_mm = plugin.postprocess(hidden, mm, None, None)
|
||||
assert out_h is hidden
|
||||
assert out_mm is mm
|
||||
|
||||
|
||||
def test_abc_cannot_instantiate_directly():
|
||||
with pytest.raises(TypeError):
|
||||
OmniModelStatePlugin()
|
||||
Reference in new issue
Block a user