[Model] Add MiniCPM-o 4.5 turn-mode MRv2 and batched audio inference (#8222)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
28 files changed
+1577
-51
No files matched your search
@@ -19,6 +19,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _StageModel(torch.nn.Module):
|
||||
logits_vocab_size = 6562
|
||||
|
||||
def make_empty_intermediate_tensors(self):
|
||||
return None
|
||||
|
||||
|
||||
@@ -1969,7 +1969,10 @@ def test_whole_euler_query_bucket_keeps_current_first_cache_layout(monkeypatch:
|
||||
(_whole_euler_chunk(1, 8), 6, 2), # narrow padded chunk: capture pads 8 -> 16
|
||||
(_whole_euler_chunk(1, 16), 16, 0), # consumes the narrow chunk's cache
|
||||
]
|
||||
caches = {"exact": (None, None), "bucketed": (None, None)}
|
||||
caches: dict[str, tuple[torch.Tensor | None, torch.Tensor | None]] = {
|
||||
"exact": (None, None),
|
||||
"bucketed": (None, None),
|
||||
}
|
||||
for chunk, mel_frames, pad_frames in chunks:
|
||||
chunk["x"][:, :, mel_frames:] = 0.0
|
||||
results = {}
|
||||
@@ -2319,3 +2322,24 @@ def test_whole_euler_groups_are_acquired_again_after_a_flush(monkeypatch: pytest
|
||||
assert calls == [16, 1, 16, 1]
|
||||
flush_always = True
|
||||
assert wrapper._group_entries(groups, fills) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", [False, True])
|
||||
def test_vocoder_restores_tf32_policy(fail):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import MiniCPMO45Code2Wav
|
||||
|
||||
previous = torch.backends.cuda.matmul.allow_tf32
|
||||
|
||||
def forward(*args, **kwargs):
|
||||
assert torch.backends.cuda.matmul.allow_tf32
|
||||
if fail:
|
||||
raise RuntimeError("injected")
|
||||
return "ok"
|
||||
|
||||
model = SimpleNamespace(_extra_config=lambda: {"token2wav_allow_tf32": True}, _forward_impl=forward)
|
||||
if fail:
|
||||
with pytest.raises(RuntimeError, match="injected"):
|
||||
MiniCPMO45Code2Wav.forward(model)
|
||||
else:
|
||||
assert MiniCPMO45Code2Wav.forward(model) == "ok"
|
||||
assert torch.backends.cuda.matmul.allow_tf32 == previous
|
||||
@@ -20,6 +20,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
def test_multimodal_runner_preserves_forwarded_token_identities(monkeypatch):
|
||||
stage_model = torch.nn.Module()
|
||||
stage_model.logits_vocab_size = 6562
|
||||
stage_model.make_empty_intermediate_tensors = lambda: None
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni.init_vllm_registered_model",
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MiniCPM-o 4.5 Talker per-step path: batched, host-sync-free, same results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
|
||||
ConditionalChatTTSConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
MiniCPMO45OmniTTSForConditionalGeneration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_VOCAB = 8
|
||||
_EOS = 7
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SamplingMetadata:
|
||||
repetition_penalties: torch.Tensor
|
||||
prompt_token_ids: torch.Tensor | None = None
|
||||
no_penalties: bool = False
|
||||
|
||||
|
||||
def _make_talker(device: str = "cpu") -> MiniCPMO45OmniTTSForConditionalGeneration:
|
||||
talker = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
|
||||
nn.Module.__init__(talker)
|
||||
talker._num_audio_tokens = _VOCAB
|
||||
talker._codec_eos_id = _EOS
|
||||
talker._force_eos_rows = None
|
||||
talker._mask_eos_rows = None
|
||||
talker._pending_force_eos_rows = None
|
||||
talker._penalty_histories = None
|
||||
talker._request_audio_states = {}
|
||||
talker._request_condition_states = {}
|
||||
talker._deferred_cleanup_ids = set()
|
||||
talker._tts_config = ConditionalChatTTSConfig()
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
talker.emb_code = nn.ModuleList([nn.Embedding(_VOCAB, 4)])
|
||||
talker.head_code = nn.ModuleList([nn.Linear(4, _VOCAB, bias=False)])
|
||||
with torch.no_grad():
|
||||
talker.emb_code[0].weight.copy_(torch.randn(_VOCAB, 4, generator=generator))
|
||||
talker.head_code[0].weight.copy_(torch.randn(_VOCAB, 4, generator=generator))
|
||||
return talker.to(device)
|
||||
|
||||
|
||||
def _states() -> dict[str, dict[str, Any]]:
|
||||
return {
|
||||
# Mid-stream row: forwards its code, EOS still masked by min_tokens.
|
||||
"req-live": {"finished": False, "step": 2, "max_tokens": 100, "min_tokens": 5, "recent_codes": [1, 2]},
|
||||
# The previous step sampled codec EOS: forward nothing, finish, force EOS.
|
||||
"req-eos": {"finished": False, "step": 60, "max_tokens": 100, "recent_codes": [4, 4, 5]},
|
||||
# Leftover decode of an already-finished request.
|
||||
"req-done": {"finished": True, "step": 3, "max_tokens": 100},
|
||||
# Reaches its codec budget on this step.
|
||||
"req-cap": {"finished": False, "step": 8, "max_tokens": 10, "recent_codes": list(range(7)) * 3},
|
||||
}
|
||||
|
||||
|
||||
def _infos(states: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"request_id": request_id, "audio_state": dict(state), "_omni_is_prefill": False}
|
||||
for request_id, state in states.items()
|
||||
]
|
||||
|
||||
|
||||
def _step(talker, infos, hidden, mocker, *, batched: bool, input_ids: torch.Tensor, penalties: torch.Tensor):
|
||||
if batched:
|
||||
returned_ids, embeds, updates = talker.preprocess_decode_batch(input_ids=input_ids, req_infos=infos)
|
||||
assert returned_ids is input_ids
|
||||
else:
|
||||
rows = [talker.preprocess(input_ids[row : row + 1], None, **info) for row, info in enumerate(infos)]
|
||||
embeds = torch.cat([row[1] for row in rows])
|
||||
updates = [row[2] for row in rows]
|
||||
for info, update in zip(infos, updates, strict=True):
|
||||
info["codes"] = update["codes"]
|
||||
output = talker.make_omni_output(
|
||||
hidden,
|
||||
model_intermediate_buffer=infos,
|
||||
request_token_spans=[(row, row + 1) for row in range(len(infos))],
|
||||
)
|
||||
logits = talker.compute_logits(output.text_hidden_states)
|
||||
captured: dict[str, torch.Tensor] = {}
|
||||
|
||||
def _sampler(logits, sampling_metadata):
|
||||
captured["logits"] = logits.clone()
|
||||
return SimpleNamespace(sampled_token_ids=logits.argmax(dim=-1, keepdim=True))
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.Sampler",
|
||||
return_value=_sampler,
|
||||
)
|
||||
sampled = talker.sample(logits, _SamplingMetadata(repetition_penalties=penalties))
|
||||
return embeds, output, logits, captured["logits"], sampled.sampled_token_ids
|
||||
|
||||
|
||||
def test_batched_decode_matches_scalar_preprocess(mocker) -> None:
|
||||
input_ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32)
|
||||
hidden = torch.randn(4, 4, generator=torch.Generator().manual_seed(1))
|
||||
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0])
|
||||
results = {}
|
||||
talkers = {}
|
||||
for batched in (False, True):
|
||||
talker = _make_talker()
|
||||
states = _states()
|
||||
talker._request_audio_states = copy.deepcopy(states)
|
||||
results[batched] = _step(
|
||||
talker, _infos(states), hidden, mocker, batched=batched, input_ids=input_ids, penalties=penalties
|
||||
)
|
||||
talkers[batched] = talker
|
||||
|
||||
# Embeddings, masked logits, penalized logits and sampled ids.
|
||||
for index in (0, 2, 3, 4):
|
||||
assert torch.equal(results[False][index], results[True][index])
|
||||
for result in results.values():
|
||||
output = result[1].multimodal_outputs
|
||||
assert [t.tolist() for t in output["codes"]["audio"]] == [[[3]], [], [], [[6]]]
|
||||
assert [t.item() for t in output["meta"]["finished"]] == [False, True, True, True]
|
||||
assert talkers[True]._request_audio_states == talkers[False]._request_audio_states
|
||||
assert talkers[True]._decode_codec_ids == {}
|
||||
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The MiniCPM-o 4.5 Talker's per-step path must not block the host on the GPU."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from tests.model_executor.models.minicpmo_4_5.test_talker_host_sync import _EOS, _infos, _make_talker, _states, _step
|
||||
from vllm_omni.utils.device_copy import index_to_device
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
def test_talker_step_does_not_synchronize_cuda(mocker) -> None:
|
||||
talker = _make_talker("cuda")
|
||||
states = _states()
|
||||
talker._request_audio_states = copy.deepcopy(states)
|
||||
infos = _infos(states)
|
||||
input_ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32, device="cuda")
|
||||
hidden = torch.randn(4, 4, device="cuda")
|
||||
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0], device="cuda")
|
||||
# Warm the pinned host allocator and CUDA context outside the check.
|
||||
index_to_device([1], "cuda")
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
try:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
with torch.inference_mode():
|
||||
*_, sampled = _step(talker, infos, hidden, mocker, batched=True, input_ids=input_ids, penalties=penalties)
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
assert sampled.reshape(-1).tolist()[1:] == [_EOS, _EOS, _EOS]
|
||||
assert talker._request_audio_states["req-live"]["recent_codes"] == [1, 2, 3]
|
||||
assert talker._request_audio_states["req-eos"]["finished"] is True
|
||||
|
||||
|
||||
def test_talker_condition_upload_does_not_synchronize_cuda() -> None:
|
||||
talker = _make_talker("cuda")
|
||||
talker.emb_text = nn.Embedding(16, 4).cuda()
|
||||
talker.projector_semantic = nn.Linear(6, 4).cuda()
|
||||
talker._normalize = True
|
||||
talker._text_eos_id = 14
|
||||
talker._tts_bos_id = 15
|
||||
token_ids = torch.tensor([2, 3, 4])
|
||||
hidden_states = torch.randn(3, 6)
|
||||
expected = talker._build_condition_embeddings(token_ids.cuda(), hidden_states.cuda())
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
try:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
with torch.inference_mode():
|
||||
condition = talker._build_condition_embeddings(token_ids, hidden_states)
|
||||
boundary = talker._build_condition_embeddings(token_ids[:0], hidden_states[:0])
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
assert torch.equal(condition, expected)
|
||||
assert boundary.shape == (2, 4)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_v1_batched_decode_and_async_snapshot_match_scalar(mocker):
|
||||
from vllm_omni.worker.gpu_ar_model_runner import _snapshot_tensor_payload_to_cpu_async
|
||||
|
||||
ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32, device="cuda")
|
||||
hidden = torch.arange(16, dtype=torch.float32, device="cuda").reshape(4, 4)
|
||||
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0], device="cuda")
|
||||
results, states = [], []
|
||||
for batched in (False, True):
|
||||
talker = _make_talker("cuda")
|
||||
talker._request_audio_states = copy.deepcopy(_states())
|
||||
results.append(
|
||||
_step(talker, _infos(_states()), hidden, mocker, batched=batched, input_ids=ids, penalties=penalties)
|
||||
)
|
||||
states.append(copy.deepcopy(talker._request_audio_states))
|
||||
for index in (0, 2, 3, 4):
|
||||
torch.testing.assert_close(results[0][index], results[1][index], rtol=0, atol=0)
|
||||
assert states[0] == states[1]
|
||||
reference, candidate = [result[1].multimodal_outputs for result in results]
|
||||
for group in ("codes", "meta"):
|
||||
for key in reference[group]:
|
||||
for left, right in zip(reference[group][key], candidate[group][key], strict=True):
|
||||
torch.testing.assert_close(left, right, rtol=0, atol=0)
|
||||
snapshot = _snapshot_tensor_payload_to_cpu_async(
|
||||
{"hidden_states": hidden, "multimodal_outputs": candidate},
|
||||
copy_stream=torch.cuda.Stream(),
|
||||
pin_memory=True,
|
||||
)
|
||||
expected_hidden = hidden.cpu().clone()
|
||||
snapshot.wait()
|
||||
torch.testing.assert_close(snapshot.payload["hidden_states"], expected_hidden, rtol=0, atol=0)
|
||||
for group in ("codes", "meta"):
|
||||
for key in reference[group]:
|
||||
for left, right in zip(
|
||||
reference[group][key], snapshot.payload["multimodal_outputs"][group][key], strict=True
|
||||
):
|
||||
torch.testing.assert_close(left.cpu(), right, rtol=0, atol=0)
|
||||
hidden.fill_(-1)
|
||||
torch.testing.assert_close(snapshot.payload["hidden_states"], expected_hidden, rtol=0, atol=0)
|
||||
@@ -0,0 +1,140 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Model Runner V2 path of MiniCPM-o 4.5's Talker: device-side codec output,
|
||||
EOS control and the 16-frame codec penalty must match the V1 host path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
_OFFLINE_CODEC_MAX_NEW_TOKENS,
|
||||
MiniCPMO45OmniTTSForConditionalGeneration,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_EOS = 6561
|
||||
|
||||
|
||||
def _talker(max_reqs: int = 8, max_position_embeddings: int = 4096):
|
||||
talker = object.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
|
||||
talker._codec_eos_id = _EOS
|
||||
talker._tts_config = SimpleNamespace(max_position_embeddings=max_position_embeddings)
|
||||
talker._mrv2_empty_speech = torch.zeros(max_reqs, dtype=torch.bool)
|
||||
talker._mrv2_forced_eos = None
|
||||
talker._mrv2_decode_rows_logged = False
|
||||
return talker
|
||||
|
||||
|
||||
def _batch(rows: list[dict], *, pad_to: int | None = None):
|
||||
"""rows: one dict per request with slot, prompt_len, span (token ids) and prefill flag."""
|
||||
ids = [token for row in rows for token in row["span"]]
|
||||
num_tokens = len(ids)
|
||||
padded = pad_to or num_tokens
|
||||
starts = np.cumsum([0] + [len(row["span"]) for row in rows])
|
||||
seq_lens = [row["computed"] + len(row["span"]) for row in rows]
|
||||
return SimpleNamespace(
|
||||
num_reqs=len(rows),
|
||||
has_prefill=any(row["prefill"] for row in rows),
|
||||
is_prefilling_np=np.array([row["prefill"] for row in rows]),
|
||||
idx_mapping_np=np.array([row["slot"] for row in rows]),
|
||||
idx_mapping=torch.tensor([row["slot"] for row in rows]),
|
||||
input_ids=torch.tensor(ids + [0] * (padded - num_tokens), dtype=torch.int32),
|
||||
logits_indices=torch.tensor(starts[1:] - 1),
|
||||
seq_lens=torch.tensor(seq_lens + [0] * 2, dtype=torch.int32),
|
||||
), padded
|
||||
|
||||
|
||||
def _req_states(prompt_lens: dict[int, int], max_reqs: int = 8):
|
||||
prompt_len = torch.zeros(max_reqs, dtype=torch.int32)
|
||||
for slot, value in prompt_lens.items():
|
||||
prompt_len[slot] = value
|
||||
return SimpleNamespace(prompt_len=SimpleNamespace(gpu=prompt_len))
|
||||
|
||||
|
||||
def test_mrv2_output_emits_decode_input_codes_with_validity_and_forced_eos() -> None:
|
||||
talker = _talker()
|
||||
rows = [
|
||||
# Final prefill chunk: no code yet (V1 emits an empty delta).
|
||||
dict(slot=2, prompt_len=5, computed=0, span=[0, 0, 0, 0, 0], prefill=True),
|
||||
# Decode with a codec id: emitted this step.
|
||||
dict(slot=0, prompt_len=4, computed=6, span=[17], prefill=False),
|
||||
# Decode whose input is codec EOS: request ended, force EOS again.
|
||||
dict(slot=1, prompt_len=4, computed=8, span=[_EOS], prefill=False),
|
||||
]
|
||||
batch, padded = _batch(rows, pad_to=8)
|
||||
buffers = [{"audio_state": {"finished": False}}, {}, {}]
|
||||
hidden = torch.zeros((padded, 4))
|
||||
out = talker.make_omni_output_mrv2(
|
||||
hidden,
|
||||
input_batch=batch,
|
||||
req_states=_req_states({cast(int, row["slot"]): cast(int, row["prompt_len"]) for row in rows}),
|
||||
model_intermediate_buffer=buffers,
|
||||
)
|
||||
assert isinstance(out, OmniOutput)
|
||||
codes = out.multimodal_outputs["codes"]["audio"]
|
||||
valid = out.multimodal_outputs["meta"]["codec_frame_valid"]
|
||||
assert codes.shape == (padded, 1) and valid.shape == (padded,)
|
||||
assert codes[:, 0].tolist() == batch.input_ids.tolist()
|
||||
assert valid.tolist() == [False] * 5 + [True, False] + [False]
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 3).tolist() == [False, False, True]
|
||||
# Consumed once; a warmup sampler call sees nothing.
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 3) is None
|
||||
|
||||
|
||||
def test_mrv2_output_empty_condition_and_length_cap() -> None:
|
||||
talker = _talker()
|
||||
prompt_lens = {3: 6, 4: 4096 - 100}
|
||||
# Empty Thinker condition: preprocess marks the request finished at prefill.
|
||||
batch, _ = _batch([dict(slot=3, prompt_len=6, computed=0, span=[0] * 6, prefill=True)])
|
||||
talker.make_omni_output_mrv2(
|
||||
torch.zeros((6, 4)),
|
||||
input_batch=batch,
|
||||
req_states=_req_states(prompt_lens),
|
||||
model_intermediate_buffer=[{"audio_state": {"finished": True}}],
|
||||
)
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [True]
|
||||
# Its later decode rows stay forced and emit nothing, whatever the input id.
|
||||
batch, _ = _batch([dict(slot=3, prompt_len=6, computed=6, span=[42], prefill=False)])
|
||||
out = talker.make_omni_output_mrv2(
|
||||
torch.zeros((1, 4)), input_batch=batch, req_states=_req_states(prompt_lens), model_intermediate_buffer=[{}]
|
||||
)
|
||||
assert out.multimodal_outputs["meta"]["codec_frame_valid"].tolist() == [False]
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [True]
|
||||
|
||||
# Offline cap: min(2048, context - prompt) samples, the last one EOS.
|
||||
limit = min(_OFFLINE_CODEC_MAX_NEW_TOKENS, 100) - 1
|
||||
for step, forced in ((limit - 1, False), (limit, True)):
|
||||
batch, _ = _batch(
|
||||
[dict(slot=4, prompt_len=prompt_lens[4], computed=prompt_lens[4] + step - 1, span=[5], prefill=False)]
|
||||
)
|
||||
out = talker.make_omni_output_mrv2(
|
||||
torch.zeros((1, 4)), input_batch=batch, req_states=_req_states(prompt_lens), model_intermediate_buffer=[{}]
|
||||
)
|
||||
assert out.multimodal_outputs["meta"]["codec_frame_valid"].tolist() == [True]
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [forced]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("forced", [None, [False, True]])
|
||||
def test_sampler_adapter_keeps_upstream_counts_and_only_forces_codec_eos(mocker, forced):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import MiniCPMO45TalkerSampler
|
||||
|
||||
output = SimpleNamespace(sampled_token_ids=torch.tensor([[2], [3]]), num_sampled=torch.tensor([1, 0]))
|
||||
base = mocker.Mock(return_value=output)
|
||||
base.req_states = object()
|
||||
mask = None if forced is None else torch.tensor(forced)
|
||||
talker = SimpleNamespace(_codec_eos_id=7, take_mrv2_forced_eos=mocker.Mock(return_value=mask))
|
||||
sampler = MiniCPMO45TalkerSampler(base, talker)
|
||||
logits, batch = torch.zeros(2, 8), object()
|
||||
assert sampler(logits, batch) is output
|
||||
assert output.sampled_token_ids.tolist() == [[2], [3 if forced is None else 7]]
|
||||
assert output.num_sampled.tolist() == [1, 0]
|
||||
base.assert_called_once_with(logits, batch)
|
||||
talker.take_mrv2_forced_eos.assert_called_once_with(batch, base.req_states, 2)
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MRv2 Thinker preserves multimodal encoding and the llm2tts row ledger."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5 import minicpmo_4_5_omni as omni
|
||||
from vllm_omni.worker_v2.omni_model_runner import OmniGPUModelRunner
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"profile,capacities,kv_gib",
|
||||
[
|
||||
("minicpmo_4_5_turn_mrv2.yaml", [16, 8, 8], 2),
|
||||
("minicpmo_4_5_turn_mrv2_h200.yaml", [16, 16, 8], 4),
|
||||
],
|
||||
)
|
||||
def test_mrv2_profile_retains_full_thinker_handoff(profile, capacities, kv_gib, monkeypatch):
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config, merge_pipeline_deploy
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# merge_pipeline_deploy resolves the platform again; test the CUDA profile
|
||||
# consistently even when this CPU test runs on a ROCm host.
|
||||
monkeypatch.setattr(current_omni_platform, "device_name", "cuda")
|
||||
deploy = Path(__file__).resolve().parents[4] / "vllm_omni/deploy" / profile
|
||||
config = _apply_platform_overrides(load_deploy_config(deploy), platform="cuda")
|
||||
stages = merge_pipeline_deploy(MINICPMO_4_5_PIPELINE, config)
|
||||
assert [s.yaml_engine_args["use_v2_model_runner"] for s in stages] == [True, True, True]
|
||||
assert [s.yaml_engine_args["async_chunk"] for s in stages] == [False, True, True]
|
||||
assert [s.yaml_engine_args["max_num_seqs"] for s in stages] == capacities
|
||||
assert stages[1].yaml_engine_args["kv_cache_memory_bytes"] == kv_gib * 1024**3
|
||||
|
||||
|
||||
def _model(mocker, *, v2=True, session="turn", async_chunk=False):
|
||||
thinker = torch.nn.Module()
|
||||
thinker.make_empty_intermediate_tensors = lambda: None
|
||||
mocker.patch.object(omni, "init_vllm_registered_model", return_value=thinker)
|
||||
mocker.patch("vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat.patch_minicpmo_remote_config")
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(),
|
||||
multimodal_config=None,
|
||||
model_stage="llm",
|
||||
use_v2_model_runner=v2,
|
||||
session_mode=session,
|
||||
async_chunk=async_chunk,
|
||||
)
|
||||
)
|
||||
return omni.MiniCPMO45OmniForConditionalGeneration(vllm_config=config)
|
||||
|
||||
|
||||
def test_mrv2_thinker_keeps_native_multimodal_embeddings_and_raw_ids(mocker):
|
||||
model = _model(mocker)
|
||||
batch = SimpleNamespace(input_ids=torch.tensor([11, 22, 33]), num_tokens_after_padding=3)
|
||||
embeddings = torch.randn(3, 4)
|
||||
runner = SimpleNamespace(
|
||||
model=model,
|
||||
supports_mm_inputs=True,
|
||||
is_first_pp_rank=True,
|
||||
model_state=SimpleNamespace(dummy_inputs_embeds=lambda _: embeddings),
|
||||
)
|
||||
ids, embeds, _ = OmniGPUModelRunner._prepare_mm_inputs(runner, None, batch, dummy_run=True)
|
||||
assert ids is batch.input_ids
|
||||
assert embeds is embeddings
|
||||
hidden = torch.randn(3, 4)
|
||||
model.thinker.forward = mocker.Mock(return_value=hidden)
|
||||
result = model(ids, torch.tensor([5, 6, 7]), inputs_embeds=embeds)
|
||||
assert result is hidden
|
||||
torch.testing.assert_close(model.thinker.forward.call_args.kwargs["inputs_embeds"], embeddings)
|
||||
|
||||
|
||||
def test_row_ledger_uses_live_batch_after_replay(mocker):
|
||||
model = _model(mocker)
|
||||
# Chunked prefill (two tokens), decode (one token), and graph padding.
|
||||
batch = SimpleNamespace(
|
||||
input_ids=torch.tensor([11, 12, 41, 0]),
|
||||
positions=torch.tensor([6, 7, 20, 0]),
|
||||
)
|
||||
hidden = torch.randn(4, 8)
|
||||
for token in [41, 53]:
|
||||
batch.input_ids[2] = token
|
||||
batch.positions[2] += 1
|
||||
out = model.make_omni_output_mrv2(
|
||||
hidden,
|
||||
input_batch=batch,
|
||||
req_states=None,
|
||||
model_intermediate_buffer=[{}, {}],
|
||||
)
|
||||
assert out.text_hidden_states is hidden
|
||||
assert out.multimodal_outputs["latent"] is hidden
|
||||
torch.testing.assert_close(out.multimodal_outputs["latent_input_ids"], batch.input_ids[:, None])
|
||||
torch.testing.assert_close(out.multimodal_outputs["latent_positions"], batch.positions[:, None])
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared HiFT must retain its native path unless explicitly opted in."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core import hifigan
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan import HiFTGenerator
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_shared_hift_keeps_native_istft_without_opt_in(monkeypatch):
|
||||
n_fft, hop = 16, 4
|
||||
window = torch.hann_window(n_fft)
|
||||
hift = SimpleNamespace(
|
||||
istft_params={"n_fft": n_fft, "hop_len": hop},
|
||||
_get_stft_window=lambda tensor: window,
|
||||
)
|
||||
|
||||
def unexpected_cache(*args):
|
||||
raise AssertionError("shared HiFT must not opt in implicitly")
|
||||
|
||||
monkeypatch.setattr(hifigan, "_istft_without_host_sync", unexpected_cache)
|
||||
magnitude, phase = torch.rand(2, 9, 20), torch.randn(2, 9, 20)
|
||||
assert HiFTGenerator._istft(hift, magnitude, phase).shape == (2, 76)
|
||||
assert not hasattr(hift, "_istft_envelopes")
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The MiniCPM-o 4.5 vocoder stage's per-step path must not block the host."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core import hifigan
|
||||
from vllm_omni.worker.gpu_generation_model_runner import _HostCopyBatch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _no_host_sync():
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
|
||||
|
||||
def test_istft_is_bitwise_and_sync_free_on_cuda():
|
||||
n_fft, hop = 16, 4
|
||||
window = torch.hann_window(n_fft, periodic=True, device="cuda")
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
magnitude = torch.rand(4, n_fft // 2 + 1, 300, device="cuda", generator=generator) * 3
|
||||
phase = torch.randn(4, n_fft // 2 + 1, 300, device="cuda", generator=generator)
|
||||
spec = torch.complex(magnitude * torch.cos(phase), magnitude * torch.sin(phase))
|
||||
envelopes: dict = {}
|
||||
expected = torch.istft(spec, n_fft, hop, n_fft, window=window)
|
||||
# The first call per frame count builds and checks the envelope.
|
||||
hifigan._istft_without_host_sync(spec, n_fft, hop, window, envelopes)
|
||||
with _no_host_sync():
|
||||
actual = hifigan._istft_without_host_sync(spec, n_fft, hop, window, envelopes)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
def test_host_copy_batch_copies_without_blocking_until_wait():
|
||||
to_host = _HostCopyBatch(pin_memory=True)
|
||||
outputs = [torch.randn(1000, device="cuda") * (row + 1) for row in range(4)]
|
||||
strided = torch.randn(8, 6, device="cuda").t()
|
||||
to_host.copy(torch.zeros(1, device="cuda"))
|
||||
to_host.wait()
|
||||
with _no_host_sync():
|
||||
copies = [to_host.copy(tensor) for tensor in outputs]
|
||||
copied_strided = to_host.copy(strided)
|
||||
to_host.wait()
|
||||
for copy, tensor in zip(copies, outputs, strict=True):
|
||||
assert copy.device.type == "cpu" and copy.is_pinned()
|
||||
assert torch.equal(copy, tensor.cpu())
|
||||
assert copied_strided.is_contiguous() and torch.equal(copied_strided, strided.cpu())
|
||||
Reference in new issue
Block a user