[Model] Add MiniCPM-o 4.5 turn-mode MRv2 and batched audio inference (#8222)

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 authored and GitHub committed 2026-09-29 11:41:39 -07:00
1 parent 43ef1e61fb
commit 5af3a16526
28 files changed
+1577 -51

No files matched your search

@@ -19,6 +19,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _StageModel(torch.nn.Module):
logits_vocab_size = 6562
def make_empty_intermediate_tensors(self):
return None
@@ -1969,7 +1969,10 @@ def test_whole_euler_query_bucket_keeps_current_first_cache_layout(monkeypatch:
(_whole_euler_chunk(1, 8), 6, 2), # narrow padded chunk: capture pads 8 -> 16
(_whole_euler_chunk(1, 16), 16, 0), # consumes the narrow chunk's cache
]
caches = {"exact": (None, None), "bucketed": (None, None)}
caches: dict[str, tuple[torch.Tensor | None, torch.Tensor | None]] = {
"exact": (None, None),
"bucketed": (None, None),
}
for chunk, mel_frames, pad_frames in chunks:
chunk["x"][:, :, mel_frames:] = 0.0
results = {}
@@ -2319,3 +2322,24 @@ def test_whole_euler_groups_are_acquired_again_after_a_flush(monkeypatch: pytest
assert calls == [16, 1, 16, 1]
flush_always = True
assert wrapper._group_entries(groups, fills) is None
@pytest.mark.parametrize("fail", [False, True])
def test_vocoder_restores_tf32_policy(fail):
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import MiniCPMO45Code2Wav
previous = torch.backends.cuda.matmul.allow_tf32
def forward(*args, **kwargs):
assert torch.backends.cuda.matmul.allow_tf32
if fail:
raise RuntimeError("injected")
return "ok"
model = SimpleNamespace(_extra_config=lambda: {"token2wav_allow_tf32": True}, _forward_impl=forward)
if fail:
with pytest.raises(RuntimeError, match="injected"):
MiniCPMO45Code2Wav.forward(model)
else:
assert MiniCPMO45Code2Wav.forward(model) == "ok"
assert torch.backends.cuda.matmul.allow_tf32 == previous
@@ -20,6 +20,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_multimodal_runner_preserves_forwarded_token_identities(monkeypatch):
stage_model = torch.nn.Module()
stage_model.logits_vocab_size = 6562
stage_model.make_empty_intermediate_tensors = lambda: None
monkeypatch.setattr(
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni.init_vllm_registered_model",
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""MiniCPM-o 4.5 Talker per-step path: batched, host-sync-free, same results."""
from __future__ import annotations
import copy
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
import pytest
import torch
import torch.nn as nn
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
ConditionalChatTTSConfig,
)
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
MiniCPMO45OmniTTSForConditionalGeneration,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_VOCAB = 8
_EOS = 7
@dataclass
class _SamplingMetadata:
repetition_penalties: torch.Tensor
prompt_token_ids: torch.Tensor | None = None
no_penalties: bool = False
def _make_talker(device: str = "cpu") -> MiniCPMO45OmniTTSForConditionalGeneration:
talker = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
nn.Module.__init__(talker)
talker._num_audio_tokens = _VOCAB
talker._codec_eos_id = _EOS
talker._force_eos_rows = None
talker._mask_eos_rows = None
talker._pending_force_eos_rows = None
talker._penalty_histories = None
talker._request_audio_states = {}
talker._request_condition_states = {}
talker._deferred_cleanup_ids = set()
talker._tts_config = ConditionalChatTTSConfig()
generator = torch.Generator().manual_seed(0)
talker.emb_code = nn.ModuleList([nn.Embedding(_VOCAB, 4)])
talker.head_code = nn.ModuleList([nn.Linear(4, _VOCAB, bias=False)])
with torch.no_grad():
talker.emb_code[0].weight.copy_(torch.randn(_VOCAB, 4, generator=generator))
talker.head_code[0].weight.copy_(torch.randn(_VOCAB, 4, generator=generator))
return talker.to(device)
def _states() -> dict[str, dict[str, Any]]:
return {
# Mid-stream row: forwards its code, EOS still masked by min_tokens.
"req-live": {"finished": False, "step": 2, "max_tokens": 100, "min_tokens": 5, "recent_codes": [1, 2]},
# The previous step sampled codec EOS: forward nothing, finish, force EOS.
"req-eos": {"finished": False, "step": 60, "max_tokens": 100, "recent_codes": [4, 4, 5]},
# Leftover decode of an already-finished request.
"req-done": {"finished": True, "step": 3, "max_tokens": 100},
# Reaches its codec budget on this step.
"req-cap": {"finished": False, "step": 8, "max_tokens": 10, "recent_codes": list(range(7)) * 3},
}
def _infos(states: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
return [
{"request_id": request_id, "audio_state": dict(state), "_omni_is_prefill": False}
for request_id, state in states.items()
]
def _step(talker, infos, hidden, mocker, *, batched: bool, input_ids: torch.Tensor, penalties: torch.Tensor):
if batched:
returned_ids, embeds, updates = talker.preprocess_decode_batch(input_ids=input_ids, req_infos=infos)
assert returned_ids is input_ids
else:
rows = [talker.preprocess(input_ids[row : row + 1], None, **info) for row, info in enumerate(infos)]
embeds = torch.cat([row[1] for row in rows])
updates = [row[2] for row in rows]
for info, update in zip(infos, updates, strict=True):
info["codes"] = update["codes"]
output = talker.make_omni_output(
hidden,
model_intermediate_buffer=infos,
request_token_spans=[(row, row + 1) for row in range(len(infos))],
)
logits = talker.compute_logits(output.text_hidden_states)
captured: dict[str, torch.Tensor] = {}
def _sampler(logits, sampling_metadata):
captured["logits"] = logits.clone()
return SimpleNamespace(sampled_token_ids=logits.argmax(dim=-1, keepdim=True))
mocker.patch(
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.Sampler",
return_value=_sampler,
)
sampled = talker.sample(logits, _SamplingMetadata(repetition_penalties=penalties))
return embeds, output, logits, captured["logits"], sampled.sampled_token_ids
def test_batched_decode_matches_scalar_preprocess(mocker) -> None:
input_ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32)
hidden = torch.randn(4, 4, generator=torch.Generator().manual_seed(1))
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0])
results = {}
talkers = {}
for batched in (False, True):
talker = _make_talker()
states = _states()
talker._request_audio_states = copy.deepcopy(states)
results[batched] = _step(
talker, _infos(states), hidden, mocker, batched=batched, input_ids=input_ids, penalties=penalties
)
talkers[batched] = talker
# Embeddings, masked logits, penalized logits and sampled ids.
for index in (0, 2, 3, 4):
assert torch.equal(results[False][index], results[True][index])
for result in results.values():
output = result[1].multimodal_outputs
assert [t.tolist() for t in output["codes"]["audio"]] == [[[3]], [], [], [[6]]]
assert [t.item() for t in output["meta"]["finished"]] == [False, True, True, True]
assert talkers[True]._request_audio_states == talkers[False]._request_audio_states
assert talkers[True]._decode_codec_ids == {}
@@ -0,0 +1,101 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The MiniCPM-o 4.5 Talker's per-step path must not block the host on the GPU."""
from __future__ import annotations
import copy
import pytest
import torch
import torch.nn as nn
from tests.model_executor.models.minicpmo_4_5.test_talker_host_sync import _EOS, _infos, _make_talker, _states, _step
from vllm_omni.utils.device_copy import index_to_device
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
def test_talker_step_does_not_synchronize_cuda(mocker) -> None:
talker = _make_talker("cuda")
states = _states()
talker._request_audio_states = copy.deepcopy(states)
infos = _infos(states)
input_ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32, device="cuda")
hidden = torch.randn(4, 4, device="cuda")
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0], device="cuda")
# Warm the pinned host allocator and CUDA context outside the check.
index_to_device([1], "cuda")
previous = torch.cuda.get_sync_debug_mode()
try:
torch.cuda.set_sync_debug_mode("error")
with torch.inference_mode():
*_, sampled = _step(talker, infos, hidden, mocker, batched=True, input_ids=input_ids, penalties=penalties)
finally:
torch.cuda.set_sync_debug_mode(previous)
assert sampled.reshape(-1).tolist()[1:] == [_EOS, _EOS, _EOS]
assert talker._request_audio_states["req-live"]["recent_codes"] == [1, 2, 3]
assert talker._request_audio_states["req-eos"]["finished"] is True
def test_talker_condition_upload_does_not_synchronize_cuda() -> None:
talker = _make_talker("cuda")
talker.emb_text = nn.Embedding(16, 4).cuda()
talker.projector_semantic = nn.Linear(6, 4).cuda()
talker._normalize = True
talker._text_eos_id = 14
talker._tts_bos_id = 15
token_ids = torch.tensor([2, 3, 4])
hidden_states = torch.randn(3, 6)
expected = talker._build_condition_embeddings(token_ids.cuda(), hidden_states.cuda())
previous = torch.cuda.get_sync_debug_mode()
try:
torch.cuda.set_sync_debug_mode("error")
with torch.inference_mode():
condition = talker._build_condition_embeddings(token_ids, hidden_states)
boundary = talker._build_condition_embeddings(token_ids[:0], hidden_states[:0])
finally:
torch.cuda.set_sync_debug_mode(previous)
assert torch.equal(condition, expected)
assert boundary.shape == (2, 4)
@torch.inference_mode()
def test_v1_batched_decode_and_async_snapshot_match_scalar(mocker):
from vllm_omni.worker.gpu_ar_model_runner import _snapshot_tensor_payload_to_cpu_async
ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32, device="cuda")
hidden = torch.arange(16, dtype=torch.float32, device="cuda").reshape(4, 4)
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0], device="cuda")
results, states = [], []
for batched in (False, True):
talker = _make_talker("cuda")
talker._request_audio_states = copy.deepcopy(_states())
results.append(
_step(talker, _infos(_states()), hidden, mocker, batched=batched, input_ids=ids, penalties=penalties)
)
states.append(copy.deepcopy(talker._request_audio_states))
for index in (0, 2, 3, 4):
torch.testing.assert_close(results[0][index], results[1][index], rtol=0, atol=0)
assert states[0] == states[1]
reference, candidate = [result[1].multimodal_outputs for result in results]
for group in ("codes", "meta"):
for key in reference[group]:
for left, right in zip(reference[group][key], candidate[group][key], strict=True):
torch.testing.assert_close(left, right, rtol=0, atol=0)
snapshot = _snapshot_tensor_payload_to_cpu_async(
{"hidden_states": hidden, "multimodal_outputs": candidate},
copy_stream=torch.cuda.Stream(),
pin_memory=True,
)
expected_hidden = hidden.cpu().clone()
snapshot.wait()
torch.testing.assert_close(snapshot.payload["hidden_states"], expected_hidden, rtol=0, atol=0)
for group in ("codes", "meta"):
for key in reference[group]:
for left, right in zip(
reference[group][key], snapshot.payload["multimodal_outputs"][group][key], strict=True
):
torch.testing.assert_close(left.cpu(), right, rtol=0, atol=0)
hidden.fill_(-1)
torch.testing.assert_close(snapshot.payload["hidden_states"], expected_hidden, rtol=0, atol=0)
@@ -0,0 +1,140 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Model Runner V2 path of MiniCPM-o 4.5's Talker: device-side codec output,
EOS control and the 16-frame codec penalty must match the V1 host path."""
from __future__ import annotations
from types import SimpleNamespace
from typing import cast
import numpy as np
import pytest
import torch
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
_OFFLINE_CODEC_MAX_NEW_TOKENS,
MiniCPMO45OmniTTSForConditionalGeneration,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_EOS = 6561
def _talker(max_reqs: int = 8, max_position_embeddings: int = 4096):
talker = object.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
talker._codec_eos_id = _EOS
talker._tts_config = SimpleNamespace(max_position_embeddings=max_position_embeddings)
talker._mrv2_empty_speech = torch.zeros(max_reqs, dtype=torch.bool)
talker._mrv2_forced_eos = None
talker._mrv2_decode_rows_logged = False
return talker
def _batch(rows: list[dict], *, pad_to: int | None = None):
"""rows: one dict per request with slot, prompt_len, span (token ids) and prefill flag."""
ids = [token for row in rows for token in row["span"]]
num_tokens = len(ids)
padded = pad_to or num_tokens
starts = np.cumsum([0] + [len(row["span"]) for row in rows])
seq_lens = [row["computed"] + len(row["span"]) for row in rows]
return SimpleNamespace(
num_reqs=len(rows),
has_prefill=any(row["prefill"] for row in rows),
is_prefilling_np=np.array([row["prefill"] for row in rows]),
idx_mapping_np=np.array([row["slot"] for row in rows]),
idx_mapping=torch.tensor([row["slot"] for row in rows]),
input_ids=torch.tensor(ids + [0] * (padded - num_tokens), dtype=torch.int32),
logits_indices=torch.tensor(starts[1:] - 1),
seq_lens=torch.tensor(seq_lens + [0] * 2, dtype=torch.int32),
), padded
def _req_states(prompt_lens: dict[int, int], max_reqs: int = 8):
prompt_len = torch.zeros(max_reqs, dtype=torch.int32)
for slot, value in prompt_lens.items():
prompt_len[slot] = value
return SimpleNamespace(prompt_len=SimpleNamespace(gpu=prompt_len))
def test_mrv2_output_emits_decode_input_codes_with_validity_and_forced_eos() -> None:
talker = _talker()
rows = [
# Final prefill chunk: no code yet (V1 emits an empty delta).
dict(slot=2, prompt_len=5, computed=0, span=[0, 0, 0, 0, 0], prefill=True),
# Decode with a codec id: emitted this step.
dict(slot=0, prompt_len=4, computed=6, span=[17], prefill=False),
# Decode whose input is codec EOS: request ended, force EOS again.
dict(slot=1, prompt_len=4, computed=8, span=[_EOS], prefill=False),
]
batch, padded = _batch(rows, pad_to=8)
buffers = [{"audio_state": {"finished": False}}, {}, {}]
hidden = torch.zeros((padded, 4))
out = talker.make_omni_output_mrv2(
hidden,
input_batch=batch,
req_states=_req_states({cast(int, row["slot"]): cast(int, row["prompt_len"]) for row in rows}),
model_intermediate_buffer=buffers,
)
assert isinstance(out, OmniOutput)
codes = out.multimodal_outputs["codes"]["audio"]
valid = out.multimodal_outputs["meta"]["codec_frame_valid"]
assert codes.shape == (padded, 1) and valid.shape == (padded,)
assert codes[:, 0].tolist() == batch.input_ids.tolist()
assert valid.tolist() == [False] * 5 + [True, False] + [False]
assert talker.take_mrv2_forced_eos(batch, None, 3).tolist() == [False, False, True]
# Consumed once; a warmup sampler call sees nothing.
assert talker.take_mrv2_forced_eos(batch, None, 3) is None
def test_mrv2_output_empty_condition_and_length_cap() -> None:
talker = _talker()
prompt_lens = {3: 6, 4: 4096 - 100}
# Empty Thinker condition: preprocess marks the request finished at prefill.
batch, _ = _batch([dict(slot=3, prompt_len=6, computed=0, span=[0] * 6, prefill=True)])
talker.make_omni_output_mrv2(
torch.zeros((6, 4)),
input_batch=batch,
req_states=_req_states(prompt_lens),
model_intermediate_buffer=[{"audio_state": {"finished": True}}],
)
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [True]
# Its later decode rows stay forced and emit nothing, whatever the input id.
batch, _ = _batch([dict(slot=3, prompt_len=6, computed=6, span=[42], prefill=False)])
out = talker.make_omni_output_mrv2(
torch.zeros((1, 4)), input_batch=batch, req_states=_req_states(prompt_lens), model_intermediate_buffer=[{}]
)
assert out.multimodal_outputs["meta"]["codec_frame_valid"].tolist() == [False]
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [True]
# Offline cap: min(2048, context - prompt) samples, the last one EOS.
limit = min(_OFFLINE_CODEC_MAX_NEW_TOKENS, 100) - 1
for step, forced in ((limit - 1, False), (limit, True)):
batch, _ = _batch(
[dict(slot=4, prompt_len=prompt_lens[4], computed=prompt_lens[4] + step - 1, span=[5], prefill=False)]
)
out = talker.make_omni_output_mrv2(
torch.zeros((1, 4)), input_batch=batch, req_states=_req_states(prompt_lens), model_intermediate_buffer=[{}]
)
assert out.multimodal_outputs["meta"]["codec_frame_valid"].tolist() == [True]
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [forced]
@pytest.mark.parametrize("forced", [None, [False, True]])
def test_sampler_adapter_keeps_upstream_counts_and_only_forces_codec_eos(mocker, forced):
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import MiniCPMO45TalkerSampler
output = SimpleNamespace(sampled_token_ids=torch.tensor([[2], [3]]), num_sampled=torch.tensor([1, 0]))
base = mocker.Mock(return_value=output)
base.req_states = object()
mask = None if forced is None else torch.tensor(forced)
talker = SimpleNamespace(_codec_eos_id=7, take_mrv2_forced_eos=mocker.Mock(return_value=mask))
sampler = MiniCPMO45TalkerSampler(base, talker)
logits, batch = torch.zeros(2, 8), object()
assert sampler(logits, batch) is output
assert output.sampled_token_ids.tolist() == [[2], [3 if forced is None else 7]]
assert output.num_sampled.tolist() == [1, 0]
base.assert_called_once_with(logits, batch)
talker.take_mrv2_forced_eos.assert_called_once_with(batch, base.req_states, 2)
@@ -0,0 +1,100 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""MRv2 Thinker preserves multimodal encoding and the llm2tts row ledger."""
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.models.minicpmo_4_5 import minicpmo_4_5_omni as omni
from vllm_omni.worker_v2.omni_model_runner import OmniGPUModelRunner
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
"profile,capacities,kv_gib",
[
("minicpmo_4_5_turn_mrv2.yaml", [16, 8, 8], 2),
("minicpmo_4_5_turn_mrv2_h200.yaml", [16, 16, 8], 4),
],
)
def test_mrv2_profile_retains_full_thinker_handoff(profile, capacities, kv_gib, monkeypatch):
from pathlib import Path
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config, merge_pipeline_deploy
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
from vllm_omni.platforms import current_omni_platform
# merge_pipeline_deploy resolves the platform again; test the CUDA profile
# consistently even when this CPU test runs on a ROCm host.
monkeypatch.setattr(current_omni_platform, "device_name", "cuda")
deploy = Path(__file__).resolve().parents[4] / "vllm_omni/deploy" / profile
config = _apply_platform_overrides(load_deploy_config(deploy), platform="cuda")
stages = merge_pipeline_deploy(MINICPMO_4_5_PIPELINE, config)
assert [s.yaml_engine_args["use_v2_model_runner"] for s in stages] == [True, True, True]
assert [s.yaml_engine_args["async_chunk"] for s in stages] == [False, True, True]
assert [s.yaml_engine_args["max_num_seqs"] for s in stages] == capacities
assert stages[1].yaml_engine_args["kv_cache_memory_bytes"] == kv_gib * 1024**3
def _model(mocker, *, v2=True, session="turn", async_chunk=False):
thinker = torch.nn.Module()
thinker.make_empty_intermediate_tensors = lambda: None
mocker.patch.object(omni, "init_vllm_registered_model", return_value=thinker)
mocker.patch("vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat.patch_minicpmo_remote_config")
config = SimpleNamespace(
model_config=SimpleNamespace(
hf_config=SimpleNamespace(),
multimodal_config=None,
model_stage="llm",
use_v2_model_runner=v2,
session_mode=session,
async_chunk=async_chunk,
)
)
return omni.MiniCPMO45OmniForConditionalGeneration(vllm_config=config)
def test_mrv2_thinker_keeps_native_multimodal_embeddings_and_raw_ids(mocker):
model = _model(mocker)
batch = SimpleNamespace(input_ids=torch.tensor([11, 22, 33]), num_tokens_after_padding=3)
embeddings = torch.randn(3, 4)
runner = SimpleNamespace(
model=model,
supports_mm_inputs=True,
is_first_pp_rank=True,
model_state=SimpleNamespace(dummy_inputs_embeds=lambda _: embeddings),
)
ids, embeds, _ = OmniGPUModelRunner._prepare_mm_inputs(runner, None, batch, dummy_run=True)
assert ids is batch.input_ids
assert embeds is embeddings
hidden = torch.randn(3, 4)
model.thinker.forward = mocker.Mock(return_value=hidden)
result = model(ids, torch.tensor([5, 6, 7]), inputs_embeds=embeds)
assert result is hidden
torch.testing.assert_close(model.thinker.forward.call_args.kwargs["inputs_embeds"], embeddings)
def test_row_ledger_uses_live_batch_after_replay(mocker):
model = _model(mocker)
# Chunked prefill (two tokens), decode (one token), and graph padding.
batch = SimpleNamespace(
input_ids=torch.tensor([11, 12, 41, 0]),
positions=torch.tensor([6, 7, 20, 0]),
)
hidden = torch.randn(4, 8)
for token in [41, 53]:
batch.input_ids[2] = token
batch.positions[2] += 1
out = model.make_omni_output_mrv2(
hidden,
input_batch=batch,
req_states=None,
model_intermediate_buffer=[{}, {}],
)
assert out.text_hidden_states is hidden
assert out.multimodal_outputs["latent"] is hidden
torch.testing.assert_close(out.multimodal_outputs["latent_input_ids"], batch.input_ids[:, None])
torch.testing.assert_close(out.multimodal_outputs["latent_positions"], batch.positions[:, None])
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared HiFT must retain its native path unless explicitly opted in."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core import hifigan
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan import HiFTGenerator
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_shared_hift_keeps_native_istft_without_opt_in(monkeypatch):
n_fft, hop = 16, 4
window = torch.hann_window(n_fft)
hift = SimpleNamespace(
istft_params={"n_fft": n_fft, "hop_len": hop},
_get_stft_window=lambda tensor: window,
)
def unexpected_cache(*args):
raise AssertionError("shared HiFT must not opt in implicitly")
monkeypatch.setattr(hifigan, "_istft_without_host_sync", unexpected_cache)
magnitude, phase = torch.rand(2, 9, 20), torch.randn(2, 9, 20)
assert HiFTGenerator._istft(hift, magnitude, phase).shape == (2, 76)
assert not hasattr(hift, "_istft_envelopes")
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The MiniCPM-o 4.5 vocoder stage's per-step path must not block the host."""
from __future__ import annotations
from contextlib import contextmanager
import pytest
import torch
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core import hifigan
from vllm_omni.worker.gpu_generation_model_runner import _HostCopyBatch
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
@contextmanager
def _no_host_sync():
previous = torch.cuda.get_sync_debug_mode()
torch.cuda.set_sync_debug_mode("error")
try:
yield
finally:
torch.cuda.set_sync_debug_mode(previous)
def test_istft_is_bitwise_and_sync_free_on_cuda():
n_fft, hop = 16, 4
window = torch.hann_window(n_fft, periodic=True, device="cuda")
generator = torch.Generator(device="cuda").manual_seed(0)
magnitude = torch.rand(4, n_fft // 2 + 1, 300, device="cuda", generator=generator) * 3
phase = torch.randn(4, n_fft // 2 + 1, 300, device="cuda", generator=generator)
spec = torch.complex(magnitude * torch.cos(phase), magnitude * torch.sin(phase))
envelopes: dict = {}
expected = torch.istft(spec, n_fft, hop, n_fft, window=window)
# The first call per frame count builds and checks the envelope.
hifigan._istft_without_host_sync(spec, n_fft, hop, window, envelopes)
with _no_host_sync():
actual = hifigan._istft_without_host_sync(spec, n_fft, hop, window, envelopes)
assert torch.equal(actual, expected)
def test_host_copy_batch_copies_without_blocking_until_wait():
to_host = _HostCopyBatch(pin_memory=True)
outputs = [torch.randn(1000, device="cuda") * (row + 1) for row in range(4)]
strided = torch.randn(8, 6, device="cuda").t()
to_host.copy(torch.zeros(1, device="cuda"))
to_host.wait()
with _no_host_sync():
copies = [to_host.copy(tensor) for tensor in outputs]
copied_strided = to_host.copy(strided)
to_host.wait()
for copy, tensor in zip(copies, outputs, strict=True):
assert copy.device.type == "cpu" and copy.is_pinned()
assert torch.equal(copy, tensor.cpu())
assert copied_strided.is_contiguous() and torch.equal(copied_strided, strided.cpu())