[Model] Add MiniCPM-o 4.5 turn-mode MRv2 and batched audio inference (#8222)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
28 files changed
+1577
-51
No files matched your search
@@ -115,3 +115,5 @@ The design contracts separate selection mechanics from backend algorithms:
|
||||
The pre-#5137 pages are preserved in the
|
||||
[legacy module archive](module/archive/README.md) for historical reference and
|
||||
are not active design contracts.
|
||||
|
||||
- [MiniCPM-o 4.5 turn-mode MRv2 performance](minicpm_o45_mrv2_performance.md)
|
||||
@@ -0,0 +1,22 @@
|
||||
# MiniCPM-o 4.5 MRv2 profiles
|
||||
|
||||
Turn mode uses the stage-level MRv2 contracts from #8184. Thinker emits live
|
||||
latent metadata outside graph replay; Talker keeps codec history and EOS
|
||||
control on device. Code2Wav reuses mainline Whole-Euler Flow graphs and shared
|
||||
prompt state. The previous experimental block compilation, tiled attention,
|
||||
channels-last and merged-CFM implementations are removed.
|
||||
|
||||
`minicpmo_4_5_turn_mrv2_h200.yaml` selects Talker capacity 16 and 4 GiB KV;
|
||||
the generic MRv2 profile selects capacity 8 and 2 GiB KV. These configurations
|
||||
require new end-to-end performance measurements after the mainline integration.
|
||||
Previously reported numbers do not describe this revised codec backend.
|
||||
|
||||
The generic MRv2 profile and the opt-in native V1 duplex H200 profile enable
|
||||
TF32 matmuls within Code2Wav forward/capture only; the previous process policy
|
||||
is restored afterwards. cuDNN's TF32 policy is independent. TF32 changes rounding.
|
||||
Duplex keeps the mainline V1 session path. Turn results do not establish duplex
|
||||
performance or interruption correctness.
|
||||
|
||||
The shared asynchronous output snapshot and batched Talker preprocessing also
|
||||
apply to V1 when async chunking/scheduling is enabled. Default V1 behavior must
|
||||
therefore be included in end-to-end regression validation.
|
||||
@@ -325,6 +325,8 @@ skip_covered = true
|
||||
|
||||
[tool.typos.default]
|
||||
extend-ignore-identifiers-re = [
|
||||
# Canonical PyTorch CUDA device property.
|
||||
"^shared_memory_per_block_optin$",
|
||||
".*_thw",
|
||||
".*thw",
|
||||
"ein",
|
||||
|
||||
@@ -97,6 +97,7 @@ def test_duplex_session_runtime_rejects_non_positive_values(tmp_path, name: str,
|
||||
("deploy_yaml", "expected"),
|
||||
[
|
||||
("minicpmo_4_5.yaml", 16),
|
||||
("minicpmo_4_5_duplex_h200.yaml", 16),
|
||||
("minicpmo_4_5_8x4090.yaml", 1),
|
||||
("minicpmo_4_5_3gpu_stage1_replicas.yaml", 16),
|
||||
],
|
||||
|
||||
@@ -19,6 +19,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _StageModel(torch.nn.Module):
|
||||
logits_vocab_size = 6562
|
||||
|
||||
def make_empty_intermediate_tensors(self):
|
||||
return None
|
||||
|
||||
|
||||
@@ -1969,7 +1969,10 @@ def test_whole_euler_query_bucket_keeps_current_first_cache_layout(monkeypatch:
|
||||
(_whole_euler_chunk(1, 8), 6, 2), # narrow padded chunk: capture pads 8 -> 16
|
||||
(_whole_euler_chunk(1, 16), 16, 0), # consumes the narrow chunk's cache
|
||||
]
|
||||
caches = {"exact": (None, None), "bucketed": (None, None)}
|
||||
caches: dict[str, tuple[torch.Tensor | None, torch.Tensor | None]] = {
|
||||
"exact": (None, None),
|
||||
"bucketed": (None, None),
|
||||
}
|
||||
for chunk, mel_frames, pad_frames in chunks:
|
||||
chunk["x"][:, :, mel_frames:] = 0.0
|
||||
results = {}
|
||||
@@ -2319,3 +2322,24 @@ def test_whole_euler_groups_are_acquired_again_after_a_flush(monkeypatch: pytest
|
||||
assert calls == [16, 1, 16, 1]
|
||||
flush_always = True
|
||||
assert wrapper._group_entries(groups, fills) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", [False, True])
|
||||
def test_vocoder_restores_tf32_policy(fail):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import MiniCPMO45Code2Wav
|
||||
|
||||
previous = torch.backends.cuda.matmul.allow_tf32
|
||||
|
||||
def forward(*args, **kwargs):
|
||||
assert torch.backends.cuda.matmul.allow_tf32
|
||||
if fail:
|
||||
raise RuntimeError("injected")
|
||||
return "ok"
|
||||
|
||||
model = SimpleNamespace(_extra_config=lambda: {"token2wav_allow_tf32": True}, _forward_impl=forward)
|
||||
if fail:
|
||||
with pytest.raises(RuntimeError, match="injected"):
|
||||
MiniCPMO45Code2Wav.forward(model)
|
||||
else:
|
||||
assert MiniCPMO45Code2Wav.forward(model) == "ok"
|
||||
assert torch.backends.cuda.matmul.allow_tf32 == previous
|
||||
@@ -20,6 +20,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
def test_multimodal_runner_preserves_forwarded_token_identities(monkeypatch):
|
||||
stage_model = torch.nn.Module()
|
||||
stage_model.logits_vocab_size = 6562
|
||||
stage_model.make_empty_intermediate_tensors = lambda: None
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni.init_vllm_registered_model",
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MiniCPM-o 4.5 Talker per-step path: batched, host-sync-free, same results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
|
||||
ConditionalChatTTSConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
MiniCPMO45OmniTTSForConditionalGeneration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_VOCAB = 8
|
||||
_EOS = 7
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SamplingMetadata:
|
||||
repetition_penalties: torch.Tensor
|
||||
prompt_token_ids: torch.Tensor | None = None
|
||||
no_penalties: bool = False
|
||||
|
||||
|
||||
def _make_talker(device: str = "cpu") -> MiniCPMO45OmniTTSForConditionalGeneration:
|
||||
talker = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
|
||||
nn.Module.__init__(talker)
|
||||
talker._num_audio_tokens = _VOCAB
|
||||
talker._codec_eos_id = _EOS
|
||||
talker._force_eos_rows = None
|
||||
talker._mask_eos_rows = None
|
||||
talker._pending_force_eos_rows = None
|
||||
talker._penalty_histories = None
|
||||
talker._request_audio_states = {}
|
||||
talker._request_condition_states = {}
|
||||
talker._deferred_cleanup_ids = set()
|
||||
talker._tts_config = ConditionalChatTTSConfig()
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
talker.emb_code = nn.ModuleList([nn.Embedding(_VOCAB, 4)])
|
||||
talker.head_code = nn.ModuleList([nn.Linear(4, _VOCAB, bias=False)])
|
||||
with torch.no_grad():
|
||||
talker.emb_code[0].weight.copy_(torch.randn(_VOCAB, 4, generator=generator))
|
||||
talker.head_code[0].weight.copy_(torch.randn(_VOCAB, 4, generator=generator))
|
||||
return talker.to(device)
|
||||
|
||||
|
||||
def _states() -> dict[str, dict[str, Any]]:
|
||||
return {
|
||||
# Mid-stream row: forwards its code, EOS still masked by min_tokens.
|
||||
"req-live": {"finished": False, "step": 2, "max_tokens": 100, "min_tokens": 5, "recent_codes": [1, 2]},
|
||||
# The previous step sampled codec EOS: forward nothing, finish, force EOS.
|
||||
"req-eos": {"finished": False, "step": 60, "max_tokens": 100, "recent_codes": [4, 4, 5]},
|
||||
# Leftover decode of an already-finished request.
|
||||
"req-done": {"finished": True, "step": 3, "max_tokens": 100},
|
||||
# Reaches its codec budget on this step.
|
||||
"req-cap": {"finished": False, "step": 8, "max_tokens": 10, "recent_codes": list(range(7)) * 3},
|
||||
}
|
||||
|
||||
|
||||
def _infos(states: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"request_id": request_id, "audio_state": dict(state), "_omni_is_prefill": False}
|
||||
for request_id, state in states.items()
|
||||
]
|
||||
|
||||
|
||||
def _step(talker, infos, hidden, mocker, *, batched: bool, input_ids: torch.Tensor, penalties: torch.Tensor):
|
||||
if batched:
|
||||
returned_ids, embeds, updates = talker.preprocess_decode_batch(input_ids=input_ids, req_infos=infos)
|
||||
assert returned_ids is input_ids
|
||||
else:
|
||||
rows = [talker.preprocess(input_ids[row : row + 1], None, **info) for row, info in enumerate(infos)]
|
||||
embeds = torch.cat([row[1] for row in rows])
|
||||
updates = [row[2] for row in rows]
|
||||
for info, update in zip(infos, updates, strict=True):
|
||||
info["codes"] = update["codes"]
|
||||
output = talker.make_omni_output(
|
||||
hidden,
|
||||
model_intermediate_buffer=infos,
|
||||
request_token_spans=[(row, row + 1) for row in range(len(infos))],
|
||||
)
|
||||
logits = talker.compute_logits(output.text_hidden_states)
|
||||
captured: dict[str, torch.Tensor] = {}
|
||||
|
||||
def _sampler(logits, sampling_metadata):
|
||||
captured["logits"] = logits.clone()
|
||||
return SimpleNamespace(sampled_token_ids=logits.argmax(dim=-1, keepdim=True))
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.Sampler",
|
||||
return_value=_sampler,
|
||||
)
|
||||
sampled = talker.sample(logits, _SamplingMetadata(repetition_penalties=penalties))
|
||||
return embeds, output, logits, captured["logits"], sampled.sampled_token_ids
|
||||
|
||||
|
||||
def test_batched_decode_matches_scalar_preprocess(mocker) -> None:
|
||||
input_ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32)
|
||||
hidden = torch.randn(4, 4, generator=torch.Generator().manual_seed(1))
|
||||
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0])
|
||||
results = {}
|
||||
talkers = {}
|
||||
for batched in (False, True):
|
||||
talker = _make_talker()
|
||||
states = _states()
|
||||
talker._request_audio_states = copy.deepcopy(states)
|
||||
results[batched] = _step(
|
||||
talker, _infos(states), hidden, mocker, batched=batched, input_ids=input_ids, penalties=penalties
|
||||
)
|
||||
talkers[batched] = talker
|
||||
|
||||
# Embeddings, masked logits, penalized logits and sampled ids.
|
||||
for index in (0, 2, 3, 4):
|
||||
assert torch.equal(results[False][index], results[True][index])
|
||||
for result in results.values():
|
||||
output = result[1].multimodal_outputs
|
||||
assert [t.tolist() for t in output["codes"]["audio"]] == [[[3]], [], [], [[6]]]
|
||||
assert [t.item() for t in output["meta"]["finished"]] == [False, True, True, True]
|
||||
assert talkers[True]._request_audio_states == talkers[False]._request_audio_states
|
||||
assert talkers[True]._decode_codec_ids == {}
|
||||
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The MiniCPM-o 4.5 Talker's per-step path must not block the host on the GPU."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from tests.model_executor.models.minicpmo_4_5.test_talker_host_sync import _EOS, _infos, _make_talker, _states, _step
|
||||
from vllm_omni.utils.device_copy import index_to_device
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
def test_talker_step_does_not_synchronize_cuda(mocker) -> None:
|
||||
talker = _make_talker("cuda")
|
||||
states = _states()
|
||||
talker._request_audio_states = copy.deepcopy(states)
|
||||
infos = _infos(states)
|
||||
input_ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32, device="cuda")
|
||||
hidden = torch.randn(4, 4, device="cuda")
|
||||
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0], device="cuda")
|
||||
# Warm the pinned host allocator and CUDA context outside the check.
|
||||
index_to_device([1], "cuda")
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
try:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
with torch.inference_mode():
|
||||
*_, sampled = _step(talker, infos, hidden, mocker, batched=True, input_ids=input_ids, penalties=penalties)
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
assert sampled.reshape(-1).tolist()[1:] == [_EOS, _EOS, _EOS]
|
||||
assert talker._request_audio_states["req-live"]["recent_codes"] == [1, 2, 3]
|
||||
assert talker._request_audio_states["req-eos"]["finished"] is True
|
||||
|
||||
|
||||
def test_talker_condition_upload_does_not_synchronize_cuda() -> None:
|
||||
talker = _make_talker("cuda")
|
||||
talker.emb_text = nn.Embedding(16, 4).cuda()
|
||||
talker.projector_semantic = nn.Linear(6, 4).cuda()
|
||||
talker._normalize = True
|
||||
talker._text_eos_id = 14
|
||||
talker._tts_bos_id = 15
|
||||
token_ids = torch.tensor([2, 3, 4])
|
||||
hidden_states = torch.randn(3, 6)
|
||||
expected = talker._build_condition_embeddings(token_ids.cuda(), hidden_states.cuda())
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
try:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
with torch.inference_mode():
|
||||
condition = talker._build_condition_embeddings(token_ids, hidden_states)
|
||||
boundary = talker._build_condition_embeddings(token_ids[:0], hidden_states[:0])
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
assert torch.equal(condition, expected)
|
||||
assert boundary.shape == (2, 4)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_v1_batched_decode_and_async_snapshot_match_scalar(mocker):
|
||||
from vllm_omni.worker.gpu_ar_model_runner import _snapshot_tensor_payload_to_cpu_async
|
||||
|
||||
ids = torch.tensor([3, _EOS, _EOS, 6], dtype=torch.int32, device="cuda")
|
||||
hidden = torch.arange(16, dtype=torch.float32, device="cuda").reshape(4, 4)
|
||||
penalties = torch.tensor([1.05, 1.2, 1.05, 1.0], device="cuda")
|
||||
results, states = [], []
|
||||
for batched in (False, True):
|
||||
talker = _make_talker("cuda")
|
||||
talker._request_audio_states = copy.deepcopy(_states())
|
||||
results.append(
|
||||
_step(talker, _infos(_states()), hidden, mocker, batched=batched, input_ids=ids, penalties=penalties)
|
||||
)
|
||||
states.append(copy.deepcopy(talker._request_audio_states))
|
||||
for index in (0, 2, 3, 4):
|
||||
torch.testing.assert_close(results[0][index], results[1][index], rtol=0, atol=0)
|
||||
assert states[0] == states[1]
|
||||
reference, candidate = [result[1].multimodal_outputs for result in results]
|
||||
for group in ("codes", "meta"):
|
||||
for key in reference[group]:
|
||||
for left, right in zip(reference[group][key], candidate[group][key], strict=True):
|
||||
torch.testing.assert_close(left, right, rtol=0, atol=0)
|
||||
snapshot = _snapshot_tensor_payload_to_cpu_async(
|
||||
{"hidden_states": hidden, "multimodal_outputs": candidate},
|
||||
copy_stream=torch.cuda.Stream(),
|
||||
pin_memory=True,
|
||||
)
|
||||
expected_hidden = hidden.cpu().clone()
|
||||
snapshot.wait()
|
||||
torch.testing.assert_close(snapshot.payload["hidden_states"], expected_hidden, rtol=0, atol=0)
|
||||
for group in ("codes", "meta"):
|
||||
for key in reference[group]:
|
||||
for left, right in zip(
|
||||
reference[group][key], snapshot.payload["multimodal_outputs"][group][key], strict=True
|
||||
):
|
||||
torch.testing.assert_close(left.cpu(), right, rtol=0, atol=0)
|
||||
hidden.fill_(-1)
|
||||
torch.testing.assert_close(snapshot.payload["hidden_states"], expected_hidden, rtol=0, atol=0)
|
||||
@@ -0,0 +1,140 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Model Runner V2 path of MiniCPM-o 4.5's Talker: device-side codec output,
|
||||
EOS control and the 16-frame codec penalty must match the V1 host path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
_OFFLINE_CODEC_MAX_NEW_TOKENS,
|
||||
MiniCPMO45OmniTTSForConditionalGeneration,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_EOS = 6561
|
||||
|
||||
|
||||
def _talker(max_reqs: int = 8, max_position_embeddings: int = 4096):
|
||||
talker = object.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
|
||||
talker._codec_eos_id = _EOS
|
||||
talker._tts_config = SimpleNamespace(max_position_embeddings=max_position_embeddings)
|
||||
talker._mrv2_empty_speech = torch.zeros(max_reqs, dtype=torch.bool)
|
||||
talker._mrv2_forced_eos = None
|
||||
talker._mrv2_decode_rows_logged = False
|
||||
return talker
|
||||
|
||||
|
||||
def _batch(rows: list[dict], *, pad_to: int | None = None):
|
||||
"""rows: one dict per request with slot, prompt_len, span (token ids) and prefill flag."""
|
||||
ids = [token for row in rows for token in row["span"]]
|
||||
num_tokens = len(ids)
|
||||
padded = pad_to or num_tokens
|
||||
starts = np.cumsum([0] + [len(row["span"]) for row in rows])
|
||||
seq_lens = [row["computed"] + len(row["span"]) for row in rows]
|
||||
return SimpleNamespace(
|
||||
num_reqs=len(rows),
|
||||
has_prefill=any(row["prefill"] for row in rows),
|
||||
is_prefilling_np=np.array([row["prefill"] for row in rows]),
|
||||
idx_mapping_np=np.array([row["slot"] for row in rows]),
|
||||
idx_mapping=torch.tensor([row["slot"] for row in rows]),
|
||||
input_ids=torch.tensor(ids + [0] * (padded - num_tokens), dtype=torch.int32),
|
||||
logits_indices=torch.tensor(starts[1:] - 1),
|
||||
seq_lens=torch.tensor(seq_lens + [0] * 2, dtype=torch.int32),
|
||||
), padded
|
||||
|
||||
|
||||
def _req_states(prompt_lens: dict[int, int], max_reqs: int = 8):
|
||||
prompt_len = torch.zeros(max_reqs, dtype=torch.int32)
|
||||
for slot, value in prompt_lens.items():
|
||||
prompt_len[slot] = value
|
||||
return SimpleNamespace(prompt_len=SimpleNamespace(gpu=prompt_len))
|
||||
|
||||
|
||||
def test_mrv2_output_emits_decode_input_codes_with_validity_and_forced_eos() -> None:
|
||||
talker = _talker()
|
||||
rows = [
|
||||
# Final prefill chunk: no code yet (V1 emits an empty delta).
|
||||
dict(slot=2, prompt_len=5, computed=0, span=[0, 0, 0, 0, 0], prefill=True),
|
||||
# Decode with a codec id: emitted this step.
|
||||
dict(slot=0, prompt_len=4, computed=6, span=[17], prefill=False),
|
||||
# Decode whose input is codec EOS: request ended, force EOS again.
|
||||
dict(slot=1, prompt_len=4, computed=8, span=[_EOS], prefill=False),
|
||||
]
|
||||
batch, padded = _batch(rows, pad_to=8)
|
||||
buffers = [{"audio_state": {"finished": False}}, {}, {}]
|
||||
hidden = torch.zeros((padded, 4))
|
||||
out = talker.make_omni_output_mrv2(
|
||||
hidden,
|
||||
input_batch=batch,
|
||||
req_states=_req_states({cast(int, row["slot"]): cast(int, row["prompt_len"]) for row in rows}),
|
||||
model_intermediate_buffer=buffers,
|
||||
)
|
||||
assert isinstance(out, OmniOutput)
|
||||
codes = out.multimodal_outputs["codes"]["audio"]
|
||||
valid = out.multimodal_outputs["meta"]["codec_frame_valid"]
|
||||
assert codes.shape == (padded, 1) and valid.shape == (padded,)
|
||||
assert codes[:, 0].tolist() == batch.input_ids.tolist()
|
||||
assert valid.tolist() == [False] * 5 + [True, False] + [False]
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 3).tolist() == [False, False, True]
|
||||
# Consumed once; a warmup sampler call sees nothing.
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 3) is None
|
||||
|
||||
|
||||
def test_mrv2_output_empty_condition_and_length_cap() -> None:
|
||||
talker = _talker()
|
||||
prompt_lens = {3: 6, 4: 4096 - 100}
|
||||
# Empty Thinker condition: preprocess marks the request finished at prefill.
|
||||
batch, _ = _batch([dict(slot=3, prompt_len=6, computed=0, span=[0] * 6, prefill=True)])
|
||||
talker.make_omni_output_mrv2(
|
||||
torch.zeros((6, 4)),
|
||||
input_batch=batch,
|
||||
req_states=_req_states(prompt_lens),
|
||||
model_intermediate_buffer=[{"audio_state": {"finished": True}}],
|
||||
)
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [True]
|
||||
# Its later decode rows stay forced and emit nothing, whatever the input id.
|
||||
batch, _ = _batch([dict(slot=3, prompt_len=6, computed=6, span=[42], prefill=False)])
|
||||
out = talker.make_omni_output_mrv2(
|
||||
torch.zeros((1, 4)), input_batch=batch, req_states=_req_states(prompt_lens), model_intermediate_buffer=[{}]
|
||||
)
|
||||
assert out.multimodal_outputs["meta"]["codec_frame_valid"].tolist() == [False]
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [True]
|
||||
|
||||
# Offline cap: min(2048, context - prompt) samples, the last one EOS.
|
||||
limit = min(_OFFLINE_CODEC_MAX_NEW_TOKENS, 100) - 1
|
||||
for step, forced in ((limit - 1, False), (limit, True)):
|
||||
batch, _ = _batch(
|
||||
[dict(slot=4, prompt_len=prompt_lens[4], computed=prompt_lens[4] + step - 1, span=[5], prefill=False)]
|
||||
)
|
||||
out = talker.make_omni_output_mrv2(
|
||||
torch.zeros((1, 4)), input_batch=batch, req_states=_req_states(prompt_lens), model_intermediate_buffer=[{}]
|
||||
)
|
||||
assert out.multimodal_outputs["meta"]["codec_frame_valid"].tolist() == [True]
|
||||
assert talker.take_mrv2_forced_eos(batch, None, 1).tolist() == [forced]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("forced", [None, [False, True]])
|
||||
def test_sampler_adapter_keeps_upstream_counts_and_only_forces_codec_eos(mocker, forced):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import MiniCPMO45TalkerSampler
|
||||
|
||||
output = SimpleNamespace(sampled_token_ids=torch.tensor([[2], [3]]), num_sampled=torch.tensor([1, 0]))
|
||||
base = mocker.Mock(return_value=output)
|
||||
base.req_states = object()
|
||||
mask = None if forced is None else torch.tensor(forced)
|
||||
talker = SimpleNamespace(_codec_eos_id=7, take_mrv2_forced_eos=mocker.Mock(return_value=mask))
|
||||
sampler = MiniCPMO45TalkerSampler(base, talker)
|
||||
logits, batch = torch.zeros(2, 8), object()
|
||||
assert sampler(logits, batch) is output
|
||||
assert output.sampled_token_ids.tolist() == [[2], [3 if forced is None else 7]]
|
||||
assert output.num_sampled.tolist() == [1, 0]
|
||||
base.assert_called_once_with(logits, batch)
|
||||
talker.take_mrv2_forced_eos.assert_called_once_with(batch, base.req_states, 2)
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MRv2 Thinker preserves multimodal encoding and the llm2tts row ledger."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5 import minicpmo_4_5_omni as omni
|
||||
from vllm_omni.worker_v2.omni_model_runner import OmniGPUModelRunner
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"profile,capacities,kv_gib",
|
||||
[
|
||||
("minicpmo_4_5_turn_mrv2.yaml", [16, 8, 8], 2),
|
||||
("minicpmo_4_5_turn_mrv2_h200.yaml", [16, 16, 8], 4),
|
||||
],
|
||||
)
|
||||
def test_mrv2_profile_retains_full_thinker_handoff(profile, capacities, kv_gib, monkeypatch):
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config, merge_pipeline_deploy
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# merge_pipeline_deploy resolves the platform again; test the CUDA profile
|
||||
# consistently even when this CPU test runs on a ROCm host.
|
||||
monkeypatch.setattr(current_omni_platform, "device_name", "cuda")
|
||||
deploy = Path(__file__).resolve().parents[4] / "vllm_omni/deploy" / profile
|
||||
config = _apply_platform_overrides(load_deploy_config(deploy), platform="cuda")
|
||||
stages = merge_pipeline_deploy(MINICPMO_4_5_PIPELINE, config)
|
||||
assert [s.yaml_engine_args["use_v2_model_runner"] for s in stages] == [True, True, True]
|
||||
assert [s.yaml_engine_args["async_chunk"] for s in stages] == [False, True, True]
|
||||
assert [s.yaml_engine_args["max_num_seqs"] for s in stages] == capacities
|
||||
assert stages[1].yaml_engine_args["kv_cache_memory_bytes"] == kv_gib * 1024**3
|
||||
|
||||
|
||||
def _model(mocker, *, v2=True, session="turn", async_chunk=False):
|
||||
thinker = torch.nn.Module()
|
||||
thinker.make_empty_intermediate_tensors = lambda: None
|
||||
mocker.patch.object(omni, "init_vllm_registered_model", return_value=thinker)
|
||||
mocker.patch("vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat.patch_minicpmo_remote_config")
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(),
|
||||
multimodal_config=None,
|
||||
model_stage="llm",
|
||||
use_v2_model_runner=v2,
|
||||
session_mode=session,
|
||||
async_chunk=async_chunk,
|
||||
)
|
||||
)
|
||||
return omni.MiniCPMO45OmniForConditionalGeneration(vllm_config=config)
|
||||
|
||||
|
||||
def test_mrv2_thinker_keeps_native_multimodal_embeddings_and_raw_ids(mocker):
|
||||
model = _model(mocker)
|
||||
batch = SimpleNamespace(input_ids=torch.tensor([11, 22, 33]), num_tokens_after_padding=3)
|
||||
embeddings = torch.randn(3, 4)
|
||||
runner = SimpleNamespace(
|
||||
model=model,
|
||||
supports_mm_inputs=True,
|
||||
is_first_pp_rank=True,
|
||||
model_state=SimpleNamespace(dummy_inputs_embeds=lambda _: embeddings),
|
||||
)
|
||||
ids, embeds, _ = OmniGPUModelRunner._prepare_mm_inputs(runner, None, batch, dummy_run=True)
|
||||
assert ids is batch.input_ids
|
||||
assert embeds is embeddings
|
||||
hidden = torch.randn(3, 4)
|
||||
model.thinker.forward = mocker.Mock(return_value=hidden)
|
||||
result = model(ids, torch.tensor([5, 6, 7]), inputs_embeds=embeds)
|
||||
assert result is hidden
|
||||
torch.testing.assert_close(model.thinker.forward.call_args.kwargs["inputs_embeds"], embeddings)
|
||||
|
||||
|
||||
def test_row_ledger_uses_live_batch_after_replay(mocker):
|
||||
model = _model(mocker)
|
||||
# Chunked prefill (two tokens), decode (one token), and graph padding.
|
||||
batch = SimpleNamespace(
|
||||
input_ids=torch.tensor([11, 12, 41, 0]),
|
||||
positions=torch.tensor([6, 7, 20, 0]),
|
||||
)
|
||||
hidden = torch.randn(4, 8)
|
||||
for token in [41, 53]:
|
||||
batch.input_ids[2] = token
|
||||
batch.positions[2] += 1
|
||||
out = model.make_omni_output_mrv2(
|
||||
hidden,
|
||||
input_batch=batch,
|
||||
req_states=None,
|
||||
model_intermediate_buffer=[{}, {}],
|
||||
)
|
||||
assert out.text_hidden_states is hidden
|
||||
assert out.multimodal_outputs["latent"] is hidden
|
||||
torch.testing.assert_close(out.multimodal_outputs["latent_input_ids"], batch.input_ids[:, None])
|
||||
torch.testing.assert_close(out.multimodal_outputs["latent_positions"], batch.positions[:, None])
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared HiFT must retain its native path unless explicitly opted in."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core import hifigan
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan import HiFTGenerator
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_shared_hift_keeps_native_istft_without_opt_in(monkeypatch):
|
||||
n_fft, hop = 16, 4
|
||||
window = torch.hann_window(n_fft)
|
||||
hift = SimpleNamespace(
|
||||
istft_params={"n_fft": n_fft, "hop_len": hop},
|
||||
_get_stft_window=lambda tensor: window,
|
||||
)
|
||||
|
||||
def unexpected_cache(*args):
|
||||
raise AssertionError("shared HiFT must not opt in implicitly")
|
||||
|
||||
monkeypatch.setattr(hifigan, "_istft_without_host_sync", unexpected_cache)
|
||||
magnitude, phase = torch.rand(2, 9, 20), torch.randn(2, 9, 20)
|
||||
assert HiFTGenerator._istft(hift, magnitude, phase).shape == (2, 76)
|
||||
assert not hasattr(hift, "_istft_envelopes")
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The MiniCPM-o 4.5 vocoder stage's per-step path must not block the host."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core import hifigan
|
||||
from vllm_omni.worker.gpu_generation_model_runner import _HostCopyBatch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _no_host_sync():
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
|
||||
|
||||
def test_istft_is_bitwise_and_sync_free_on_cuda():
|
||||
n_fft, hop = 16, 4
|
||||
window = torch.hann_window(n_fft, periodic=True, device="cuda")
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
magnitude = torch.rand(4, n_fft // 2 + 1, 300, device="cuda", generator=generator) * 3
|
||||
phase = torch.randn(4, n_fft // 2 + 1, 300, device="cuda", generator=generator)
|
||||
spec = torch.complex(magnitude * torch.cos(phase), magnitude * torch.sin(phase))
|
||||
envelopes: dict = {}
|
||||
expected = torch.istft(spec, n_fft, hop, n_fft, window=window)
|
||||
# The first call per frame count builds and checks the envelope.
|
||||
hifigan._istft_without_host_sync(spec, n_fft, hop, window, envelopes)
|
||||
with _no_host_sync():
|
||||
actual = hifigan._istft_without_host_sync(spec, n_fft, hop, window, envelopes)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
def test_host_copy_batch_copies_without_blocking_until_wait():
|
||||
to_host = _HostCopyBatch(pin_memory=True)
|
||||
outputs = [torch.randn(1000, device="cuda") * (row + 1) for row in range(4)]
|
||||
strided = torch.randn(8, 6, device="cuda").t()
|
||||
to_host.copy(torch.zeros(1, device="cuda"))
|
||||
to_host.wait()
|
||||
with _no_host_sync():
|
||||
copies = [to_host.copy(tensor) for tensor in outputs]
|
||||
copied_strided = to_host.copy(strided)
|
||||
to_host.wait()
|
||||
for copy, tensor in zip(copies, outputs, strict=True):
|
||||
assert copy.device.type == "cpu" and copy.is_pinned()
|
||||
assert torch.equal(copy, tensor.cpu())
|
||||
assert copied_strided.is_contiguous() and torch.equal(copied_strided, strided.cpu())
|
||||
@@ -112,6 +112,7 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
|
||||
|
||||
runner.model = SimpleNamespace(compute_logits=None, logitsprocs_need_output_token_ids=needs_history)
|
||||
runner.model.mrv2_sampling_context = sampling_context
|
||||
runner.sampler = None
|
||||
runner.sample = MagicMock(return_value=sampler_out)
|
||||
runner.sample.side_effect = lambda *_: sampler_out if sampling_active is needs_history else pytest.fail()
|
||||
logprobs_mock = MagicMock(side_effect=lambda *_: pytest.fail("inside ctx") if sampling_active else {})
|
||||
@@ -304,3 +305,72 @@ def test_request_reference_codes_preserve_local_axis(prefill_first, padded):
|
||||
assert outputs[index]["codes.ref"].data_ptr() != ref.data_ptr()
|
||||
for i in range(2):
|
||||
assert torch.equal(outputs[i]["codes.audio"], codes[offsets[i] : offsets[i + 1]])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_model_owned_audio_finalizer_runs_after_copy_without_hidden(monkeypatch, streaming):
|
||||
monkeypatch.setattr(torch.cuda, "set_stream", lambda _stream: None)
|
||||
batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
query_start_loc_np=np.array([0, 1, 4]),
|
||||
num_scheduled_tokens=np.array([1, 3]),
|
||||
num_tokens_after_padding=4,
|
||||
)
|
||||
snapshot = torch.tensor([[10, 11], [20, 21]])
|
||||
seen = []
|
||||
|
||||
def finalize(payload, counts):
|
||||
seen.append(counts)
|
||||
return {"codes": {"audio": [payload["snapshot"][0:1].clone(), torch.empty(0, dtype=torch.long)]}}
|
||||
|
||||
result = _async_output(
|
||||
req_ids=["audio", "partial"],
|
||||
sampler_output=SamplerOutput(torch.tensor([[99], [100]]), None, None, None),
|
||||
num_sampled_tokens=torch.tensor([1, 0]),
|
||||
multimodal_outputs={"snapshot": snapshot},
|
||||
input_batch=batch,
|
||||
async_chunk=streaming,
|
||||
finalize_multimodal=finalize,
|
||||
).get_output()
|
||||
snapshot.zero_()
|
||||
assert seen == [[1, 0]]
|
||||
assert result.sampled_token_ids == [[99], []]
|
||||
assert result.inter_stage_outputs[0]["codes.audio"].tolist() == [[10, 11]]
|
||||
assert result.inter_stage_outputs[1]["codes.audio"].numel() == 0
|
||||
assert (result.pooler_output is None) == streaming
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_request_owned_snapshot_skips_generic_partition(monkeypatch, streaming):
|
||||
from vllm_omni.worker_v2.output_snapshot import RequestOutputSnapshot
|
||||
|
||||
monkeypatch.setattr(torch.cuda, "set_stream", lambda _stream: None)
|
||||
monkeypatch.setattr(
|
||||
OmniARModelRunner,
|
||||
"_build_async_chunk_outputs_from_mm",
|
||||
lambda *args: pytest.fail("already partitioned payload was repartitioned"),
|
||||
)
|
||||
batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
query_start_loc_np=np.array([0, 1, 4]),
|
||||
num_scheduled_tokens=np.array([1, 3]),
|
||||
num_tokens_after_padding=4,
|
||||
)
|
||||
snapshot = torch.tensor([[10, 11], [20, 21]])
|
||||
|
||||
def finalize(payload, counts):
|
||||
return RequestOutputSnapshot([{"codes.audio": payload["snapshot"][0:1].clone()}, None])
|
||||
|
||||
result = _async_output(
|
||||
req_ids=["audio", "partial"],
|
||||
sampler_output=SamplerOutput(torch.tensor([[99], [100]]), None, None, None),
|
||||
num_sampled_tokens=torch.tensor([1, 0]),
|
||||
multimodal_outputs={"snapshot": snapshot},
|
||||
input_batch=batch,
|
||||
async_chunk=streaming,
|
||||
finalize_multimodal=finalize,
|
||||
).get_output()
|
||||
snapshot.zero_()
|
||||
assert result.inter_stage_outputs[0]["codes.audio"].tolist() == [[10, 11]]
|
||||
assert result.inter_stage_outputs[1] is None
|
||||
assert (result.pooler_output is None) == streaming
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
from vllm_omni.worker_v2.omni_sampler import OmniSampler, OmniSamplingOutput, sample_with_output
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _output():
|
||||
counts, rejected = torch.tensor([1, 0]), torch.tensor([0, 0])
|
||||
return SamplerOutput(torch.tensor([[7], [0]]), None, None, counts, rejected), counts, rejected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapped", [False, True])
|
||||
def test_standard_sampling_keeps_counts_grammar_and_lifecycle(wrapped):
|
||||
base = Mock()
|
||||
base.req_states = object()
|
||||
sampler = OmniSampler(base) if wrapped else base
|
||||
expected = _output()
|
||||
standard = Mock(return_value=expected)
|
||||
hidden, batch, grammar = torch.zeros(2, 4), object(), object()
|
||||
result = sample_with_output(sampler, standard, hidden, batch, base.req_states, grammar)
|
||||
assert result.sampler_output is expected[0]
|
||||
assert result.num_sampled is expected[1] and result.num_rejected is expected[2]
|
||||
assert result.multimodal_outputs is None
|
||||
standard.assert_called_once_with(hidden, batch, grammar)
|
||||
sampler.add_request(3, "request")
|
||||
sampler.apply_staged_writes()
|
||||
base.add_request.assert_called_once_with(3, "request")
|
||||
base.apply_staged_writes.assert_called_once_with()
|
||||
assert sampler.req_states is base.req_states
|
||||
sampler(hidden, batch)
|
||||
base.assert_called_once_with(hidden, batch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [{}, {"codes.audio": torch.tensor([[1, 2]])}])
|
||||
def test_audio_sampler_uses_registered_adapter_without_standard_resampling(payload):
|
||||
expected = OmniSamplingOutput(*_output(), multimodal_outputs=payload)
|
||||
|
||||
class AudioSampler(OmniSampler):
|
||||
def sample_step(self, hidden, batch, states, grammar, standard):
|
||||
assert states is self.req_states
|
||||
if grammar is not None:
|
||||
raise ValueError("unsupported grammar")
|
||||
return expected
|
||||
|
||||
sampler = AudioSampler(SimpleNamespace(req_states=object()))
|
||||
standard = Mock(side_effect=AssertionError("must not resample"))
|
||||
result = sample_with_output(sampler, standard, torch.zeros(2, 4), object(), sampler.req_states, None)
|
||||
assert result is expected and result.multimodal_outputs is payload
|
||||
with pytest.raises(ValueError, match="unsupported grammar"):
|
||||
sample_with_output(sampler, standard, torch.zeros(2, 4), object(), sampler.req_states, object())
|
||||
standard.assert_not_called()
|
||||
|
||||
|
||||
def test_adapter_preserves_stock_sampler_staged_write_fast_path():
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
|
||||
assert OmniSampler(object.__new__(Sampler)).omni_static_staged_writes
|
||||
assert not OmniSampler(SimpleNamespace()).omni_static_staged_writes
|
||||
assert OmniSampler(SimpleNamespace(omni_static_staged_writes=True)).omni_static_staged_writes
|
||||
@@ -0,0 +1,10 @@
|
||||
# Opt-in single-H200 native full-duplex codec profile (V1).
|
||||
# Inherits the native session/stage capacities, sampling and chunk sizes.
|
||||
# Uses the mainline Whole-Euler Flow and shared prompt-state implementation.
|
||||
# TF32 changes FP32 GEMM rounding; validate audio quality for your workload.
|
||||
base_config: minicpmo_4_5.yaml
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
extra:
|
||||
token2wav_allow_tf32: true
|
||||
@@ -0,0 +1,57 @@
|
||||
# Opt-in CUDA Model Runner V2 profile for MiniCPM-o 4.5 turn serving: the
|
||||
# Thinker (stage 0), Talker (stage 1) and Code2Wav (stage 2) run on MRv2.
|
||||
# The Thinker uses native multimodal encoding and tensor-only FULL graphs;
|
||||
# downstream stages use the native data plane. These stages support
|
||||
# turn-based sessions only (duplex needs the V1 chunk adapter).
|
||||
#
|
||||
# On MRv2 the Talker keeps its sampled codec id, 16-frame penalty history and
|
||||
# EOS state on the device (no per-step D2H wait), and Code2Wav schedules
|
||||
# individual chunks instead of holding a slot per utterance, so more than
|
||||
# max_num_seqs streams can be live under load.
|
||||
base_config: minicpmo_4_5_turn.yaml
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
extra:
|
||||
hift_graph_capture_batch_sizes: [1, 2, 4, 8]
|
||||
# TF32 tensor cores for the vocoder's fp32 flow GEMMs: ~25% less GPU
|
||||
# time per chunk (batch 1-8). A numerics change: the HiFT input mel moves
|
||||
# by ~3e-4 relative L2; dev-short WER/UTMOS unchanged (64 prompts).
|
||||
token2wav_allow_tf32: true
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_runner: v2
|
||||
# llm2tts consumes the completed Thinker payload via the orchestrator.
|
||||
# Only Talker -> Code2Wav is a streaming connector edge. Keep the latent
|
||||
# rows on the full-payload output channel instead of partitioning them
|
||||
# into an async connector that this stage does not have.
|
||||
async_chunk: false
|
||||
- stage_id: 1
|
||||
model_runner: v2
|
||||
max_num_seqs: 8
|
||||
- stage_id: 2
|
||||
model_runner: v2
|
||||
max_num_seqs: 8
|
||||
async_scheduling: false
|
||||
env:
|
||||
# Decode waiting first chunks in their own step, but only while every
|
||||
# stream with a later chunk ready still holds 0.5 s of unplayed audio,
|
||||
# so first packets do not come at the cost of playback underruns.
|
||||
VLLM_OMNI_CODEC_FIRST_CHUNK_EXPRESS: "1"
|
||||
VLLM_OMNI_CODEC_FIRST_CHUNK_EXPRESS_SLACK_S: "0.5"
|
||||
|
||||
platforms:
|
||||
cuda:
|
||||
stages:
|
||||
- stage_id: 1
|
||||
# Reserve enough KV for eight concurrent 4096-token Talker requests.
|
||||
kv_cache_memory_bytes: 2147483648
|
||||
npu:
|
||||
model_runner: v1
|
||||
xpu:
|
||||
model_runner: v1
|
||||
rocm:
|
||||
model_runner: v1
|
||||
musa:
|
||||
model_runner: v1
|
||||
@@ -0,0 +1,22 @@
|
||||
# Opt-in single-H200 throughput profile for turn-mode MiniCPM-o 4.5.
|
||||
# Use NVIDIA MPS when sharing the GPU across stages. Warm representative
|
||||
# streaming shapes before admitting traffic: graph capture can delay
|
||||
# first audio by seconds. Talker capacity 16 reduces C32 admission-tail
|
||||
# latency at similar throughput; C16 P99 varied across independent starts.
|
||||
# This reserves 4 GiB of Talker KV (2 GiB more than the generic MRv2 profile).
|
||||
# Keep room for lazy graph caches on the 141 GiB H200.
|
||||
#
|
||||
# FP32 weights/caches are retained, but the opt-in TF32 policy changes
|
||||
# rounding. Quality evidence and limits:
|
||||
# docs/design/minicpm_o45_mrv2_performance.md
|
||||
base_config: minicpmo_4_5_turn_mrv2.yaml
|
||||
|
||||
stages:
|
||||
- stage_id: 1
|
||||
max_num_seqs: 16
|
||||
|
||||
platforms:
|
||||
cuda:
|
||||
stages:
|
||||
- stage_id: 1
|
||||
kv_cache_memory_bytes: 4294967296
|
||||
@@ -143,6 +143,53 @@ class ResBlock(torch.nn.Module):
|
||||
return folded
|
||||
|
||||
|
||||
def _istft_envelope(window: torch.Tensor, n_fft: int, hop_length: int, n_frames: int) -> torch.Tensor:
|
||||
"""Trimmed overlap-add window envelope that ``torch.istft`` divides by.
|
||||
|
||||
Built with the same ops as ``at::native::istft`` (squared window folded by
|
||||
``unfold_backward``, centre padding trimmed). ``torch.istft`` also checks
|
||||
here that no sample has a vanishing envelope, which reads the result back
|
||||
to the host; the envelope only depends on the window and the frame count,
|
||||
so the check runs once per shape instead of once per call.
|
||||
"""
|
||||
expected_len = n_fft + hop_length * (n_frames - 1)
|
||||
squared = window.pow(2).expand(1, n_frames, n_fft)
|
||||
envelope = torch.ops.aten.unfold_backward(squared, [1, expected_len], 1, n_fft, hop_length)
|
||||
envelope = envelope[:, n_fft // 2 : expected_len - n_fft // 2]
|
||||
if bool(envelope.abs().min().lt(1e-11)):
|
||||
raise RuntimeError("window overlap add min: True")
|
||||
return envelope
|
||||
|
||||
|
||||
def _istft_without_host_sync(
|
||||
spec: torch.Tensor,
|
||||
n_fft: int,
|
||||
hop_length: int,
|
||||
window: torch.Tensor,
|
||||
envelopes: dict[tuple, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""``torch.istft(spec, n_fft, hop_length, n_fft, window=window)`` without a host sync.
|
||||
|
||||
Mirrors ``at::native::istft`` for this call (``center=True``, one-sided,
|
||||
unnormalized, ``win_length == n_fft``, no ``length``): the same inverse
|
||||
real FFT, window product, ``unfold_backward`` overlap-add, trim and
|
||||
division, so the waveform is bitwise identical. The envelope, and with it
|
||||
the only device-to-host read, comes from ``envelopes`` after the first
|
||||
call for a frame count.
|
||||
"""
|
||||
batch, _, n_frames = spec.shape
|
||||
key = (n_frames, n_fft, hop_length, window.device, window.dtype, window.data_ptr())
|
||||
envelope = envelopes.get(key)
|
||||
if envelope is None:
|
||||
envelope = _istft_envelope(window, n_fft, hop_length, n_frames)
|
||||
envelopes[key] = envelope
|
||||
frames = torch.fft.irfft(spec.transpose(1, 2), n=n_fft, dim=-1)
|
||||
frames = frames * window.view(1, 1, n_fft)
|
||||
expected_len = n_fft + hop_length * (n_frames - 1)
|
||||
signal = torch.ops.aten.unfold_backward(frames, [batch, expected_len], 1, n_fft, hop_length)
|
||||
return signal[:, n_fft // 2 : expected_len - n_fft // 2] / envelope
|
||||
|
||||
|
||||
def _carry_phase_at_boundary(
|
||||
cum_total: torch.Tensor,
|
||||
phase_acc: torch.Tensor | None,
|
||||
@@ -620,6 +667,10 @@ class HiFTGenerator(nn.Module):
|
||||
)
|
||||
self.f0_predictor = f0_predictor
|
||||
|
||||
def enable_cached_istft(self) -> None:
|
||||
"""Opt into CUDA ISTFT with a cached overlap envelope."""
|
||||
self._use_cached_istft = True
|
||||
|
||||
def remove_weight_norm(self) -> int:
|
||||
"""Fold the generator's frozen weight norms into plain weights.
|
||||
|
||||
@@ -675,12 +726,28 @@ class HiFTGenerator(nn.Module):
|
||||
magnitude = torch.clip(magnitude, max=1e2)
|
||||
real = magnitude * torch.cos(phase)
|
||||
imag = magnitude * torch.sin(phase)
|
||||
return torch.istft(
|
||||
torch.complex(real, imag),
|
||||
spec = torch.complex(real, imag)
|
||||
window = self._get_stft_window(magnitude)
|
||||
if (
|
||||
not getattr(self, "_use_cached_istft", False)
|
||||
or spec.dim() != 3
|
||||
or window.dim() != 1
|
||||
or window.shape[0] != self.istft_params["n_fft"]
|
||||
):
|
||||
return torch.istft(
|
||||
spec,
|
||||
self.istft_params["n_fft"],
|
||||
self.istft_params["hop_len"],
|
||||
self.istft_params["n_fft"],
|
||||
window=window,
|
||||
)
|
||||
envelopes = self.__dict__.setdefault("_istft_envelopes", {})
|
||||
return _istft_without_host_sync(
|
||||
spec,
|
||||
self.istft_params["n_fft"],
|
||||
self.istft_params["hop_len"],
|
||||
self.istft_params["n_fft"],
|
||||
window=self._get_stft_window(magnitude),
|
||||
window,
|
||||
envelopes,
|
||||
)
|
||||
|
||||
def _istft_on_cpu(self, magnitude, phase):
|
||||
|
||||
@@ -268,6 +268,9 @@ class BatchedToken2Wav(nn.Module):
|
||||
self._trt_stepper = trt_stepper
|
||||
self.flow = token2wav.flow
|
||||
self.hift = token2wav.hift
|
||||
enable_cached_istft = getattr(self.hift, "enable_cached_istft", None)
|
||||
if callable(enable_cached_istft):
|
||||
enable_cached_istft()
|
||||
encoder = getattr(self.flow, "encoder", None)
|
||||
if encoder is not None:
|
||||
_undecorate_dynamo(encoder, "forward_chunk")
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.utils.device_copy import to_device_nonblocking
|
||||
|
||||
from .batched_token2wav import (
|
||||
BatchedToken2Wav,
|
||||
@@ -146,10 +147,12 @@ def _normalize_reference(
|
||||
|
||||
|
||||
def _codec_tensor(value: Any, fallback: torch.Tensor) -> torch.Tensor:
|
||||
# Codec ids arrive from the connector as host data. A pageable host copy
|
||||
# would block the host once per request per step; stage them pinned.
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.reshape(-1).to(device=fallback.device, dtype=torch.long)
|
||||
return to_device_nonblocking(value.reshape(-1).to(dtype=torch.long), fallback.device)
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return torch.as_tensor(value, device=fallback.device, dtype=torch.long).reshape(-1)
|
||||
return to_device_nonblocking(torch.as_tensor(value, dtype=torch.long).reshape(-1), fallback.device)
|
||||
return fallback.reshape(-1).to(dtype=torch.long)
|
||||
|
||||
|
||||
@@ -730,6 +733,33 @@ class MiniCPMO45Code2Wav(nn.Module):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
# This stage owns the vocoder process. Restore its previous matmul
|
||||
# policy after eager execution/capture; cuDNN's policy is independent.
|
||||
previous_tf32 = torch.backends.cuda.matmul.allow_tf32
|
||||
try:
|
||||
if self._extra_config().get("token2wav_allow_tf32", False):
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
return self._forward_impl(
|
||||
input_ids,
|
||||
positions,
|
||||
intermediate_tensors,
|
||||
inputs_embeds,
|
||||
runtime_additional_information,
|
||||
**kwargs,
|
||||
)
|
||||
finally:
|
||||
torch.backends.cuda.matmul.allow_tf32 = previous_tf32
|
||||
|
||||
@torch.inference_mode()
|
||||
def _forward_impl(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
del positions, intermediate_tensors, inputs_embeds
|
||||
ids = input_ids if isinstance(input_ids, torch.Tensor) else torch.empty(0, dtype=torch.long)
|
||||
|
||||
@@ -94,6 +94,19 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
patch_minicpmo_remote_config(config)
|
||||
|
||||
self.model_stage = vllm_config.model_config.model_stage
|
||||
self._use_v2_model_runner = bool(getattr(vllm_config.model_config, "use_v2_model_runner", False))
|
||||
if (
|
||||
self.model_stage == "llm"
|
||||
and self._use_v2_model_runner
|
||||
and getattr(vllm_config.model_config, "session_mode", "turn") != "turn"
|
||||
):
|
||||
raise NotImplementedError("MiniCPM-o duplex Thinker requires model_runner: v1")
|
||||
if (
|
||||
self.model_stage == "llm"
|
||||
and self._use_v2_model_runner
|
||||
and getattr(vllm_config.model_config, "async_chunk", False)
|
||||
):
|
||||
raise ValueError("MiniCPM-o MRv2 Thinker requires async_chunk: false for its full llm2tts payload")
|
||||
# The Thinker's row ledger needs real token identities even when
|
||||
# embeddings are supplied, including during CUDA graph capture/replay.
|
||||
self.requires_raw_input_tokens = self.model_stage == "llm"
|
||||
@@ -155,6 +168,22 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
if hasattr(self.talker, "init_multi_modal"):
|
||||
self.talker.init_multi_modal(config)
|
||||
self.model = self.talker
|
||||
# The runner looks this hook up on the wrapper. Without it every
|
||||
# decode row runs scalar preprocess, which reads its codec id with
|
||||
# a blocking ``.item()``.
|
||||
batch_decode = getattr(self.talker, "preprocess_decode_batch", None)
|
||||
if callable(batch_decode):
|
||||
self.preprocess_decode_batch = batch_decode
|
||||
# Model Runner V2 hooks: device-side codec output, EOS control and
|
||||
# codec penalty (see MiniCPMO45OmniTTSForConditionalGeneration).
|
||||
for hook_name in ("preprocess_decode_batch_mrv2", "make_omni_output_mrv2", "mrv2_custom_sampler"):
|
||||
hook = getattr(self.talker, hook_name, None)
|
||||
if callable(hook):
|
||||
setattr(self, hook_name, hook)
|
||||
self.mrv2_decode_preprocess_is_identity = bool(
|
||||
getattr(self.talker, "mrv2_decode_preprocess_is_identity", False)
|
||||
)
|
||||
self.logits_vocab_size = int(self.talker.logits_vocab_size)
|
||||
else:
|
||||
raise ValueError(f"Invalid model stage: {self.model_stage}. Must be one of: 'llm', 'tts'")
|
||||
|
||||
@@ -173,7 +202,13 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
# preprocess for duplex audio, while the Talker converts the
|
||||
# tts_token_ids/tts_hidden_states handoff into its conditioning
|
||||
# embeddings and initializes request-local codec generation state.
|
||||
self.has_preprocess = self.model_stage in {"llm", "tts"}
|
||||
# Turn-mode Thinker inputs use the native multimodal encoder/cache on
|
||||
# MRv2. Marking it as a custom-preprocess model disables that encoder
|
||||
# path in the runner. Duplex still uses the V1 preprocess hook.
|
||||
self.has_preprocess = self.model_stage == "tts" or not self._use_v2_model_runner
|
||||
# Neither AR stage has a postprocess, so step outputs can use the
|
||||
# runner's async snapshot instead of a blocking per-step D2H.
|
||||
self.use_async_omni_output = self.model_stage in {"llm", "tts"}
|
||||
|
||||
if self.model_stage == "llm" and getattr(vllm_config.model_config, "session_mode", "turn") == "duplex":
|
||||
# Build the Stage-0 duplex runtime (remote-code processor and
|
||||
@@ -617,6 +652,12 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
if text_hidden_states.ndim == 3 and text_hidden_states.shape[0] == 1:
|
||||
text_hidden_states = text_hidden_states.squeeze(0)
|
||||
|
||||
if getattr(self, "_use_v2_model_runner", False):
|
||||
# FULL graphs return tensors. The row ledger is reconstructed
|
||||
# from this step's live input batch after replay, never from
|
||||
# Python values or token buffers captured during warmup.
|
||||
return text_hidden_states
|
||||
|
||||
# Return hidden states with latent in multimodal_outputs for stage_input_processors
|
||||
multimodal_outputs = {"latent": text_hidden_states}
|
||||
# Keep per-forward row identities alongside the latent payload.
|
||||
@@ -691,6 +732,27 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
|
||||
raise ValueError(f"Unsupported model stage: {self.model_stage}")
|
||||
|
||||
def make_omni_output_mrv2(self, model_outputs, *, input_batch, req_states, model_intermediate_buffer):
|
||||
"""Attach the Thinker row identities used by the llm2tts bridge."""
|
||||
if self.model_stage != "llm":
|
||||
return self.talker.make_omni_output_mrv2(
|
||||
model_outputs,
|
||||
input_batch=input_batch,
|
||||
req_states=req_states,
|
||||
model_intermediate_buffer=model_intermediate_buffer,
|
||||
)
|
||||
if any(isinstance(info, dict) and info.get("duplex") for info in model_intermediate_buffer):
|
||||
raise NotImplementedError("MiniCPM-o duplex Thinker requires model_runner: v1")
|
||||
num_tokens = model_outputs.shape[0]
|
||||
return OmniOutput(
|
||||
text_hidden_states=model_outputs,
|
||||
multimodal_outputs={
|
||||
"latent": model_outputs,
|
||||
"latent_input_ids": input_batch.input_ids[:num_tokens].reshape(-1, 1),
|
||||
"latent_positions": input_batch.positions[:num_tokens].reshape(-1, 1),
|
||||
},
|
||||
)
|
||||
|
||||
def make_omni_output(self, model_outputs, **kwargs):
|
||||
if self.model_stage != "tts":
|
||||
return model_outputs
|
||||
|
||||
@@ -14,8 +14,10 @@ Pipeline:
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
@@ -34,6 +36,8 @@ from vllm_omni.model_executor.models.minicpmo_4_5 import (
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.utils.device_copy import index_to_device, to_device_nonblocking
|
||||
from vllm_omni.worker_v2.omni_sampler import OmniSampler
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -105,6 +109,38 @@ def _restore_weight_norm_weight(weight_g: torch.Tensor, weight_v: torch.Tensor)
|
||||
return torch._weight_norm(weight_v, weight_g, dim=0)
|
||||
|
||||
|
||||
class _HostCodecIds:
|
||||
"""A batch of device codec ids, copied to pinned host memory without a host sync.
|
||||
|
||||
The copy is queued before the Talker forward, so ``values()`` waits only
|
||||
for the previous step's sampler (the same point vLLM's penalty path already
|
||||
waits for), not for this step's forward.
|
||||
"""
|
||||
|
||||
__slots__ = ("_event", "_host", "_values")
|
||||
|
||||
def __init__(self, ids: torch.Tensor) -> None:
|
||||
self._event: torch.cuda.Event | None = None
|
||||
self._host: torch.Tensor | None = None
|
||||
self._values: list[int] | None = None
|
||||
ids = ids.detach().reshape(-1)
|
||||
if ids.device.type == "cuda":
|
||||
self._host = torch.empty(ids.shape, dtype=ids.dtype, device="cpu", pin_memory=True)
|
||||
self._host.copy_(ids, non_blocking=True)
|
||||
self._event = torch.cuda.Event()
|
||||
self._event.record()
|
||||
else:
|
||||
self._values = ids.tolist()
|
||||
|
||||
def values(self) -> list[int]:
|
||||
if self._values is None:
|
||||
assert self._event is not None and self._host is not None
|
||||
self._event.synchronize()
|
||||
self._values = self._host.tolist()
|
||||
self._event = self._host = None
|
||||
return self._values
|
||||
|
||||
|
||||
def _apply_batched_repetition_penalty(
|
||||
logits: torch.Tensor,
|
||||
histories: Sequence[torch.Tensor],
|
||||
@@ -149,7 +185,8 @@ def _apply_batched_repetition_penalty(
|
||||
# The vocabulary fixes the output size. CUDA bincount still reads the
|
||||
# maximum id back to the host even when minlength is provided.
|
||||
encoded = encoded_rows[0] if len(encoded_rows) == 1 else torch.cat(encoded_rows)
|
||||
encoded = encoded.to(device=logits.device)
|
||||
# Pinned, non-blocking: a pageable H2D would wait for queued GPU work.
|
||||
encoded = to_device_nonblocking(encoded, logits.device)
|
||||
frequencies = torch.zeros((end - start) * vocab_size, dtype=torch.long, device=logits.device)
|
||||
frequencies.scatter_add_(0, encoded, torch.ones_like(encoded))
|
||||
frequencies = frequencies.reshape(end - start, vocab_size)
|
||||
@@ -159,6 +196,150 @@ def _apply_batched_repetition_penalty(
|
||||
return penalized
|
||||
|
||||
|
||||
def _apply_codec_window_penalty_gpu(
|
||||
logits: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
all_token_ids: torch.Tensor,
|
||||
total_len: torch.Tensor,
|
||||
prompt_len: torch.Tensor,
|
||||
penalty: torch.Tensor,
|
||||
*,
|
||||
window_size: int,
|
||||
) -> None:
|
||||
"""In place: ``_apply_batched_repetition_penalty`` on the runner's device token history.
|
||||
|
||||
Model Runner V2 keeps every request's sampled ids (``all_token_ids``) and
|
||||
lengths on the device, so the 16-frame window is gathered there instead of
|
||||
being rebuilt on the host each step. Row ``r`` is scored over its last
|
||||
``window_size`` output ids, ``[max(total_len - window, prompt_len),
|
||||
total_len)``: the codes sampled so far, exactly the history the V1 path
|
||||
builds in ``make_omni_output``. Same arithmetic as the V1 helper, so the
|
||||
penalized logits are identical.
|
||||
"""
|
||||
num_rows, vocab_size = logits.shape
|
||||
if num_rows == 0:
|
||||
return
|
||||
slots = slots.long()
|
||||
end = total_len.index_select(0, slots).long()
|
||||
start = torch.maximum(end - window_size, prompt_len.index_select(0, slots).long())
|
||||
offsets = torch.arange(window_size, device=logits.device, dtype=torch.long)
|
||||
positions = end.unsqueeze(1) - window_size + offsets.unsqueeze(0)
|
||||
valid = positions >= start.unsqueeze(1)
|
||||
rows = all_token_ids.index_select(0, slots)
|
||||
history = rows.gather(1, positions.clamp_min(0)).long()
|
||||
# Invalid positions count into a spare column that is dropped below.
|
||||
history = torch.where(valid & (history >= 0) & (history < vocab_size), history, vocab_size)
|
||||
frequencies = torch.zeros((num_rows, vocab_size + 1), dtype=torch.long, device=logits.device)
|
||||
frequencies.scatter_add_(1, history, torch.ones_like(history))
|
||||
alpha = torch.pow(
|
||||
penalty.index_select(0, slots).to(dtype=logits.dtype).unsqueeze(1),
|
||||
frequencies[:, :vocab_size].to(dtype=logits.dtype),
|
||||
)
|
||||
logits.copy_(torch.where(logits < 0, logits * alpha, logits / alpha))
|
||||
|
||||
|
||||
class _CodecWindowPenaltiesState:
|
||||
"""MRv2 sampler penalties for the MiniCPM-o Talker.
|
||||
|
||||
Replaces the sampler's presence-based repetition penalty (whole prompt and
|
||||
output) with MiniCPMTTS's frequency penalty over the last 16 codes, scored
|
||||
on the device. Frequency and presence penalties, if a request sets them,
|
||||
still go through the upstream state, which keeps the output bin counts.
|
||||
"""
|
||||
|
||||
def __init__(self, base: Any, *, window_size: int) -> None:
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
|
||||
|
||||
self.base = base
|
||||
self.req_states = base.req_states
|
||||
self.window_size = int(window_size)
|
||||
max_num_reqs = int(self.req_states.max_num_reqs)
|
||||
self.repetition_penalty = UvaBackedTensor(max_num_reqs, dtype=torch.float32)
|
||||
self.repetition_penalty.np.fill(1.0)
|
||||
self.repetition_penalty.copy_to_uva()
|
||||
self.use_window = np.zeros(max_num_reqs, dtype=bool)
|
||||
self.use_penalty = np.zeros(max_num_reqs, dtype=bool)
|
||||
|
||||
@property
|
||||
def output_bin_counts(self) -> torch.Tensor:
|
||||
return self.base.output_bin_counts
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: Any) -> None:
|
||||
repetition = float(getattr(sampling_params, "repetition_penalty", 1.0))
|
||||
self.repetition_penalty.np[req_idx] = repetition
|
||||
self.use_window[req_idx] = repetition != 1.0
|
||||
self.base.add_request(
|
||||
req_idx,
|
||||
SimpleNamespace(
|
||||
repetition_penalty=1.0,
|
||||
frequency_penalty=float(getattr(sampling_params, "frequency_penalty", 0.0)),
|
||||
presence_penalty=float(getattr(sampling_params, "presence_penalty", 0.0)),
|
||||
),
|
||||
)
|
||||
self.use_penalty[req_idx] = bool(self.use_window[req_idx] or self.base.use_penalty[req_idx])
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
self.repetition_penalty.copy_to_uva()
|
||||
self.base.apply_staged_writes()
|
||||
|
||||
def apply_penalties(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
input_ids: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor,
|
||||
) -> None:
|
||||
if np.any(self.use_window[idx_mapping_np]):
|
||||
_apply_codec_window_penalty_gpu(
|
||||
logits,
|
||||
expanded_idx_mapping,
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
self.req_states.prompt_len.gpu,
|
||||
self.repetition_penalty.gpu,
|
||||
window_size=self.window_size,
|
||||
)
|
||||
self.base.apply_penalties(logits, expanded_idx_mapping, idx_mapping_np, input_ids, expanded_local_pos)
|
||||
|
||||
|
||||
def _install_mrv2_talker_sampler(sampler: Any, talker: "MiniCPMO45OmniTTSForConditionalGeneration") -> Any:
|
||||
"""Wrap the runner's MRv2 sampler with codec penalties and EOS control.
|
||||
|
||||
The upstream sampler already applies ``min_tokens`` (codec EOS is a stage
|
||||
stop token), temperature, top-k/top-p and seeded sampling on the device.
|
||||
The Talker adds its 16-frame codec penalty and overrides rows the model
|
||||
terminates (``_mrv2_forced_eos``) with codec EOS after sampling, exactly
|
||||
where V1's ``_force_eos_on_sampled_ids`` does.
|
||||
"""
|
||||
|
||||
sampler.penalties_state = _CodecWindowPenaltiesState(sampler.penalties_state, window_size=_CODEC_PENALTY_WINDOW)
|
||||
talker._mrv2_empty_speech = torch.zeros(
|
||||
int(sampler.req_states.max_num_reqs), dtype=torch.bool, device=sampler.req_states.device
|
||||
)
|
||||
logger.info(
|
||||
"MiniCPM-o Talker: MRv2 sampler with device-side %d-frame codec penalty and EOS control",
|
||||
_CODEC_PENALTY_WINDOW,
|
||||
)
|
||||
return MiniCPMO45TalkerSampler(sampler, talker)
|
||||
|
||||
|
||||
class MiniCPMO45TalkerSampler(OmniSampler):
|
||||
omni_static_staged_writes = True
|
||||
|
||||
def __init__(self, base_sampler, talker):
|
||||
super().__init__(base_sampler)
|
||||
self.talker = talker
|
||||
|
||||
def __call__(self, logits: torch.Tensor, input_batch: Any) -> Any:
|
||||
forced = self.talker.take_mrv2_forced_eos(input_batch, self.req_states, logits.shape[0])
|
||||
output = self.base_sampler(logits, input_batch)
|
||||
if forced is not None:
|
||||
sampled = output.sampled_token_ids
|
||||
sampled.masked_fill_(forced.view(-1, *([1] * (sampled.ndim - 1))), int(self.talker._codec_eos_id))
|
||||
return output
|
||||
|
||||
|
||||
class _MiniCPMTTSProjector(nn.Module):
|
||||
"""Checkpoint-compatible hidden-state projector used by MiniCPMTTS."""
|
||||
|
||||
@@ -188,6 +369,9 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
self._mask_eos_rows: list[bool] | None = None
|
||||
self._pending_force_eos_rows: list[bool] | None = None
|
||||
self._penalty_histories: list[torch.Tensor] | None = None
|
||||
# Host copies of this step's decode input ids, keyed by request id:
|
||||
# make_omni_output reads them for EOS detection and penalty history.
|
||||
self._decode_codec_ids: dict[str, tuple[_HostCodecIds, int]] = {}
|
||||
self._request_audio_states: dict[str, dict[str, Any]] = {}
|
||||
# Mirrors upstream TTSStreamingGenerator._chunk_info: one committed
|
||||
# condition plus, during a rollover, one immutable recompute recipe.
|
||||
@@ -212,9 +396,17 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
self.has_preprocess = True
|
||||
self.has_postprocess = False
|
||||
# Same-step codes travel through make_omni_output from the previous
|
||||
# sampled id (decode preprocess embeds that id via emb_code).
|
||||
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {("codes", "audio")}
|
||||
# sampled id (decode preprocess embeds that id via emb_code). They are
|
||||
# CPU transport deltas, so no buffer key is GPU-resident.
|
||||
self._init_native_talker(prefix)
|
||||
# Model Runner V2 keeps the sampled id, codec history and EOS state on
|
||||
# the GPU (see make_omni_output_mrv2 and mrv2_custom_sampler).
|
||||
self._use_v2_model_runner = bool(getattr(vllm_config.model_config, "use_v2_model_runner", False))
|
||||
# Per request slot: the Thinker handed over an empty condition.
|
||||
self._mrv2_empty_speech: torch.Tensor | None = None
|
||||
# Rows the sampler must force to codec EOS, computed with this step's output.
|
||||
self._mrv2_forced_eos: torch.Tensor | None = None
|
||||
self._mrv2_decode_rows_logged = False
|
||||
|
||||
def _init_native_talker(self, prefix: str) -> None:
|
||||
if self._tts_config is None:
|
||||
@@ -255,11 +447,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
|
||||
def _boundary_embeddings(self) -> torch.Tensor:
|
||||
"""Embed the ``<text_eos><audio_bos>`` tail every condition ends with."""
|
||||
ids = torch.tensor(
|
||||
[self._text_eos_id, self._tts_bos_id],
|
||||
device=self.emb_text.weight.device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
ids = index_to_device([self._text_eos_id, self._tts_bos_id], self.emb_text.weight.device)
|
||||
return self.emb_text(ids)
|
||||
|
||||
def _build_condition_embeddings(
|
||||
@@ -278,8 +466,9 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
return self._boundary_embeddings()
|
||||
device = self.emb_text.weight.device
|
||||
dtype = self.emb_text.weight.dtype
|
||||
token_ids = tts_token_ids.to(device=device, dtype=torch.long).reshape(-1)
|
||||
hidden = tts_hidden_states.to(device=device, dtype=dtype)
|
||||
# Pinned, non-blocking H2D: a pageable copy would stall every row in the step.
|
||||
token_ids = to_device_nonblocking(tts_token_ids, device).to(dtype=torch.long).reshape(-1)
|
||||
hidden = to_device_nonblocking(tts_hidden_states, device).to(dtype=dtype)
|
||||
if hidden.shape[0] != token_ids.shape[0] and token_ids.shape[0] != 1:
|
||||
raise ValueError(
|
||||
"MiniCPM-o Talker condition length mismatch: "
|
||||
@@ -289,7 +478,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
hidden_embeds = self.projector_semantic(hidden)
|
||||
if self._normalize:
|
||||
hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1)
|
||||
audio_bos = self.emb_text(torch.tensor([self._tts_bos_id], device=device, dtype=torch.long))
|
||||
audio_bos = self.emb_text(index_to_device([self._tts_bos_id], device))
|
||||
condition = text_embeds + hidden_embeds
|
||||
if native_duplex:
|
||||
# Match MiniCPMTTS.generate_chunk's streaming condition.
|
||||
@@ -565,6 +754,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
# Decode: vLLM's previous sampled codec id is this step's input.
|
||||
# Embed it with the codec table (not the 32k Llama embed_tokens) and
|
||||
# hand the same id to make_omni_output so Code2Wav sees it this step.
|
||||
# The runner sends one-token decode rows through preprocess_decode_batch,
|
||||
# which defers this ``.item()`` EOS check to make_omni_output.
|
||||
code = input_ids.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(-1)[-1:]
|
||||
embeds = self.emb_code[0](code)
|
||||
code_id = int(code.item())
|
||||
@@ -580,6 +771,185 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
delta = torch.tensor([[code_id]], dtype=torch.long, device="cpu")
|
||||
return input_ids, embeds, {"codes": {"audio": delta}}
|
||||
|
||||
def _decode_codec_id_map(self) -> dict[str, tuple[_HostCodecIds, int]]:
|
||||
pending = getattr(self, "_decode_codec_ids", None)
|
||||
if pending is None:
|
||||
pending = {}
|
||||
self._decode_codec_ids = pending
|
||||
return pending
|
||||
|
||||
def preprocess_decode_batch(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor,
|
||||
req_infos: list[dict[str, Any]],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, list[dict[str, Any]]]:
|
||||
"""Embed every one-token decode row at once, without reading ids on the host.
|
||||
|
||||
Same result per row as scalar ``preprocess``, except that its per-row
|
||||
``.item()`` moves to ``make_omni_output``: one non-blocking copy of all
|
||||
rows' ids is queued here, before the forward, and read there to build the
|
||||
CPU delta and apply the codec-EOS check. Nothing reads the request's
|
||||
``finished`` state between the two points.
|
||||
"""
|
||||
num_rows = len(req_infos)
|
||||
if input_ids.numel() != num_rows:
|
||||
raise ValueError(f"MiniCPM-o Talker batched decode needs one id per row: {input_ids.numel()} != {num_rows}")
|
||||
weight = self.emb_code[0].weight
|
||||
codes = input_ids.reshape(-1).to(device=weight.device, dtype=torch.long)
|
||||
embeds = self.emb_code[0](codes)
|
||||
# The runner and connector consume CPU codec ids.
|
||||
empty_codes = torch.empty(0, dtype=torch.long, device="cpu")
|
||||
request_states = self._request_audio_states
|
||||
updates: list[dict[str, Any]] = []
|
||||
live_rows: list[tuple[int, str]] = []
|
||||
zero_rows: list[int] = []
|
||||
for row, info in enumerate(req_infos):
|
||||
request_id = str(info.get("request_id", "0"))
|
||||
state = info.get("audio_state")
|
||||
if info.get("_omni_is_prefill", False) or not isinstance(state, dict):
|
||||
# A row without Talker state builds its condition like a prefill.
|
||||
_, row_embeds, update = self.preprocess(input_ids=input_ids[row : row + 1], input_embeds=None, **info)
|
||||
embeds[row : row + 1] = row_embeds
|
||||
updates.append(update)
|
||||
continue
|
||||
stored = request_states.get(request_id)
|
||||
if isinstance(stored, dict):
|
||||
state = stored
|
||||
if state.get("finished"):
|
||||
# Leftover decode of a finished request: shape-correct zeros.
|
||||
zero_rows.append(row)
|
||||
updates.append({"codes": {"audio": empty_codes}})
|
||||
continue
|
||||
live_rows.append((row, request_id))
|
||||
# The id is not on the host yet: make_omni_output builds this row's
|
||||
# delta from the host copy. The empty entry keeps no stale delta.
|
||||
updates.append({"codes": {"audio": empty_codes}})
|
||||
if zero_rows:
|
||||
embeds.index_fill_(0, index_to_device(zero_rows, embeds.device), 0.0)
|
||||
if live_rows:
|
||||
host_ids = _HostCodecIds(codes)
|
||||
pending = self._decode_codec_id_map()
|
||||
for row, request_id in live_rows:
|
||||
pending[request_id] = (host_ids, row)
|
||||
return input_ids, embeds, updates
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model Runner V2
|
||||
# ------------------------------------------------------------------
|
||||
#
|
||||
# Under MRv2 a decode row's input id is the previous step's sample, which
|
||||
# the runner gathers on the device (``last_sampled_tokens``) and embeds
|
||||
# into its static input buffer with ``embed_input_ids`` (``emb_code``).
|
||||
# That is the whole decode preprocess, so decode rows skip the per-row
|
||||
# hook. Everything V1 derives on the host from that id each step -- the
|
||||
# codec delta, codec-EOS detection, the step count behind the length cap
|
||||
# and the 16-frame penalty history -- is derived on the device from the
|
||||
# runner's request state instead, and nothing waits for a D2H copy.
|
||||
|
||||
#: Decode rows need no per-row preprocess beyond the runner's embedding.
|
||||
mrv2_decode_preprocess_is_identity = True
|
||||
|
||||
@property
|
||||
def logits_vocab_size(self) -> int:
|
||||
return int(self._num_audio_tokens)
|
||||
|
||||
def preprocess_decode_batch_mrv2(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor,
|
||||
input_embeds: torch.Tensor,
|
||||
req_infos: list[dict[str, Any]],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, list[dict[str, Any]]]:
|
||||
"""Decode rows are ``emb_code`` of their input id, already in ``input_embeds``."""
|
||||
num_rows = len(req_infos)
|
||||
empty = input_embeds.new_empty((num_rows, 0))
|
||||
return input_ids, input_embeds, empty, empty, [{} for _ in range(num_rows)]
|
||||
|
||||
def mrv2_custom_sampler(self, sampler: Any) -> tuple[Any, None]:
|
||||
return _install_mrv2_talker_sampler(sampler, self), None
|
||||
|
||||
def make_omni_output_mrv2(
|
||||
self,
|
||||
model_outputs: torch.Tensor | OmniOutput,
|
||||
*,
|
||||
input_batch: Any,
|
||||
req_states: Any,
|
||||
model_intermediate_buffer: list[dict[str, Any]],
|
||||
) -> OmniOutput:
|
||||
"""Device-side ``make_omni_output`` for Model Runner V2.
|
||||
|
||||
Emits every token row's input id as ``codes.audio`` plus
|
||||
``meta.codec_frame_valid``: true only for a decode row whose input is a
|
||||
codec id of a request that has not ended. That id is the code V1 hands
|
||||
to Code2Wav on this step; the stage payload builder drops the other
|
||||
rows on the host, after the asynchronous output copy. Rows the model
|
||||
terminates this step (empty condition, codec EOS already sampled, or
|
||||
the offline length cap) are recorded for the sampler.
|
||||
"""
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
hidden = model_outputs
|
||||
empty_speech = self._mrv2_empty_speech
|
||||
if empty_speech is None:
|
||||
raise RuntimeError("MiniCPM-o Talker MRv2 output requires its MRv2 sampler (mrv2_custom_sampler)")
|
||||
num_reqs = int(input_batch.num_reqs)
|
||||
device = hidden.device
|
||||
if input_batch.has_prefill:
|
||||
# Prefill rows only: record whether the Thinker handed over an
|
||||
# empty condition (preprocess marks such a request finished).
|
||||
slots: list[int] = []
|
||||
flags: list[bool] = []
|
||||
for row in np.flatnonzero(input_batch.is_prefilling_np[:num_reqs]).tolist():
|
||||
info = model_intermediate_buffer[row]
|
||||
if info.get("native_duplex") is True:
|
||||
raise NotImplementedError("MiniCPM-o native duplex Talker requires model_runner: v1")
|
||||
state = info.get("audio_state")
|
||||
flags.append(bool(state.get("finished")) if isinstance(state, dict) else False)
|
||||
slots.append(int(input_batch.idx_mapping_np[row]))
|
||||
if slots:
|
||||
empty_speech.index_copy_(
|
||||
0,
|
||||
index_to_device(slots, device),
|
||||
index_to_device(flags, device, dtype=torch.bool),
|
||||
)
|
||||
num_tokens = int(hidden.shape[0])
|
||||
token_ids = input_batch.input_ids[:num_tokens]
|
||||
last_rows = input_batch.logits_indices[:num_reqs].long()
|
||||
slot_ids = input_batch.idx_mapping[:num_reqs].long()
|
||||
prompt_len = req_states.prompt_len.gpu.index_select(0, slot_ids).long()
|
||||
# Codes emitted before this step's input (V1's ``state["step"]``).
|
||||
step = input_batch.seq_lens[:num_reqs].long() - prompt_len
|
||||
last_ids = token_ids.index_select(0, last_rows)
|
||||
empty = empty_speech.index_select(0, slot_ids)
|
||||
decode = step > 0
|
||||
eos_input = decode & (last_ids == int(self._codec_eos_id))
|
||||
valid = decode & ~eos_input & ~empty
|
||||
# MiniCPMTTS.generate's max_new_token, clamped to the Talker context
|
||||
# (see ``preprocess``); the sample after the last allowed code is EOS.
|
||||
remaining = int(self._tts_config.max_position_embeddings) - prompt_len
|
||||
limit = torch.clamp(remaining, min=1, max=_OFFLINE_CODEC_MAX_NEW_TOKENS) - 1
|
||||
self._mrv2_forced_eos = empty | eos_input | (step >= limit)
|
||||
frame_valid = torch.zeros(num_tokens, dtype=torch.bool, device=device)
|
||||
frame_valid.index_copy_(0, last_rows, valid)
|
||||
if not self._mrv2_decode_rows_logged and not input_batch.has_prefill:
|
||||
self._mrv2_decode_rows_logged = True
|
||||
logger.info("MiniCPM-o Talker: MRv2 device-side codec output active (no host read of sampled ids)")
|
||||
return OmniOutput(
|
||||
text_hidden_states=hidden,
|
||||
multimodal_outputs={
|
||||
"codes": {"audio": token_ids.to(dtype=torch.long).reshape(num_tokens, 1)},
|
||||
"meta": {"codec_frame_valid": frame_valid},
|
||||
},
|
||||
)
|
||||
|
||||
def take_mrv2_forced_eos(self, input_batch: Any, req_states: Any, num_rows: int) -> torch.Tensor | None:
|
||||
"""This step's forced-EOS rows, once; ``None`` outside a model step (warmup)."""
|
||||
forced, self._mrv2_forced_eos = self._mrv2_forced_eos, None
|
||||
if forced is None or int(forced.shape[0]) != int(num_rows):
|
||||
return None
|
||||
return forced
|
||||
|
||||
def make_omni_output(
|
||||
self,
|
||||
model_outputs: torch.Tensor | OmniOutput,
|
||||
@@ -606,6 +976,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
mask_eos_rows = [False] * len(infos)
|
||||
empty_history = torch.empty(0, dtype=torch.long, device="cpu")
|
||||
penalty_histories = [empty_history for _ in infos]
|
||||
pending_codec_ids = self._decode_codec_id_map()
|
||||
codec_eos_id = int(self._codec_eos_id)
|
||||
for index, info in enumerate(infos):
|
||||
info_dict = info if isinstance(info, dict) else {}
|
||||
native_duplex = info_dict.get("native_duplex") is True
|
||||
@@ -657,9 +1029,20 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
if not isinstance(state, dict):
|
||||
state = dict(info.get("audio_state", {}) or {})
|
||||
self._request_audio_states[request_id] = state
|
||||
empty_speech = bool(state.get("finished"))
|
||||
codes = info.get("codes", {})
|
||||
audio = codes.get("audio") if isinstance(codes, Mapping) else None
|
||||
pending = pending_codec_ids.pop(request_id, None)
|
||||
if pending is not None:
|
||||
# A preprocess_decode_batch row: build what scalar preprocess
|
||||
# would have stored, from this step's host copy of the ids.
|
||||
host_ids, row = pending
|
||||
code_id = host_ids.values()[row]
|
||||
if code_id == codec_eos_id:
|
||||
state["finished"] = True
|
||||
audio = None
|
||||
else:
|
||||
audio = torch.tensor([[code_id]], dtype=torch.long, device="cpu")
|
||||
empty_speech = bool(state.get("finished"))
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
audio = audio.to(device="cpu", dtype=torch.long)
|
||||
codec_deltas[index] = audio.reshape(-1, 1)
|
||||
@@ -725,9 +1108,11 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
def _flush_deferred_cleanup(self) -> None:
|
||||
request_audio_states = getattr(self, "_request_audio_states", {})
|
||||
request_condition_states = getattr(self, "_request_condition_states", {})
|
||||
decode_codec_ids = getattr(self, "_decode_codec_ids", {})
|
||||
for request_id in self._deferred_cleanup_ids:
|
||||
request_audio_states.pop(request_id, None)
|
||||
request_condition_states.pop(request_id, None)
|
||||
decode_codec_ids.pop(request_id, None)
|
||||
self._deferred_cleanup_ids.clear()
|
||||
|
||||
def _dummy_hidden_states(
|
||||
@@ -789,18 +1174,21 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
# we just forced (it is in the stage's ``stop_token_ids``), leaving an
|
||||
# all -inf row and a request that never releases.
|
||||
self._pending_force_eos_rows = force_eos if need_force else None
|
||||
# Row masks go up through pinned memory and are applied with where /
|
||||
# masked_fill: a pageable H2D or boolean-mask indexing blocks the host.
|
||||
if need_force or need_mask:
|
||||
logits = logits.clone()
|
||||
eos_id = int(self._codec_eos_id)
|
||||
if need_force:
|
||||
assert force_eos is not None
|
||||
forced = torch.tensor(force_eos, dtype=torch.bool, device=logits.device)
|
||||
logits[forced] = float("-inf")
|
||||
logits[forced, eos_id] = 0.0
|
||||
forced = index_to_device(force_eos, logits.device, dtype=torch.bool)
|
||||
eos_only = torch.full_like(logits[:1], float("-inf"))
|
||||
eos_only[:, eos_id] = 0.0
|
||||
logits = torch.where(forced.unsqueeze(1), eos_only, logits)
|
||||
if need_mask:
|
||||
assert mask_eos is not None
|
||||
masked = torch.tensor(mask_eos, dtype=torch.bool, device=logits.device)
|
||||
logits[masked, eos_id] = float("-inf")
|
||||
masked = index_to_device(mask_eos, logits.device, dtype=torch.bool)
|
||||
logits[:, eos_id].masked_fill_(masked, float("-inf"))
|
||||
return logits
|
||||
|
||||
def sample(self, logits, sampling_metadata):
|
||||
@@ -867,8 +1255,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
sampled = getattr(output, "sampled_token_ids", None)
|
||||
if not isinstance(sampled, torch.Tensor) or sampled.shape[0] != len(force_eos):
|
||||
return output
|
||||
rows = torch.tensor(force_eos, dtype=torch.bool, device=sampled.device)
|
||||
sampled[rows] = int(self._codec_eos_id)
|
||||
rows = index_to_device(force_eos, sampled.device, dtype=torch.bool)
|
||||
sampled.masked_fill_(rows.view(-1, *([1] * (sampled.ndim - 1))), int(self._codec_eos_id))
|
||||
return output
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
|
||||
@@ -47,6 +47,7 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
supports_native_mrv2_data_plane=True,
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
# The llm2tts bridge discards this boundary and every row
|
||||
@@ -65,6 +66,9 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
|
||||
custom_process_input_func=f"{_PROC}.llm2tts",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.tts2code2wav_full_payload",
|
||||
async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk",
|
||||
# Takes effect only when the deploy selects model_runner v2 for
|
||||
# this stage (turn sessions only; duplex stays on V1).
|
||||
supports_native_mrv2_data_plane=True,
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
# MiniCPM-o 4.5 codec EOS is tts_config.num_audio_tokens - 1.
|
||||
@@ -81,6 +85,7 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="MiniCPMO45Code2Wav",
|
||||
supports_native_mrv2_data_plane=True,
|
||||
sync_process_input_func=f"{_PROC}.tts2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
requires_full_payload_input=True,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""MiniCPM-o 4.5 Thinker-to-Talker and Talker-to-Code2Wav bridges."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -203,6 +203,14 @@ def _extract_codec_delta(pooling_output: Any, request_id: str) -> list[int]:
|
||||
return []
|
||||
codes = pooling_output.get("codes")
|
||||
audio = codes.get("audio") if isinstance(codes, Mapping) else pooling_output.get("codes.audio")
|
||||
valid = meta.get("codec_frame_valid") if isinstance(meta, Mapping) else None
|
||||
if valid is None:
|
||||
valid = pooling_output.get("meta.codec_frame_valid")
|
||||
if isinstance(valid, torch.Tensor) and isinstance(audio, torch.Tensor):
|
||||
# Model Runner V2 emits one id per token row plus its validity
|
||||
# (decode rows of live requests); keep only the codec frames.
|
||||
rows = valid.detach().to(device="cpu").reshape(-1).bool()
|
||||
audio = audio.detach().to(device="cpu").reshape(rows.numel(), -1)[rows]
|
||||
return _codec_scalars(audio)
|
||||
if isinstance(pooling_output, Sequence) and not isinstance(
|
||||
pooling_output,
|
||||
@@ -227,7 +235,7 @@ def _drop_codec_state(transfer_manager: Any, request_id: str) -> None:
|
||||
else:
|
||||
request_payload.pop(request_id, None)
|
||||
code_accumulators = getattr(transfer_manager, "code_prompt_token_ids", None)
|
||||
if hasattr(code_accumulators, "pop"):
|
||||
if isinstance(code_accumulators, dict):
|
||||
code_accumulators.pop(request_id, None)
|
||||
|
||||
|
||||
@@ -316,7 +324,8 @@ def tts2code2wav_async_chunk(
|
||||
container[_MINICPMO45_ASYNC_STATE] = state
|
||||
|
||||
pending = state["pending"]
|
||||
pending.extend(_extract_codec_delta(multimodal_output, request_id))
|
||||
delta = _extract_codec_delta(multimodal_output, request_id)
|
||||
pending.extend(delta)
|
||||
pending_text_utf8 = state.setdefault("pending_text_utf8", [])
|
||||
current_text_utf8 = (
|
||||
segment_text_utf8.detach().to(device="cpu", dtype=torch.uint8).reshape(-1).tolist()
|
||||
@@ -602,7 +611,7 @@ def _decode_native_duplex_token_ids(
|
||||
request_id: str,
|
||||
) -> str | None:
|
||||
decode_token_ids = getattr(streaming_context, "source_token_decoder", None)
|
||||
if not isinstance(decode_token_ids, Callable):
|
||||
if not callable(decode_token_ids):
|
||||
return None
|
||||
decode_ids = [int(token_id) for token_id in token_ids]
|
||||
try:
|
||||
@@ -1006,7 +1015,9 @@ def llm2tts(
|
||||
if data_plane_metadata is not None:
|
||||
model_intermediate_buffer["duplex"] = data_plane_metadata
|
||||
meta["native_duplex_segment_text"] = thinker_text
|
||||
meta.setdefault("override_keys", []).extend(
|
||||
override_keys = meta.setdefault("override_keys", [])
|
||||
assert isinstance(override_keys, list)
|
||||
override_keys.extend(
|
||||
[
|
||||
"llm_output_text",
|
||||
["meta", "native_duplex_segment_text"],
|
||||
|
||||
@@ -35,7 +35,8 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.outputs import OmniModelRunnerOutput
|
||||
from vllm_omni.utils.mm_outputs import partition_flat_payload
|
||||
from vllm_omni.worker_v2.omni_model_runner import OmniGPUModelRunner
|
||||
from vllm_omni.worker_v2.output_snapshot import PackedOutputSnapshot, pack_output_snapshot
|
||||
from vllm_omni.worker_v2.omni_sampler import sample_with_output
|
||||
from vllm_omni.worker_v2.output_snapshot import PackedOutputSnapshot, RequestOutputSnapshot, pack_output_snapshot
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_ASYNC_MM_SNAPSHOT_MAX_BUCKETS_PER_SLOT = 64
|
||||
@@ -187,11 +188,13 @@ class OmniARModelRunner(OmniGPUModelRunner):
|
||||
),
|
||||
)
|
||||
with sampling_context:
|
||||
sampler_output, num_sampled, num_rejected = self.sample(
|
||||
text_hidden,
|
||||
input_batch,
|
||||
grammar_output,
|
||||
sampling_output = sample_with_output(
|
||||
self.sampler, self.sample, text_hidden, input_batch, self.req_states, grammar_output
|
||||
)
|
||||
sampler_output = sampling_output.sampler_output
|
||||
num_sampled, num_rejected = sampling_output.num_sampled, sampling_output.num_rejected
|
||||
if sampling_output.multimodal_outputs is not None:
|
||||
multimodal_outputs = sampling_output.multimodal_outputs
|
||||
run_eager_mtp = getattr(self.model_state, "run_eager_mtp", None)
|
||||
if multimodal_outputs and run_eager_mtp is not None:
|
||||
run_eager_mtp(
|
||||
@@ -255,11 +258,12 @@ class OmniARModelRunner(OmniGPUModelRunner):
|
||||
num_sampled_tokens=num_sampled,
|
||||
main_stream=self.main_stream,
|
||||
copy_stream=self.output_copy_stream,
|
||||
text_hidden=text_hidden if need_pooler else None,
|
||||
text_hidden=text_hidden if need_pooler and sampling_output.include_hidden_states else None,
|
||||
multimodal_outputs=multimodal_outputs if need_pooler else None,
|
||||
input_batch=input_batch if need_pooler else None,
|
||||
async_chunk=bool(getattr(self.model_config, "async_chunk", False)),
|
||||
finalize_output=(None if materialize_native else self._finalize_native_data_plane_output),
|
||||
finalize_multimodal=sampling_output.finalize_multimodal,
|
||||
check_ep_fault=self.check_ep_fault,
|
||||
routed_experts=routed_experts,
|
||||
extra_multimodal_outputs=extra_outputs,
|
||||
@@ -268,7 +272,8 @@ class OmniARModelRunner(OmniGPUModelRunner):
|
||||
_guard_graph_replay_for_pooler_copy(
|
||||
self.main_stream,
|
||||
async_output.copy_event,
|
||||
need_pooler=need_pooler,
|
||||
need_pooler=need_pooler
|
||||
and (sampling_output.include_hidden_states or not sampling_output.owns_multimodal_outputs),
|
||||
async_chunk=bool(getattr(self.model_config, "async_chunk", False)),
|
||||
)
|
||||
|
||||
@@ -671,6 +676,7 @@ class OmniAsyncOutput(AsyncModelRunnerOutput):
|
||||
input_batch: Any | None = None,
|
||||
async_chunk: bool = False,
|
||||
finalize_output: Any | None = None,
|
||||
finalize_multimodal: Any | None = None,
|
||||
check_ep_fault: bool = False,
|
||||
routed_experts: RoutedExpertsTensors | None = None,
|
||||
extra_multimodal_outputs: tuple[dict[str, Any], torch.cuda.Event] | None = None,
|
||||
@@ -682,10 +688,13 @@ class OmniAsyncOutput(AsyncModelRunnerOutput):
|
||||
self.copy_event = copy_event if copy_event is not None else torch.cuda.Event(blocking=True)
|
||||
self._async_chunk = bool(async_chunk)
|
||||
self._finalize_output = finalize_output
|
||||
self._finalize_multimodal = finalize_multimodal
|
||||
self._has_fault: torch.Tensor | None = None
|
||||
|
||||
# Snapshot input_batch metadata needed for pooler_output slicing
|
||||
self._need_pooler = text_hidden is not None or (self._async_chunk and bool(multimodal_outputs))
|
||||
self._need_pooler = text_hidden is not None or (
|
||||
(self._async_chunk or finalize_multimodal is not None) and bool(multimodal_outputs)
|
||||
)
|
||||
self._query_start_loc_np: np.ndarray | None = None
|
||||
self._num_scheduled_tokens: np.ndarray | None = None
|
||||
self._num_reqs: int = 0
|
||||
@@ -759,7 +768,7 @@ class OmniAsyncOutput(AsyncModelRunnerOutput):
|
||||
self._hidden_cpu: torch.Tensor | None = None
|
||||
self._mm_cpu: dict[str, Any] = {}
|
||||
self._mm_snapshot: dict[str, Any] = {}
|
||||
if self._need_pooler and self._async_chunk:
|
||||
if self._need_pooler and (self._async_chunk or self._finalize_multimodal is not None):
|
||||
# CUDA graph replay reuses the model's output buffers. Take
|
||||
# ownership directly in pinned host memory on the output copy
|
||||
# stream so deferred finalization never performs a blocking
|
||||
@@ -836,16 +845,27 @@ class OmniAsyncOutput(AsyncModelRunnerOutput):
|
||||
# * pooler_output -> sync/full-payload path (inline pooling_output bridge)
|
||||
# * multimodal_outputs -> wire multimodal_output, which the async_chunk
|
||||
# stage-input processor (talker2code2wav_async_chunk) reads for codes.
|
||||
if self._need_pooler and self._async_chunk:
|
||||
pooler_inter, pooler_client = OmniARModelRunner._build_async_chunk_outputs_from_mm(
|
||||
self._mm_snapshot,
|
||||
self._query_start_loc_np,
|
||||
self._num_scheduled_tokens,
|
||||
self._num_reqs,
|
||||
self._total_tokens,
|
||||
self._padded_total_tokens,
|
||||
)
|
||||
self.model_runner_output.pooler_output = None
|
||||
if self._need_pooler and (self._async_chunk or self._finalize_multimodal is not None):
|
||||
if self._finalize_multimodal is not None:
|
||||
self._mm_snapshot = self._finalize_multimodal(self._mm_snapshot, num_sampled_tokens)
|
||||
pooler_inter: list[dict[str, Any] | None] | None
|
||||
pooler_client: list[dict[str, Any] | None] | None
|
||||
if isinstance(self._mm_snapshot, RequestOutputSnapshot):
|
||||
pooler_inter, pooler_client = self._mm_snapshot.inter_stage, self._mm_snapshot.client
|
||||
if len(pooler_inter) != self._num_reqs or (
|
||||
pooler_client is not None and len(pooler_client) != self._num_reqs
|
||||
):
|
||||
raise ValueError("Model-owned output snapshot does not match the request batch")
|
||||
else:
|
||||
pooler_inter, pooler_client = OmniARModelRunner._build_async_chunk_outputs_from_mm(
|
||||
self._mm_snapshot,
|
||||
self._query_start_loc_np,
|
||||
self._num_scheduled_tokens,
|
||||
self._num_reqs,
|
||||
self._total_tokens,
|
||||
self._padded_total_tokens,
|
||||
)
|
||||
self.model_runner_output.pooler_output = None if self._async_chunk else pooler_inter
|
||||
self.model_runner_output.inter_stage_outputs = pooler_inter
|
||||
self.model_runner_output.multimodal_outputs = (
|
||||
[_ensure_tensor_values(_async_copy_mm_value(p)) if p else {} for p in pooler_client]
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""One sampler registration contract for token and post-sampling audio output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.v1.core.sched.output import GrammarOutput
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
from vllm_omni.worker_v2.output_snapshot import RequestOutputSnapshot
|
||||
|
||||
StandardSample = Callable[
|
||||
[torch.Tensor, InputBatch, GrammarOutput | None], tuple[SamplerOutput, torch.Tensor, torch.Tensor]
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OmniSamplingOutput:
|
||||
sampler_output: SamplerOutput
|
||||
num_sampled: torch.Tensor
|
||||
num_rejected: torch.Tensor
|
||||
# None preserves forward's payload. An empty dict explicitly replaces it.
|
||||
# Tensors must remain valid until the runner finishes its output copy.
|
||||
multimodal_outputs: dict[str, Any] | None = None
|
||||
include_hidden_states: bool = True
|
||||
owns_multimodal_outputs: bool = False
|
||||
# Invoked on CPU-owned snapshots after D2H, with effective per-request counts.
|
||||
finalize_multimodal: Callable[[dict[str, Any], list[int]], dict[str, Any] | RequestOutputSnapshot] | None = None
|
||||
|
||||
|
||||
class OmniSampler:
|
||||
"""Register through ModelState.custom_sampler, just like an upstream sampler.
|
||||
|
||||
Delegate request lifecycle/state to the original sampler without copying
|
||||
its attributes. Models can override __call__ for logits-only adjustments,
|
||||
or sample_step when sampling also produces audio. Both use one registered
|
||||
sampler and one result contract; there is no separate ModelState step hook.
|
||||
"""
|
||||
|
||||
def __init__(self, base_sampler: Sampler):
|
||||
self.base_sampler = base_sampler
|
||||
|
||||
@property
|
||||
def omni_static_staged_writes(self) -> bool:
|
||||
# Preserve the runner's no-new-requests fast path for the stock sampler.
|
||||
return type(self.base_sampler) is Sampler or bool(
|
||||
getattr(self.base_sampler, "omni_static_staged_writes", False)
|
||||
)
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self.base_sampler, name)
|
||||
|
||||
def __call__(self, logits: torch.Tensor, input_batch: InputBatch) -> SamplerOutput:
|
||||
return self.base_sampler(logits, input_batch)
|
||||
|
||||
def sample_step(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
grammar_output: GrammarOutput | None,
|
||||
standard_sample: StandardSample,
|
||||
) -> OmniSamplingOutput:
|
||||
# standard_sample calls this adapter's __call__ through the runner.
|
||||
return OmniSamplingOutput(*standard_sample(hidden_states, input_batch, grammar_output))
|
||||
|
||||
|
||||
def sample_with_output(
|
||||
sampler: Sampler | OmniSampler | None,
|
||||
standard_sample: StandardSample,
|
||||
hidden_states: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
grammar_output: GrammarOutput | None,
|
||||
) -> OmniSamplingOutput:
|
||||
if isinstance(sampler, OmniSampler):
|
||||
return sampler.sample_step(hidden_states, input_batch, req_states, grammar_output, standard_sample)
|
||||
return OmniSamplingOutput(*standard_sample(hidden_states, input_batch, grammar_output))
|
||||
@@ -6,12 +6,25 @@ from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.utils._pytree import tree_flatten, tree_unflatten
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RequestOutputSnapshot:
|
||||
"""Already partitioned, CPU-owned payloads in the current request order.
|
||||
|
||||
Model finalizers may return this after D2H completes to avoid recursively
|
||||
cloning and partitioning data whose request ownership is already known.
|
||||
"""
|
||||
|
||||
inter_stage: list[dict[str, Any] | None]
|
||||
client: list[dict[str, Any] | None] | None = None
|
||||
|
||||
|
||||
class PackedOutputSnapshot(dict):
|
||||
"""A normal payload mapping with an internal batched-copy plan.
|
||||
|
||||
|
||||
Reference in new issue
Block a user