[perf] Minicpm o4.5 optimization (#5228)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
rein yangandSy03 authored and GitHub committed 2026-07-24 17:20:39 +08:00
1 parent acf941da4d
commit ad3a162eb8
54 files changed
+3861 -3997

No files matched your search

@@ -0,0 +1,400 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Strict, state-explicit batching for MiniCPM-o 4.5 Token2wav."""
from __future__ import annotations
from contextlib import nullcontext
from dataclasses import dataclass
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
_SILENCE_TOKEN = 4218
def tensor_signature(value: torch.Tensor) -> tuple[tuple[int, ...], str, str]:
return tuple(value.shape), str(value.dtype), value.device.type
def state_shape_signature(state: BatchedToken2WavState) -> tuple[Any, ...]:
flow = tuple((name, tensor_signature(state.flow_cache[name])) for name in sorted(state.flow_cache))
hift = tuple((name, tensor_signature(state.hift_cache[name])) for name in sorted(state.hift_cache))
return flow, hift
@dataclass(frozen=True)
class PromptFeatures:
speech_tokens: torch.Tensor
speaker_embedding: torch.Tensor
mels: torch.Tensor
@dataclass(frozen=True)
class BatchedToken2WavState:
flow_cache: dict[str, torch.Tensor]
hift_cache: dict[str, torch.Tensor]
class BatchedToken2Wav(nn.Module):
"""Drive Token2wav's modules with dynamically-sized, request-owned caches.
This class intentionally never calls ``Token2wav.stream`` or
``Token2wav.__call__``. The upstream object is used only as a one-time
asset loader and prompt feature extractor.
"""
def __init__(self, token2wav: Any):
super().__init__()
self._token2wav = token2wav
self.flow = token2wav.flow
self.hift = token2wav.hift
self.float16 = bool(token2wav.float16)
self.n_timesteps = int(token2wav.n_timesteps)
self.mel_cache_len = int(token2wav.mel_cache_len)
self.source_cache_len = int(token2wav.source_cache_len)
self.register_buffer(
"speech_window",
token2wav.speech_window.detach().clone(),
persistent=False,
)
self._prompt_features: dict[tuple[str, str], PromptFeatures] = {}
def prepare_prompt(self, prompt_cache_id: str, prompt_wav: str) -> PromptFeatures:
cache_key = (prompt_cache_id, prompt_wav)
cached = self._prompt_features.get(cache_key)
if cached is None:
# The generation runner may wrap model.forward in bf16 autocast,
# and vLLM constructs the model under a bf16 default dtype, while
# S3Tokenizer prompt extraction uses fp32 convolution weights.
previous_dtype = torch.get_default_dtype()
try:
torch.set_default_dtype(torch.float32)
with torch.amp.autocast("cuda", enabled=False):
values = self._token2wav._prepare_prompt(prompt_wav)
finally:
torch.set_default_dtype(previous_dtype)
cached = PromptFeatures(
speech_tokens=values[0],
speaker_embedding=values[2],
mels=values[3],
)
self._prompt_features[cache_key] = cached
return cached
def evict_prompt(self, prompt_cache_id: str, prompt_wav: str) -> None:
"""Release request-owned prompt features after stream completion."""
self._prompt_features.pop((prompt_cache_id, prompt_wav), None)
@staticmethod
def _repeat_prompt(features: PromptFeatures, batch_size: int) -> tuple[torch.Tensor, ...]:
return (
features.speech_tokens.expand(batch_size, -1),
features.speaker_embedding.expand(batch_size, -1),
features.mels.expand(batch_size, -1, -1),
)
def _autocast(self, device: torch.device):
if device.type != "cuda":
return nullcontext()
if not self.float16:
return torch.amp.autocast("cuda", enabled=False)
return torch.amp.autocast(
"cuda",
dtype=torch.float16,
)
def _encode_chunk(
self,
tokens: torch.Tensor,
*,
last_chunk: bool,
cnn_cache: torch.Tensor | None,
att_cache: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
embedded = self.flow.input_embedding(tokens)
hidden, new_cnn, new_att = self.flow.encoder.forward_chunk(
xs=embedded,
last_chunk=last_chunk,
cnn_cache=cnn_cache,
att_cache=att_cache,
)
return self.flow.encoder_proj(hidden), new_cnn, new_att
@staticmethod
def _estimator_buffers(
estimator: nn.Module,
x: torch.Tensor,
old_att: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
blocks = estimator.blocks
depth = len(blocks)
batch_size = int(x.shape[0])
chunk_size = int(x.shape[2])
old_att_len = int(old_att.shape[3]) if old_att is not None else 0
block0 = blocks[0]
cnn_channels = int(block0.conv.in_channels + block0.conv.out_channels)
cnn_width = int(block0.conv.block[1].causal_padding[0])
heads = int(block0.attn.num_heads)
att_width = int(block0.attn.head_dim * 2)
cnn = x.new_empty((depth, batch_size, cnn_channels, cnn_width))
att = x.new_empty((depth, batch_size, heads, old_att_len + chunk_size, att_width))
return cnn, att
def _estimator_step(
self,
estimator: nn.Module,
*,
x: torch.Tensor,
mu: torch.Tensor,
time: torch.Tensor,
speakers: torch.Tensor,
cond: torch.Tensor,
cnn_cache: torch.Tensor | None,
att_cache: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
time_embedding = estimator.t_embedder(time).unsqueeze(1)
width = int(x.shape[-1])
speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width)
estimator_input = torch.cat((x, mu, speaker_features, cond), dim=1)
cnn_out, att_out = self._estimator_buffers(estimator, estimator_input, att_cache)
old_cnn: Any = cnn_cache if cnn_cache is not None else [None] * len(estimator.blocks)
old_att: Any = att_cache if att_cache is not None else [None] * len(estimator.blocks)
result = estimator.blocks_forward_chunk(
estimator_input,
time_embedding,
None,
old_cnn,
old_att,
cnn_out,
att_out,
)
return result, cnn_out, att_out
def _decode_cfm(
self,
mu: torch.Tensor,
speakers: torch.Tensor,
cond: torch.Tensor,
*,
cnn_cache: torch.Tensor | None,
att_cache: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
decoder = self.flow.decoder
estimator = decoder.estimator
batch_size = int(mu.shape[0])
offset = int(att_cache.shape[4]) if att_cache is not None else 0
end = offset + int(mu.shape[2])
if end > int(decoder.rand_noise.shape[2]):
raise RuntimeError(
"MiniCPMO45Code2WavBatchError "
f'{{"reason":"noise_capacity","required":{end},'
f'"available":{int(decoder.rand_noise.shape[2])}}}'
)
x = decoder.rand_noise[:, :, offset:end].expand(batch_size, -1, -1).clone()
timeline = torch.linspace(
0,
1,
self.n_timesteps + 1,
device=mu.device,
dtype=mu.dtype,
)
timeline = 1 - torch.cos(timeline * 0.5 * torch.pi)
time = timeline[0].expand(batch_size)
mu_cfg = torch.cat((mu, torch.zeros_like(mu)), dim=0)
speakers_cfg = torch.cat((speakers, torch.zeros_like(speakers)), dim=0)
cond_cfg = torch.cat((cond, torch.zeros_like(cond)), dim=0)
next_cnn: list[torch.Tensor] = []
next_att: list[torch.Tensor] = []
dt = timeline[1] - timeline[0]
for step in range(self.n_timesteps):
old_cnn = cnn_cache[step] if cnn_cache is not None else None
old_att = att_cache[step] if att_cache is not None else None
estimate, step_cnn, step_att = self._estimator_step(
estimator,
x=torch.cat((x, x), dim=0),
mu=mu_cfg,
time=torch.cat((time, time), dim=0),
speakers=speakers_cfg,
cond=cond_cfg,
cnn_cache=old_cnn,
att_cache=old_att,
)
conditional, unconditional = estimate.split(batch_size, dim=0)
velocity = (1.0 + decoder.inference_cfg_rate) * conditional - decoder.inference_cfg_rate * unconditional
x = x + dt * velocity
time = time + dt
if step + 1 < self.n_timesteps:
dt = timeline[step + 2] - time[0]
next_cnn.append(step_cnn)
next_att.append(step_att)
return x, torch.stack(next_cnn), torch.stack(next_att)
@staticmethod
def _split_flow_cache(cache: dict[str, torch.Tensor], batch_size: int) -> list[dict[str, torch.Tensor]]:
result: list[dict[str, torch.Tensor]] = []
for row in range(batch_size):
result.append(
{
"conformer_cnn_cache": cache["conformer_cnn_cache"][row : row + 1].detach().clone(),
"conformer_att_cache": cache["conformer_att_cache"][:, row : row + 1].detach().clone(),
"estimator_cnn_cache": torch.cat(
(
cache["estimator_cnn_cache"][:, :, row : row + 1],
cache["estimator_cnn_cache"][:, :, batch_size + row : batch_size + row + 1],
),
dim=2,
).detach(),
"estimator_att_cache": torch.cat(
(
cache["estimator_att_cache"][:, :, row : row + 1],
cache["estimator_att_cache"][:, :, batch_size + row : batch_size + row + 1],
),
dim=2,
).detach(),
}
)
return result
@staticmethod
def _stack_flow_cache(states: list[BatchedToken2WavState]) -> dict[str, torch.Tensor]:
flows = [state.flow_cache for state in states]
conditional_cnn = [flow["estimator_cnn_cache"][:, :, 0:1] for flow in flows]
unconditional_cnn = [flow["estimator_cnn_cache"][:, :, 1:2] for flow in flows]
conditional_att = [flow["estimator_att_cache"][:, :, 0:1] for flow in flows]
unconditional_att = [flow["estimator_att_cache"][:, :, 1:2] for flow in flows]
return {
"conformer_cnn_cache": torch.cat([flow["conformer_cnn_cache"] for flow in flows], dim=0),
"conformer_att_cache": torch.cat([flow["conformer_att_cache"] for flow in flows], dim=1),
"estimator_cnn_cache": torch.cat((*conditional_cnn, *unconditional_cnn), dim=2),
"estimator_att_cache": torch.cat((*conditional_att, *unconditional_att), dim=2),
}
def setup_batch(
self,
features: PromptFeatures,
batch_size: int,
) -> list[BatchedToken2WavState]:
prompt_tokens, speakers, prompt_mels = self._repeat_prompt(features, batch_size)
lookahead = prompt_tokens.new_full((batch_size, 3), _SILENCE_TOKEN)
with self._autocast(prompt_tokens.device):
hidden, conformer_cnn, conformer_att = self._encode_chunk(
torch.cat((prompt_tokens, lookahead), dim=1),
last_chunk=False,
cnn_cache=None,
att_cache=None,
)
projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1))
_, estimator_cnn, estimator_att = self._decode_cfm(
hidden.transpose(1, 2).contiguous(),
projected_speakers,
prompt_mels.transpose(1, 2).contiguous(),
cnn_cache=None,
att_cache=None,
)
flow_cache = {
"conformer_cnn_cache": conformer_cnn,
"conformer_att_cache": conformer_att,
"estimator_cnn_cache": estimator_cnn,
"estimator_att_cache": estimator_att,
}
split = self._split_flow_cache(flow_cache, batch_size)
mel_channels = int(prompt_mels.shape[2])
return [
BatchedToken2WavState(
flow_cache=row,
hift_cache={
"mel": prompt_mels.new_zeros((1, mel_channels, 0)),
"source": prompt_mels.new_zeros((1, 1, 0)),
"speech": prompt_mels.new_zeros((1, 0)),
},
)
for row in split
]
@staticmethod
def _fade_in_out(
speech: torch.Tensor,
previous: torch.Tensor,
window: torch.Tensor,
) -> torch.Tensor:
overlap = int(window.shape[0] // 2)
result = speech.clone()
result[..., :overlap] = result[..., :overlap] * window[:overlap] + previous[..., -overlap:] * window[overlap:]
return result
def decode_batch(
self,
tokens: torch.Tensor,
features: PromptFeatures,
states: list[BatchedToken2WavState],
*,
last_chunk: bool,
) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]:
batch_size = int(tokens.shape[0])
if batch_size != len(states):
raise ValueError(f"tokens batch {batch_size} != state batch {len(states)}")
flow_cache = self._stack_flow_cache(states)
speakers = features.speaker_embedding.expand(batch_size, -1)
with self._autocast(tokens.device):
hidden, conformer_cnn, conformer_att = self._encode_chunk(
tokens,
last_chunk=last_chunk,
cnn_cache=flow_cache["conformer_cnn_cache"],
att_cache=flow_cache["conformer_att_cache"],
)
projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1))
cond = torch.zeros_like(hidden).transpose(1, 2).contiguous()
chunk_mel, estimator_cnn, estimator_att = self._decode_cfm(
hidden.transpose(1, 2).contiguous(),
projected_speakers,
cond,
cnn_cache=flow_cache["estimator_cnn_cache"],
att_cache=flow_cache["estimator_att_cache"],
)
prompt_len = int(features.mels.shape[1])
if estimator_att.shape[4] > prompt_len + 100:
estimator_att = torch.cat(
(estimator_att[..., :prompt_len, :], estimator_att[..., -100:, :]),
dim=4,
)
if conformer_att.shape[3] > prompt_len + 100:
conformer_att = torch.cat(
(conformer_att[..., :prompt_len, :], conformer_att[..., -100:, :]),
dim=3,
)
new_flow = self._split_flow_cache(
{
"conformer_cnn_cache": conformer_cnn,
"conformer_att_cache": conformer_att,
"estimator_cnn_cache": estimator_cnn,
"estimator_att_cache": estimator_att,
},
batch_size,
)
old_mel = torch.cat([state.hift_cache["mel"] for state in states], dim=0)
old_source = torch.cat([state.hift_cache["source"] for state in states], dim=0)
old_speech = torch.cat([state.hift_cache["speech"] for state in states], dim=0)
mel = torch.cat((old_mel, chunk_mel), dim=2)
speech, source = self.hift(mel, old_source)
if old_speech.shape[-1] > 0:
window = self.speech_window.to(device=speech.device, dtype=speech.dtype)
speech = self._fade_in_out(speech, old_speech, window)
next_hift = {
"mel": mel[..., -self.mel_cache_len :].detach(),
"source": source[..., -self.source_cache_len :].detach(),
"speech": speech[..., -self.source_cache_len :].detach(),
}
emitted = speech if last_chunk else speech[..., : -self.source_cache_len]
next_states = [
BatchedToken2WavState(
flow_cache=new_flow[row],
hift_cache={name: value[row : row + 1].detach().clone() for name, value in next_hift.items()},
)
for row in range(batch_size)
]
audios = [emitted[row].reshape(-1).to(dtype=torch.float32) for row in range(batch_size)]
return audios, next_states
@@ -0,0 +1,547 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Strict batched codec-to-waveform stage for MiniCPM-o 4.5."""
from __future__ import annotations
import hashlib
import json
import os
import tempfile
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import soundfile as sf
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm_omni.model_executor.models.output_templates import OmniOutput
from .batched_token2wav import (
BatchedToken2Wav,
BatchedToken2WavState,
state_shape_signature,
)
def _batch_error(reason: str, **details: Any) -> RuntimeError:
payload = {"reason": reason, **details}
return RuntimeError(f"MiniCPMO45Code2WavBatchError {json.dumps(payload, sort_keys=True)}")
def _scalar(value: Any, default: Any = None) -> Any:
if isinstance(value, torch.Tensor):
return value.reshape(-1)[0].item() if value.numel() else default
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
return _scalar(value[0], default) if value else default
return default if value is None else value
def _codec_tensor(value: Any, fallback: torch.Tensor) -> torch.Tensor:
if isinstance(value, torch.Tensor):
return value.reshape(-1).to(device=fallback.device, dtype=torch.long)
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
return torch.as_tensor(value, device=fallback.device, dtype=torch.long).reshape(-1)
return fallback.reshape(-1).to(dtype=torch.long)
@dataclass(frozen=True)
class _RequestState:
cache_epoch: int
chunk_seq: int
prompt_cache_id: str
prompt_wav: str
token2wav: BatchedToken2WavState
@dataclass(frozen=True)
class _WorkItem:
output_index: int
state_id: str
request_id: str
cache_epoch: int
chunk_seq: int
prompt_cache_id: str
prompt_wav: str
last_chunk: bool
tokens: torch.Tensor
previous: _RequestState | None
segment_text: str
duplex_turn_id: int | None
duplex_epoch: int | None
segment_end: bool
turn_end: bool
class MiniCPMO45Code2Wav(nn.Module):
"""LLM_GENERATION model that admits only true exact-shape GPU batches."""
input_modalities = "audio"
have_multimodal_outputs = True
enable_update_additional_information = True
requires_raw_input_tokens = True
requires_request_ids = True
has_preprocess = False
has_postprocess = False
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__()
del prefix
self.vllm_config = vllm_config
self.model_path = str(vllm_config.model_config.model)
self.backend: BatchedToken2Wav | None = None
self._states: dict[str, _RequestState] = {}
self._owned_prompt_wavs: dict[str, tuple[str, str]] = {}
extra = self._extra_config()
self._min_batch_size = int(extra.get("code2wav_min_batch_size", 1))
if self._min_batch_size < 1:
raise ValueError("MiniCPM-o Code2Wav code2wav_min_batch_size must be >= 1")
self._default_prompt_id = str(extra.get("prompt_cache_id", "HT_ref_audio"))
self._default_prompt_wav = str(
extra.get(
"prompt_wav",
Path(self.model_path) / "assets" / "HT_ref_audio.wav",
)
)
def _extra_config(self) -> dict[str, Any]:
model_config = getattr(self.vllm_config, "model_config", None)
connector = getattr(model_config, "stage_connector_config", None)
if isinstance(connector, Mapping):
extra = connector.get("extra", connector)
else:
extra = getattr(connector, "extra", None)
return dict(extra) if isinstance(extra, Mapping) else {}
def _release_prompt(self, state_id: str) -> None:
owned = self._owned_prompt_wavs.pop(state_id, None)
if owned is None:
return
prompt_cache_id, prompt_wav = owned
if self.backend is not None:
self.backend.evict_prompt(prompt_cache_id, prompt_wav)
try:
os.unlink(prompt_wav)
except FileNotFoundError:
pass
def _write_reference_prompt(
self,
state_id: str,
reference: object,
sample_rate: object,
) -> tuple[str, str]:
waveform = torch.as_tensor(reference, dtype=torch.float32).reshape(-1).cpu()
if waveform.numel() == 0:
return self._default_prompt_id, self._default_prompt_wav
sample_rate_hz = int(_scalar(sample_rate, 24000))
if sample_rate_hz <= 0:
raise _batch_error(
"invalid_ref_audio_sample_rate",
request_id=state_id,
sample_rate=sample_rate_hz,
)
digest = hashlib.sha256()
digest.update(str(sample_rate_hz).encode("ascii"))
digest.update(waveform.numpy().tobytes())
prompt_cache_id = f"{state_id}:{digest.hexdigest()}"
current = self._owned_prompt_wavs.get(state_id)
if current is not None and current[0] == prompt_cache_id and os.path.exists(current[1]):
return current
self._release_prompt(state_id)
handle = tempfile.NamedTemporaryFile(prefix="minicpmo45-ref-", suffix=".wav", delete=False)
prompt_wav = handle.name
handle.close()
try:
sf.write(prompt_wav, waveform.numpy(), sample_rate_hz, format="WAV")
except Exception:
try:
os.unlink(prompt_wav)
except FileNotFoundError:
pass
raise
self._owned_prompt_wavs[state_id] = (prompt_cache_id, prompt_wav)
return prompt_cache_id, prompt_wav
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
return torch.zeros((input_ids.numel(), 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(self, hidden_states: Any, sampling_metadata: Any = None) -> None:
return None
@staticmethod
def _split_segments(input_ids: torch.Tensor, counts: Any) -> list[torch.Tensor]:
flat = input_ids.reshape(-1)
if counts is None:
return [flat]
if not isinstance(counts, Sequence) or isinstance(counts, (str, bytes, bytearray)):
raise _batch_error("invalid_seq_token_counts", value_type=type(counts).__name__)
normalized = [int(value) for value in counts]
if any(value < 0 for value in normalized):
raise _batch_error("negative_seq_token_count", counts=normalized)
if sum(normalized) != int(flat.numel()):
raise _batch_error(
"seq_token_count_mismatch",
counts=normalized,
total=int(flat.numel()),
)
return list(torch.split(flat, normalized))
def _parse_item(
self,
index: int,
state_id: str,
segment: torch.Tensor,
info: Mapping[str, Any],
) -> _WorkItem:
meta = info.get("meta")
if not isinstance(meta, Mapping):
meta = info
request_id = str(_scalar(meta.get("request_id"), _scalar(info.get("request_id"), "")))
if not request_id:
raise _batch_error("missing_request_id", output_index=index)
cache_epoch = int(_scalar(meta.get("cache_epoch"), 0))
chunk_seq = int(_scalar(meta.get("chunk_seq"), 0))
if cache_epoch < 0 or chunk_seq < 0:
raise _batch_error(
"negative_stream_position",
request_id=request_id,
cache_epoch=cache_epoch,
chunk_seq=chunk_seq,
)
last_chunk = bool(_scalar(meta.get("last_chunk"), False))
codes = info.get("codes")
audio = codes.get("audio") if isinstance(codes, Mapping) else None
reference = codes.get("ref") if isinstance(codes, Mapping) else None
tokens = _codec_tensor(audio, segment)
if last_chunk and int(_scalar(meta.get("code_flat_numel"), tokens.numel())) == 0:
# The generation scheduler reserves one placeholder token for an
# empty terminal chunk. The producer's explicit length is the
# authority, so do not decode that placeholder as codec data.
tokens = segment.new_empty(0, dtype=torch.long)
previous = self._states.get(state_id)
if previous is None:
if chunk_seq != 0:
raise _batch_error(
"missing_state_for_chunk",
request_id=request_id,
cache_epoch=cache_epoch,
chunk_seq=chunk_seq,
)
elif cache_epoch < previous.cache_epoch:
raise _batch_error(
"stale_cache_epoch",
request_id=request_id,
expected=previous.cache_epoch,
actual=cache_epoch,
)
elif cache_epoch > previous.cache_epoch:
if chunk_seq != 0:
raise _batch_error(
"new_epoch_requires_first_chunk",
request_id=request_id,
cache_epoch=cache_epoch,
chunk_seq=chunk_seq,
)
self._release_prompt(state_id)
previous = None
elif chunk_seq != previous.chunk_seq + 1:
raise _batch_error(
"stale_or_reordered_chunk",
request_id=request_id,
expected=previous.chunk_seq + 1,
actual=chunk_seq,
)
explicit_prompt_id = _scalar(meta.get("prompt_cache_id"))
explicit_prompt_wav = _scalar(meta.get("prompt_wav"))
if previous is not None and explicit_prompt_id is None and explicit_prompt_wav is None:
prompt_cache_id = previous.prompt_cache_id
prompt_wav = previous.prompt_wav
elif reference is not None:
prompt_cache_id, prompt_wav = self._write_reference_prompt(
state_id,
reference,
meta.get("ref_audio_sr"),
)
else:
prompt_cache_id = str(explicit_prompt_id or self._default_prompt_id)
prompt_wav = str(explicit_prompt_wav or self._default_prompt_wav)
if previous is not None and prompt_cache_id != previous.prompt_cache_id:
raise _batch_error(
"prompt_changed_midstream",
request_id=request_id,
expected=previous.prompt_cache_id,
actual=prompt_cache_id,
)
if previous is not None and prompt_wav != previous.prompt_wav:
raise _batch_error(
"prompt_changed_midstream",
request_id=request_id,
expected=previous.prompt_wav,
actual=prompt_wav,
)
duplex_turn_id = _scalar(meta.get("duplex_turn_id"))
duplex_epoch = _scalar(meta.get("duplex_epoch"))
return _WorkItem(
output_index=index,
state_id=state_id,
request_id=request_id,
cache_epoch=cache_epoch,
chunk_seq=chunk_seq,
prompt_cache_id=prompt_cache_id,
prompt_wav=prompt_wav,
last_chunk=last_chunk,
tokens=tokens,
previous=previous,
segment_text=str(_scalar(meta.get("native_duplex_segment_text"), "")),
duplex_turn_id=int(duplex_turn_id) if duplex_turn_id is not None else None,
duplex_epoch=int(duplex_epoch) if duplex_epoch is not None else None,
segment_end=bool(_scalar(meta.get("segment_end"), False)),
turn_end=bool(_scalar(meta.get("turn_end"), False)),
)
@staticmethod
def _bucket_key(item: _WorkItem) -> tuple[Any, ...]:
cache_signature: Any
if item.previous is None:
cache_signature = ("uninitialized",)
else:
cache_signature = state_shape_signature(item.previous.token2wav)
return (
item.prompt_cache_id,
item.prompt_wav,
int(item.tokens.numel()),
cache_signature,
item.last_chunk,
item.cache_epoch,
)
@torch.inference_mode()
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
del positions, intermediate_tensors, inputs_embeds
ids = input_ids if isinstance(input_ids, torch.Tensor) else torch.empty(0, dtype=torch.long)
segments = self._split_segments(ids, kwargs.get("seq_token_counts"))
empty = torch.empty(0, dtype=torch.float32, device=ids.device)
sample_rate = torch.tensor(24000, dtype=torch.int32)
if not runtime_additional_information:
count = len(segments)
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"model_outputs": [empty for _ in range(count)],
"sr": [sample_rate for _ in range(count)],
},
)
if len(runtime_additional_information) != len(segments):
raise _batch_error(
"runtime_info_count_mismatch",
segments=len(segments),
runtime_infos=len(runtime_additional_information),
)
if self.backend is None:
raise _batch_error("backend_not_loaded")
state_ids = kwargs.get("request_ids")
if state_ids is None:
state_ids = []
for index, info in enumerate(runtime_additional_information):
if not isinstance(info, Mapping):
state_ids.append(str(index))
continue
meta = info.get("meta")
source = meta if isinstance(meta, Mapping) else info
state_ids.append(str(_scalar(source.get("request_id"), index)))
if len(state_ids) != len(segments):
raise _batch_error(
"request_id_count_mismatch",
segments=len(segments),
request_ids=len(state_ids),
)
items: list[_WorkItem] = []
for index, (state_id, segment, info) in enumerate(
zip(state_ids, segments, runtime_additional_information, strict=True)
):
if not isinstance(info, Mapping):
raise _batch_error(
"invalid_runtime_info",
output_index=index,
value_type=type(info).__name__,
)
items.append(self._parse_item(index, str(state_id), segment, info))
state_ids = [item.state_id for item in items]
if len(state_ids) != len(set(state_ids)):
raise _batch_error("duplicate_request_in_forward", request_ids=state_ids)
outputs = [empty for _ in segments]
sentinels = [item for item in items if item.last_chunk and item.tokens.numel() == 0]
compute_items = [item for item in items if item.tokens.numel() > 0]
invalid_empty = [item.request_id for item in items if not item.last_chunk and item.tokens.numel() == 0]
if invalid_empty:
raise _batch_error("empty_nonfinal_chunk", request_ids=invalid_empty)
buckets: dict[tuple[Any, ...], list[_WorkItem]] = {}
for item in compute_items:
buckets.setdefault(self._bucket_key(item), []).append(item)
undersized = [
{
"size": len(bucket),
"request_ids": [item.request_id for item in bucket],
"codec_len": int(bucket[0].tokens.numel()),
}
for bucket in buckets.values()
if len(bucket) < self._min_batch_size
]
if undersized:
raise _batch_error(
"exact_shape_bucket_below_minimum",
minimum=self._min_batch_size,
buckets=undersized,
)
pending: dict[str, _RequestState | None] = {item.state_id: None for item in sentinels}
for bucket in buckets.values():
batch_size = len(bucket)
try:
features = self.backend.prepare_prompt(
bucket[0].prompt_cache_id,
bucket[0].prompt_wav,
)
if bucket[0].previous is None:
states = self.backend.setup_batch(features, batch_size)
else:
states = [item.previous.token2wav for item in bucket if item.previous is not None]
tokens = torch.stack([item.tokens for item in bucket], dim=0)
audios, next_states = self.backend.decode_batch(
tokens,
features,
states,
last_chunk=bucket[0].last_chunk,
)
except Exception as exc:
if isinstance(exc, RuntimeError) and str(exc).startswith("MiniCPMO45Code2WavBatchError "):
raise
raise _batch_error(
"backend_unsupported_or_failed",
request_ids=[item.request_id for item in bucket],
error_type=type(exc).__name__,
error=str(exc),
) from exc
if len(audios) != batch_size or len(next_states) != batch_size:
raise _batch_error(
"backend_result_size_mismatch",
expected=batch_size,
audios=len(audios),
states=len(next_states),
)
for item, audio, next_state in zip(bucket, audios, next_states, strict=True):
outputs[item.output_index] = audio.reshape(-1).to(dtype=torch.float32)
pending[item.state_id] = (
None
if item.last_chunk
else _RequestState(
cache_epoch=item.cache_epoch,
chunk_seq=item.chunk_seq,
prompt_cache_id=item.prompt_cache_id,
prompt_wav=item.prompt_wav,
token2wav=next_state,
)
)
for request_id, state in pending.items():
if state is None:
self._states.pop(request_id, None)
self._release_prompt(request_id)
else:
self._states[request_id] = state
sample_rate_tensor = torch.as_tensor(sample_rate, dtype=torch.int32)
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"model_outputs": outputs,
"sr": [sample_rate_tensor.clone() for _ in outputs],
"meta.llm_output_text_utf8": [
torch.tensor(list(item.segment_text.encode("utf-8")), dtype=torch.uint8) for item in items
],
"meta.tts_is_last_chunk": [torch.tensor(item.last_chunk, dtype=torch.bool) for item in items],
"meta.segment_end": [torch.tensor(item.segment_end, dtype=torch.bool) for item in items],
"meta.turn_end": [torch.tensor(item.turn_end, dtype=torch.bool) for item in items],
"meta.duplex_turn_id": [
torch.tensor(item.duplex_turn_id, dtype=torch.int64)
if item.duplex_turn_id is not None
else torch.empty(0, dtype=torch.int64)
for item in items
],
"meta.duplex_epoch": [
torch.tensor(item.duplex_epoch, dtype=torch.int64)
if item.duplex_epoch is not None
else torch.empty(0, dtype=torch.int64)
for item in items
],
},
)
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
for request_id in finished_req_ids:
state_id = str(request_id)
self._states.pop(state_id, None)
self._release_prompt(state_id)
def make_omni_output(self, model_outputs: Any, **_: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
return OmniOutput(*model_outputs)
raise TypeError(f"MiniCPMO45Code2Wav expected OmniOutput, got {type(model_outputs).__name__}")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
for _ in weights:
pass
if self.backend is not None:
return {name for name, _ in self.named_parameters()}
from stepaudio2.token2wav import Token2wav
extra = self._extra_config()
prompt_path = Path(self._default_prompt_wav)
if not prompt_path.is_file():
raise FileNotFoundError(f"MiniCPM-o Code2Wav prompt audio not found: {prompt_path}")
token2wav_path = Path(self.model_path) / "assets" / "token2wav"
if not token2wav_path.is_dir():
raise FileNotFoundError(f"MiniCPM-o Code2Wav assets not found: {token2wav_path}")
use_float16 = bool(extra.get("token2wav_float16", False))
previous_dtype = torch.get_default_dtype()
try:
# vLLM constructs bf16 models under a bf16 default-dtype context.
# Token2wav contains fp32-only S3Tokenizer/HiFT modules, so build
# its independent assets in their native precision.
torch.set_default_dtype(torch.float32)
token2wav = Token2wav(
str(token2wav_path),
float16=use_float16,
n_timesteps=int(extra.get("token2wav_n_timesteps", 10)),
)
finally:
torch.set_default_dtype(previous_dtype)
self.backend = BatchedToken2Wav(token2wav)
# Token2wav loads flow.pt and hift.pt inside its constructor instead of
# from the parent MiniCPM checkpoint iterator. Report those registered
# parameters as initialized so vLLM's strict loader audit does not
# misclassify the independently loaded Stage-2 weights as missing.
return {name for name, _ in self.named_parameters()}
@@ -34,7 +34,6 @@ from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key
from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy
from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
@@ -58,9 +57,11 @@ logger = init_logger(__name__)
class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE):
"""MiniCPM-o 4.5 Omni model for conditional generation.
This model has two pipeline stages:
- llm: multimodal thinker and text generation
- tts: talker generation followed by its built-in Token2Wav vocoder
Three-stage pipeline:
- thinker (model_stage="llm"): image / video / audio encoders + 3D
resampler + the omni LLM that emits text + hidden states.
- talker (model_stage="tts"): native continuous MiniCPMTTS AR that emits
codec-token deltas for the separate Code2Wav stage.
"""
@classmethod
@@ -106,7 +107,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
elif self.model_stage == "tts":
self.thinker = None
# Initialize talker model (LLM generation)
# The Talker is always the runner-owned continuous codec producer.
self.talker = init_vllm_registered_model(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "talker"),
@@ -125,12 +126,18 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
self.make_empty_intermediate_tensors = (
(self.thinker.make_empty_intermediate_tensors)
if self.model_stage == "llm" and self.thinker is not None
else self.talker.make_empty_intermediate_tensors
if self.talker is not None
else lambda: None
)
self._language_model_names = ["model"]
self.prefer_model_sampler = self.model_stage in {"llm", "tts"}
self.has_preprocess = self.model_stage == "llm"
# Both AR stages require model-specific embeddings. The Thinker uses
# preprocess for duplex audio, while the Talker converts the
# tts_token_ids/tts_hidden_states handoff into its conditioning
# embeddings and initializes request-local codec generation state.
self.has_preprocess = self.model_stage in {"llm", "tts"}
@cached_property
def sampler(self):
@@ -299,6 +306,8 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
and sampler. This hook only turns the current duplex audio append into
the prompt embeddings consumed by the normal runner forward.
"""
if self.model_stage == "tts":
return self.talker.preprocess(input_ids=input_ids, input_embeds=input_embeds, **kwargs)
if self.model_stage != "llm":
embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids)
return input_ids, embeds, {}
@@ -474,174 +483,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
"""vLLM V1 encoder profiling calls this; the inherited Protocol stub returns None."""
return self.get_multimodal_embeddings(**kwargs)
def _run_tts_request(
self,
*,
input_ids: torch.Tensor | None,
positions: torch.Tensor | None,
inputs_embeds: torch.Tensor | None,
talker_info: dict[str, Any],
device: torch.device,
) -> tuple[dict[str, Any] | None, bool, bool]:
meta_info = talker_info.get("meta") if isinstance(talker_info.get("meta"), dict) else {}
if talker_info.get("native_duplex") is True:
# ids/hidden_states may be accumulated for the talker KV stream,
# but displayed transcript belongs to the current thinker segment.
tts_text = meta_info.get("native_duplex_segment_text", "")
else:
tts_text = talker_info.get("llm_output_text", "")
if isinstance(tts_text, list):
tts_text = (
tts_text[-1]
if talker_info.get("native_duplex") is True and tts_text
else (tts_text[0] if tts_text else "")
)
if not isinstance(tts_text, str):
tts_text = ""
with torch.inference_mode():
talker_result = self.talker(
input_ids=input_ids,
positions=positions,
inputs_embeds=inputs_embeds,
additional_information=talker_info,
)
chunk_flags = getattr(self.talker, "_ar_last_chunk_flags", [True])
chunk_is_last = bool(chunk_flags[-1]) if chunk_flags else True
turn_end_flags = getattr(self.talker, "_ar_turn_end_flags", [False])
turn_ended = bool(turn_end_flags[-1]) if turn_end_flags else False
if talker_info.get("native_duplex") is True:
emitted_text = getattr(self.talker, "_ar_last_emitted_text", "")
tts_text = emitted_text if isinstance(emitted_text, str) else ""
if not (isinstance(talker_result, tuple) and len(talker_result) == 2):
return None, chunk_is_last, turn_ended
mel_spec, waveform = talker_result
mm_out: dict[str, Any] = {}
duplex_info = talker_info.get("duplex") if isinstance(talker_info.get("duplex"), dict) else {}
if talker_info.get("native_duplex") is True:
turn_id = duplex_info.get("turn_id")
if isinstance(turn_id, int):
mm_out["meta.duplex_turn_id"] = torch.tensor([turn_id], dtype=torch.int32, device=device)
epoch = duplex_info.get("epoch")
if isinstance(epoch, int):
mm_out["meta.duplex_epoch"] = torch.tensor([epoch], dtype=torch.int32, device=device)
mm_out["meta.tts_is_last_chunk"] = torch.tensor(
[int(chunk_is_last)],
dtype=torch.int32,
device=device,
)
if talker_info.get("native_duplex") is True:
mm_out["meta.turn_end"] = torch.tensor(
[int(turn_ended)],
dtype=torch.int32,
device=device,
)
if tts_text or talker_info.get("native_duplex") is True:
mm_out["meta.llm_output_text_utf8"] = torch.tensor(
list(tts_text.encode("utf-8")),
dtype=torch.uint8,
device=device,
)
mm_out["meta.audio_text_total_chars"] = torch.tensor(
[len(tts_text)],
dtype=torch.int32,
device=device,
)
if mel_spec is not None:
mm_out["mel_spec"] = [mel_spec]
if waveform is not None:
mm_out["model_outputs"] = [waveform]
elif mel_spec is not None:
mm_out["model_outputs"] = [mel_spec]
return mm_out, chunk_is_last, turn_ended
@staticmethod
def _slice_tts_request_rows(value: torch.Tensor | None, start: int, end: int) -> torch.Tensor | None:
return value[start:end] if isinstance(value, torch.Tensor) else value
def _run_batched_tts_requests(
self,
*,
input_ids: torch.Tensor | None,
positions: torch.Tensor | None,
inputs_embeds: torch.Tensor | None,
runtime_info: list[object],
request_token_spans: object,
num_tokens: int,
hidden_dim: int,
device: torch.device,
) -> OmniOutput:
if not isinstance(request_token_spans, list) or len(request_token_spans) != len(runtime_info):
raise RuntimeError(
"MiniCPM-o 4.5 batched TTS requires one request_token_spans entry "
f"per request; got {request_token_spans!r} for {len(runtime_info)} requests"
)
request_ids: list[str] = []
request_outputs: list[dict[str, Any]] = []
row_chunk_flags = [True] * num_tokens
row_turn_end_flags = [False] * num_tokens
previous_end = 0
for index, raw_info in enumerate(runtime_info):
if not isinstance(raw_info, dict):
raise RuntimeError(f"MiniCPM-o 4.5 TTS request metadata at index {index} is not a dict")
span = request_token_spans[index]
if not (
isinstance(span, (list, tuple)) and len(span) == 2 and all(isinstance(value, int) for value in span)
):
raise RuntimeError(f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: {span!r}")
start, end = span
if start != previous_end or start < 0 or end < start or end > num_tokens:
raise RuntimeError(
f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: "
f"expected start {previous_end} and 0 <= start <= end <= {num_tokens}, got {span!r}"
)
previous_end = end
if start == end:
continue
mm_out, chunk_is_last, turn_ended = self._run_tts_request(
input_ids=self._slice_tts_request_rows(input_ids, start, end),
positions=self._slice_tts_request_rows(positions, start, end),
inputs_embeds=self._slice_tts_request_rows(inputs_embeds, start, end),
talker_info=raw_info,
device=device,
)
row_chunk_flags[start:end] = [chunk_is_last] * (end - start)
row_turn_end_flags[start:end] = [turn_ended] * (end - start)
if mm_out is not None:
request_ids.append(get_stream_request_key(raw_info))
request_outputs.append(mm_out)
# vLLM pads the flattened token tensor to a CUDA Graph capture size.
# The spans describe only scheduled request rows, so an uncovered tail
# is graph padding rather than a missing request. Contiguity checks
# above still reject gaps between real requests.
self.talker._ar_last_chunk_flags = row_chunk_flags
self.talker._ar_turn_end_flags = row_turn_end_flags
dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
if not request_outputs:
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
sparse_output: dict[str, Any] = {
"meta.req_id": request_ids,
"meta.sparse_audio": ["1"],
}
output_keys = set().union(*(output.keys() for output in request_outputs))
for key in output_keys:
values: list[Any] = []
for output in request_outputs:
value = output.get(key)
if isinstance(value, list) and len(value) == 1:
value = value[0]
if value is None:
dtype = torch.uint8 if key == "meta.llm_output_text_utf8" else torch.float32
value = torch.empty(0, dtype=dtype, device=device)
values.append(value)
sparse_output[key] = values
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=sparse_output)
def forward(
self,
input_ids: torch.Tensor,
@@ -658,8 +499,9 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
Forward pass for MiniCPM-o Omni model.
Workflow:
1) LLM: multimodal thinker → hidden states and text tokens
2) TTS: talker + Token2Wav → speech waveform
1) Thinker (model_stage="llm"): Image / video / audio encoders +
3D resampler + omni LLM → text + hidden states.
2) Talker (model_stage="tts"): native MiniCPMTTS AR → codec deltas.
"""
if self.model_stage == "llm":
# Normalize to batched inputs if caller provides 1D/2D unbatched tensors
@@ -778,57 +620,24 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
multimodal_outputs=multimodal_outputs,
)
# Talker stage: runs TTS generation and its built-in Token2Wav vocoder.
# Talker stage: runner-owned native AR only. Waveform generation belongs
# to the separate Code2Wav stage.
if self.model_stage == "tts":
if input_ids is not None:
num_tokens = input_ids.shape[0]
device = input_ids.device
elif inputs_embeds is not None:
num_tokens = inputs_embeds.shape[0]
device = inputs_embeds.device
else:
num_tokens = 1
device = current_omni_platform.get_torch_device()
hidden_dim = self.config.hidden_size if hasattr(self.config, "hidden_size") else 2560
# Profile/dummy run: both input_ids and inputs_embeds are None.
# Note: SupportsMultiModal preprocessing converts input_ids to
# inputs_embeds, so input_ids=None alone does NOT indicate a dummy run.
if input_ids is None and inputs_embeds is None:
dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
runtime_info = kwargs.get("runtime_additional_information")
request_token_spans = kwargs.get("request_token_spans")
if isinstance(runtime_info, list) and (len(runtime_info) > 1 or request_token_spans is not None):
return self._run_batched_tts_requests(
input_ids=input_ids,
positions=positions,
inputs_embeds=inputs_embeds,
runtime_info=runtime_info,
request_token_spans=request_token_spans,
num_tokens=num_tokens,
hidden_dim=hidden_dim,
device=device,
)
talker_info = {}
if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0:
talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {}
dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
mm_out, _, _ = self._run_tts_request(
return self.talker(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
talker_info=talker_info,
device=device,
**kwargs,
)
if mm_out is not None:
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=mm_out)
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
raise ValueError(f"Unsupported model stage: {self.model_stage}")
def make_omni_output(self, model_outputs, **kwargs):
if self.model_stage != "tts":
return model_outputs
return self.talker.make_omni_output(model_outputs, **kwargs)
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
# Handle OmniOutput type
if isinstance(hidden_states, OmniOutput):
@@ -1261,7 +1070,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
# MiniCPM-o checkpoint prefixes → stage mapping:
# thinker: vpm, resampler, llm, apm, audio_projection_layer
# talker: tts (ConditionalChatTTS)
# talker: tts (native MiniCPMTTS AR codec producer)
for k, v in weights:
if k.startswith(("vpm.", "resampler.", "llm.", "apm.", "audio_projection_layer.")):
thinker_weights.append((k, v))
File diff suppressed because it is too large. Load diff
@@ -3,12 +3,11 @@
"""MiniCPM-o 4.5 pipeline topology (frozen).
Stage 0: Thinker — multimodal understanding + text generation.
Stage 1: Talker — MiniCPMTTS + Token2Wav, emits the final audio waveform.
Stage 1: Talker — MiniCPMTTS, emits codec tokens.
Stage 2: Code2Wav — codec tokens to the final audio waveform.
The thinker -> talker bridge passes the hidden states + token ids extracted
from the thinker output through ``minicpmo_4_5_omni.llm2tts``; the talker
runs MiniCPMTTS and the on-device Token2wav vocoder in the same process and
returns the waveform directly as the pipeline's final audio output.
The thinker -> talker bridge uses ``llm2tts``. The talker -> Code2Wav bridge
streams request-routed codec chunks.
"""
from vllm_omni.config.stage_config import (
@@ -56,28 +55,24 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
StagePipelineConfig(
stage_id=1,
model_stage="tts",
# Stage 1 shares the top-level wrapper class
# (``MiniCPMO45OmniForConditionalGeneration``) inherited from
# ``model_arch`` above. The wrapper dispatches on ``model_stage``
# and, for ``"tts"``, instantiates the standalone TTS submodule
# (``MiniCPMO45OmniTTSForConditionalGeneration``) internally.
# Routing through the wrapper is required so that the runner-side
# ``runtime_additional_information`` payload reaches the talker
# (the standalone TTS class only reads ``additional_information``,
# so wiring stage 1 directly to it would always trigger the dummy
# path) and so the resulting waveform is packaged as
# ``OmniOutput.multimodal_outputs["model_outputs"]`` instead of
# being returned as a bare tuple that the AR runner would mistake
# for hidden states. ``hf_config_name="tts_config"`` keeps KV
# cache / mrope sizing scoped to the talker sub-config.
execution_type=StageExecutionType.LLM_AR,
input_sources=(0,),
hf_config_name="tts_config",
engine_output_type="latent",
custom_process_input_func=f"{_PROC}.llm2tts",
async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk",
sampling_constraints={"detokenize": False},
),
StagePipelineConfig(
stage_id=2,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(1,),
final_output=True,
final_output_type="audio",
hf_config_name="tts_config",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.llm2tts",
sampling_constraints={"detokenize": False},
model_arch="MiniCPMO45Code2Wav",
sampling_constraints={"detokenize": True},
),
),
)
@@ -360,6 +360,11 @@ _OMNI_MODELS = {
"minicpmo_4_5_omni_tts",
"MiniCPMO45OmniTTSForConditionalGeneration",
),
"MiniCPMO45Code2Wav": (
"minicpmo_4_5",
"minicpmo_4_5_code2wav",
"MiniCPMO45Code2Wav",
),
"AuraQwen3VLForConditionalGeneration": (
"aura_omni",
"qwen3_vl",