[perf] Minicpm o4.5 optimization (#5228)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
acf941da4d
commit
ad3a162eb8
54 files changed
+3861
-3997
No files matched your search
@@ -0,0 +1,400 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Strict, state-explicit batching for MiniCPM-o 4.5 Token2wav."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
_SILENCE_TOKEN = 4218
|
||||
|
||||
|
||||
def tensor_signature(value: torch.Tensor) -> tuple[tuple[int, ...], str, str]:
|
||||
return tuple(value.shape), str(value.dtype), value.device.type
|
||||
|
||||
|
||||
def state_shape_signature(state: BatchedToken2WavState) -> tuple[Any, ...]:
|
||||
flow = tuple((name, tensor_signature(state.flow_cache[name])) for name in sorted(state.flow_cache))
|
||||
hift = tuple((name, tensor_signature(state.hift_cache[name])) for name in sorted(state.hift_cache))
|
||||
return flow, hift
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PromptFeatures:
|
||||
speech_tokens: torch.Tensor
|
||||
speaker_embedding: torch.Tensor
|
||||
mels: torch.Tensor
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchedToken2WavState:
|
||||
flow_cache: dict[str, torch.Tensor]
|
||||
hift_cache: dict[str, torch.Tensor]
|
||||
|
||||
|
||||
class BatchedToken2Wav(nn.Module):
|
||||
"""Drive Token2wav's modules with dynamically-sized, request-owned caches.
|
||||
|
||||
This class intentionally never calls ``Token2wav.stream`` or
|
||||
``Token2wav.__call__``. The upstream object is used only as a one-time
|
||||
asset loader and prompt feature extractor.
|
||||
"""
|
||||
|
||||
def __init__(self, token2wav: Any):
|
||||
super().__init__()
|
||||
self._token2wav = token2wav
|
||||
self.flow = token2wav.flow
|
||||
self.hift = token2wav.hift
|
||||
self.float16 = bool(token2wav.float16)
|
||||
self.n_timesteps = int(token2wav.n_timesteps)
|
||||
self.mel_cache_len = int(token2wav.mel_cache_len)
|
||||
self.source_cache_len = int(token2wav.source_cache_len)
|
||||
self.register_buffer(
|
||||
"speech_window",
|
||||
token2wav.speech_window.detach().clone(),
|
||||
persistent=False,
|
||||
)
|
||||
self._prompt_features: dict[tuple[str, str], PromptFeatures] = {}
|
||||
|
||||
def prepare_prompt(self, prompt_cache_id: str, prompt_wav: str) -> PromptFeatures:
|
||||
cache_key = (prompt_cache_id, prompt_wav)
|
||||
cached = self._prompt_features.get(cache_key)
|
||||
if cached is None:
|
||||
# The generation runner may wrap model.forward in bf16 autocast,
|
||||
# and vLLM constructs the model under a bf16 default dtype, while
|
||||
# S3Tokenizer prompt extraction uses fp32 convolution weights.
|
||||
previous_dtype = torch.get_default_dtype()
|
||||
try:
|
||||
torch.set_default_dtype(torch.float32)
|
||||
with torch.amp.autocast("cuda", enabled=False):
|
||||
values = self._token2wav._prepare_prompt(prompt_wav)
|
||||
finally:
|
||||
torch.set_default_dtype(previous_dtype)
|
||||
cached = PromptFeatures(
|
||||
speech_tokens=values[0],
|
||||
speaker_embedding=values[2],
|
||||
mels=values[3],
|
||||
)
|
||||
self._prompt_features[cache_key] = cached
|
||||
return cached
|
||||
|
||||
def evict_prompt(self, prompt_cache_id: str, prompt_wav: str) -> None:
|
||||
"""Release request-owned prompt features after stream completion."""
|
||||
self._prompt_features.pop((prompt_cache_id, prompt_wav), None)
|
||||
|
||||
@staticmethod
|
||||
def _repeat_prompt(features: PromptFeatures, batch_size: int) -> tuple[torch.Tensor, ...]:
|
||||
return (
|
||||
features.speech_tokens.expand(batch_size, -1),
|
||||
features.speaker_embedding.expand(batch_size, -1),
|
||||
features.mels.expand(batch_size, -1, -1),
|
||||
)
|
||||
|
||||
def _autocast(self, device: torch.device):
|
||||
if device.type != "cuda":
|
||||
return nullcontext()
|
||||
if not self.float16:
|
||||
return torch.amp.autocast("cuda", enabled=False)
|
||||
return torch.amp.autocast(
|
||||
"cuda",
|
||||
dtype=torch.float16,
|
||||
)
|
||||
|
||||
def _encode_chunk(
|
||||
self,
|
||||
tokens: torch.Tensor,
|
||||
*,
|
||||
last_chunk: bool,
|
||||
cnn_cache: torch.Tensor | None,
|
||||
att_cache: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
embedded = self.flow.input_embedding(tokens)
|
||||
hidden, new_cnn, new_att = self.flow.encoder.forward_chunk(
|
||||
xs=embedded,
|
||||
last_chunk=last_chunk,
|
||||
cnn_cache=cnn_cache,
|
||||
att_cache=att_cache,
|
||||
)
|
||||
return self.flow.encoder_proj(hidden), new_cnn, new_att
|
||||
|
||||
@staticmethod
|
||||
def _estimator_buffers(
|
||||
estimator: nn.Module,
|
||||
x: torch.Tensor,
|
||||
old_att: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
blocks = estimator.blocks
|
||||
depth = len(blocks)
|
||||
batch_size = int(x.shape[0])
|
||||
chunk_size = int(x.shape[2])
|
||||
old_att_len = int(old_att.shape[3]) if old_att is not None else 0
|
||||
block0 = blocks[0]
|
||||
cnn_channels = int(block0.conv.in_channels + block0.conv.out_channels)
|
||||
cnn_width = int(block0.conv.block[1].causal_padding[0])
|
||||
heads = int(block0.attn.num_heads)
|
||||
att_width = int(block0.attn.head_dim * 2)
|
||||
cnn = x.new_empty((depth, batch_size, cnn_channels, cnn_width))
|
||||
att = x.new_empty((depth, batch_size, heads, old_att_len + chunk_size, att_width))
|
||||
return cnn, att
|
||||
|
||||
def _estimator_step(
|
||||
self,
|
||||
estimator: nn.Module,
|
||||
*,
|
||||
x: torch.Tensor,
|
||||
mu: torch.Tensor,
|
||||
time: torch.Tensor,
|
||||
speakers: torch.Tensor,
|
||||
cond: torch.Tensor,
|
||||
cnn_cache: torch.Tensor | None,
|
||||
att_cache: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
time_embedding = estimator.t_embedder(time).unsqueeze(1)
|
||||
width = int(x.shape[-1])
|
||||
speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width)
|
||||
estimator_input = torch.cat((x, mu, speaker_features, cond), dim=1)
|
||||
cnn_out, att_out = self._estimator_buffers(estimator, estimator_input, att_cache)
|
||||
old_cnn: Any = cnn_cache if cnn_cache is not None else [None] * len(estimator.blocks)
|
||||
old_att: Any = att_cache if att_cache is not None else [None] * len(estimator.blocks)
|
||||
result = estimator.blocks_forward_chunk(
|
||||
estimator_input,
|
||||
time_embedding,
|
||||
None,
|
||||
old_cnn,
|
||||
old_att,
|
||||
cnn_out,
|
||||
att_out,
|
||||
)
|
||||
return result, cnn_out, att_out
|
||||
|
||||
def _decode_cfm(
|
||||
self,
|
||||
mu: torch.Tensor,
|
||||
speakers: torch.Tensor,
|
||||
cond: torch.Tensor,
|
||||
*,
|
||||
cnn_cache: torch.Tensor | None,
|
||||
att_cache: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
decoder = self.flow.decoder
|
||||
estimator = decoder.estimator
|
||||
batch_size = int(mu.shape[0])
|
||||
offset = int(att_cache.shape[4]) if att_cache is not None else 0
|
||||
end = offset + int(mu.shape[2])
|
||||
if end > int(decoder.rand_noise.shape[2]):
|
||||
raise RuntimeError(
|
||||
"MiniCPMO45Code2WavBatchError "
|
||||
f'{{"reason":"noise_capacity","required":{end},'
|
||||
f'"available":{int(decoder.rand_noise.shape[2])}}}'
|
||||
)
|
||||
x = decoder.rand_noise[:, :, offset:end].expand(batch_size, -1, -1).clone()
|
||||
timeline = torch.linspace(
|
||||
0,
|
||||
1,
|
||||
self.n_timesteps + 1,
|
||||
device=mu.device,
|
||||
dtype=mu.dtype,
|
||||
)
|
||||
timeline = 1 - torch.cos(timeline * 0.5 * torch.pi)
|
||||
time = timeline[0].expand(batch_size)
|
||||
mu_cfg = torch.cat((mu, torch.zeros_like(mu)), dim=0)
|
||||
speakers_cfg = torch.cat((speakers, torch.zeros_like(speakers)), dim=0)
|
||||
cond_cfg = torch.cat((cond, torch.zeros_like(cond)), dim=0)
|
||||
next_cnn: list[torch.Tensor] = []
|
||||
next_att: list[torch.Tensor] = []
|
||||
dt = timeline[1] - timeline[0]
|
||||
for step in range(self.n_timesteps):
|
||||
old_cnn = cnn_cache[step] if cnn_cache is not None else None
|
||||
old_att = att_cache[step] if att_cache is not None else None
|
||||
estimate, step_cnn, step_att = self._estimator_step(
|
||||
estimator,
|
||||
x=torch.cat((x, x), dim=0),
|
||||
mu=mu_cfg,
|
||||
time=torch.cat((time, time), dim=0),
|
||||
speakers=speakers_cfg,
|
||||
cond=cond_cfg,
|
||||
cnn_cache=old_cnn,
|
||||
att_cache=old_att,
|
||||
)
|
||||
conditional, unconditional = estimate.split(batch_size, dim=0)
|
||||
velocity = (1.0 + decoder.inference_cfg_rate) * conditional - decoder.inference_cfg_rate * unconditional
|
||||
x = x + dt * velocity
|
||||
time = time + dt
|
||||
if step + 1 < self.n_timesteps:
|
||||
dt = timeline[step + 2] - time[0]
|
||||
next_cnn.append(step_cnn)
|
||||
next_att.append(step_att)
|
||||
return x, torch.stack(next_cnn), torch.stack(next_att)
|
||||
|
||||
@staticmethod
|
||||
def _split_flow_cache(cache: dict[str, torch.Tensor], batch_size: int) -> list[dict[str, torch.Tensor]]:
|
||||
result: list[dict[str, torch.Tensor]] = []
|
||||
for row in range(batch_size):
|
||||
result.append(
|
||||
{
|
||||
"conformer_cnn_cache": cache["conformer_cnn_cache"][row : row + 1].detach().clone(),
|
||||
"conformer_att_cache": cache["conformer_att_cache"][:, row : row + 1].detach().clone(),
|
||||
"estimator_cnn_cache": torch.cat(
|
||||
(
|
||||
cache["estimator_cnn_cache"][:, :, row : row + 1],
|
||||
cache["estimator_cnn_cache"][:, :, batch_size + row : batch_size + row + 1],
|
||||
),
|
||||
dim=2,
|
||||
).detach(),
|
||||
"estimator_att_cache": torch.cat(
|
||||
(
|
||||
cache["estimator_att_cache"][:, :, row : row + 1],
|
||||
cache["estimator_att_cache"][:, :, batch_size + row : batch_size + row + 1],
|
||||
),
|
||||
dim=2,
|
||||
).detach(),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _stack_flow_cache(states: list[BatchedToken2WavState]) -> dict[str, torch.Tensor]:
|
||||
flows = [state.flow_cache for state in states]
|
||||
conditional_cnn = [flow["estimator_cnn_cache"][:, :, 0:1] for flow in flows]
|
||||
unconditional_cnn = [flow["estimator_cnn_cache"][:, :, 1:2] for flow in flows]
|
||||
conditional_att = [flow["estimator_att_cache"][:, :, 0:1] for flow in flows]
|
||||
unconditional_att = [flow["estimator_att_cache"][:, :, 1:2] for flow in flows]
|
||||
return {
|
||||
"conformer_cnn_cache": torch.cat([flow["conformer_cnn_cache"] for flow in flows], dim=0),
|
||||
"conformer_att_cache": torch.cat([flow["conformer_att_cache"] for flow in flows], dim=1),
|
||||
"estimator_cnn_cache": torch.cat((*conditional_cnn, *unconditional_cnn), dim=2),
|
||||
"estimator_att_cache": torch.cat((*conditional_att, *unconditional_att), dim=2),
|
||||
}
|
||||
|
||||
def setup_batch(
|
||||
self,
|
||||
features: PromptFeatures,
|
||||
batch_size: int,
|
||||
) -> list[BatchedToken2WavState]:
|
||||
prompt_tokens, speakers, prompt_mels = self._repeat_prompt(features, batch_size)
|
||||
lookahead = prompt_tokens.new_full((batch_size, 3), _SILENCE_TOKEN)
|
||||
with self._autocast(prompt_tokens.device):
|
||||
hidden, conformer_cnn, conformer_att = self._encode_chunk(
|
||||
torch.cat((prompt_tokens, lookahead), dim=1),
|
||||
last_chunk=False,
|
||||
cnn_cache=None,
|
||||
att_cache=None,
|
||||
)
|
||||
projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1))
|
||||
_, estimator_cnn, estimator_att = self._decode_cfm(
|
||||
hidden.transpose(1, 2).contiguous(),
|
||||
projected_speakers,
|
||||
prompt_mels.transpose(1, 2).contiguous(),
|
||||
cnn_cache=None,
|
||||
att_cache=None,
|
||||
)
|
||||
flow_cache = {
|
||||
"conformer_cnn_cache": conformer_cnn,
|
||||
"conformer_att_cache": conformer_att,
|
||||
"estimator_cnn_cache": estimator_cnn,
|
||||
"estimator_att_cache": estimator_att,
|
||||
}
|
||||
split = self._split_flow_cache(flow_cache, batch_size)
|
||||
mel_channels = int(prompt_mels.shape[2])
|
||||
return [
|
||||
BatchedToken2WavState(
|
||||
flow_cache=row,
|
||||
hift_cache={
|
||||
"mel": prompt_mels.new_zeros((1, mel_channels, 0)),
|
||||
"source": prompt_mels.new_zeros((1, 1, 0)),
|
||||
"speech": prompt_mels.new_zeros((1, 0)),
|
||||
},
|
||||
)
|
||||
for row in split
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _fade_in_out(
|
||||
speech: torch.Tensor,
|
||||
previous: torch.Tensor,
|
||||
window: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
overlap = int(window.shape[0] // 2)
|
||||
result = speech.clone()
|
||||
result[..., :overlap] = result[..., :overlap] * window[:overlap] + previous[..., -overlap:] * window[overlap:]
|
||||
return result
|
||||
|
||||
def decode_batch(
|
||||
self,
|
||||
tokens: torch.Tensor,
|
||||
features: PromptFeatures,
|
||||
states: list[BatchedToken2WavState],
|
||||
*,
|
||||
last_chunk: bool,
|
||||
) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]:
|
||||
batch_size = int(tokens.shape[0])
|
||||
if batch_size != len(states):
|
||||
raise ValueError(f"tokens batch {batch_size} != state batch {len(states)}")
|
||||
flow_cache = self._stack_flow_cache(states)
|
||||
speakers = features.speaker_embedding.expand(batch_size, -1)
|
||||
with self._autocast(tokens.device):
|
||||
hidden, conformer_cnn, conformer_att = self._encode_chunk(
|
||||
tokens,
|
||||
last_chunk=last_chunk,
|
||||
cnn_cache=flow_cache["conformer_cnn_cache"],
|
||||
att_cache=flow_cache["conformer_att_cache"],
|
||||
)
|
||||
projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1))
|
||||
cond = torch.zeros_like(hidden).transpose(1, 2).contiguous()
|
||||
chunk_mel, estimator_cnn, estimator_att = self._decode_cfm(
|
||||
hidden.transpose(1, 2).contiguous(),
|
||||
projected_speakers,
|
||||
cond,
|
||||
cnn_cache=flow_cache["estimator_cnn_cache"],
|
||||
att_cache=flow_cache["estimator_att_cache"],
|
||||
)
|
||||
|
||||
prompt_len = int(features.mels.shape[1])
|
||||
if estimator_att.shape[4] > prompt_len + 100:
|
||||
estimator_att = torch.cat(
|
||||
(estimator_att[..., :prompt_len, :], estimator_att[..., -100:, :]),
|
||||
dim=4,
|
||||
)
|
||||
if conformer_att.shape[3] > prompt_len + 100:
|
||||
conformer_att = torch.cat(
|
||||
(conformer_att[..., :prompt_len, :], conformer_att[..., -100:, :]),
|
||||
dim=3,
|
||||
)
|
||||
new_flow = self._split_flow_cache(
|
||||
{
|
||||
"conformer_cnn_cache": conformer_cnn,
|
||||
"conformer_att_cache": conformer_att,
|
||||
"estimator_cnn_cache": estimator_cnn,
|
||||
"estimator_att_cache": estimator_att,
|
||||
},
|
||||
batch_size,
|
||||
)
|
||||
old_mel = torch.cat([state.hift_cache["mel"] for state in states], dim=0)
|
||||
old_source = torch.cat([state.hift_cache["source"] for state in states], dim=0)
|
||||
old_speech = torch.cat([state.hift_cache["speech"] for state in states], dim=0)
|
||||
mel = torch.cat((old_mel, chunk_mel), dim=2)
|
||||
speech, source = self.hift(mel, old_source)
|
||||
if old_speech.shape[-1] > 0:
|
||||
window = self.speech_window.to(device=speech.device, dtype=speech.dtype)
|
||||
speech = self._fade_in_out(speech, old_speech, window)
|
||||
next_hift = {
|
||||
"mel": mel[..., -self.mel_cache_len :].detach(),
|
||||
"source": source[..., -self.source_cache_len :].detach(),
|
||||
"speech": speech[..., -self.source_cache_len :].detach(),
|
||||
}
|
||||
emitted = speech if last_chunk else speech[..., : -self.source_cache_len]
|
||||
next_states = [
|
||||
BatchedToken2WavState(
|
||||
flow_cache=new_flow[row],
|
||||
hift_cache={name: value[row : row + 1].detach().clone() for name, value in next_hift.items()},
|
||||
)
|
||||
for row in range(batch_size)
|
||||
]
|
||||
audios = [emitted[row].reshape(-1).to(dtype=torch.float32) for row in range(batch_size)]
|
||||
return audios, next_states
|
||||
@@ -0,0 +1,547 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Strict batched codec-to-waveform stage for MiniCPM-o 4.5."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
from .batched_token2wav import (
|
||||
BatchedToken2Wav,
|
||||
BatchedToken2WavState,
|
||||
state_shape_signature,
|
||||
)
|
||||
|
||||
|
||||
def _batch_error(reason: str, **details: Any) -> RuntimeError:
|
||||
payload = {"reason": reason, **details}
|
||||
return RuntimeError(f"MiniCPMO45Code2WavBatchError {json.dumps(payload, sort_keys=True)}")
|
||||
|
||||
|
||||
def _scalar(value: Any, default: Any = None) -> Any:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.reshape(-1)[0].item() if value.numel() else default
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return _scalar(value[0], default) if value else default
|
||||
return default if value is None else value
|
||||
|
||||
|
||||
def _codec_tensor(value: Any, fallback: torch.Tensor) -> torch.Tensor:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.reshape(-1).to(device=fallback.device, dtype=torch.long)
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return torch.as_tensor(value, device=fallback.device, dtype=torch.long).reshape(-1)
|
||||
return fallback.reshape(-1).to(dtype=torch.long)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RequestState:
|
||||
cache_epoch: int
|
||||
chunk_seq: int
|
||||
prompt_cache_id: str
|
||||
prompt_wav: str
|
||||
token2wav: BatchedToken2WavState
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _WorkItem:
|
||||
output_index: int
|
||||
state_id: str
|
||||
request_id: str
|
||||
cache_epoch: int
|
||||
chunk_seq: int
|
||||
prompt_cache_id: str
|
||||
prompt_wav: str
|
||||
last_chunk: bool
|
||||
tokens: torch.Tensor
|
||||
previous: _RequestState | None
|
||||
segment_text: str
|
||||
duplex_turn_id: int | None
|
||||
duplex_epoch: int | None
|
||||
segment_end: bool
|
||||
turn_end: bool
|
||||
|
||||
|
||||
class MiniCPMO45Code2Wav(nn.Module):
|
||||
"""LLM_GENERATION model that admits only true exact-shape GPU batches."""
|
||||
|
||||
input_modalities = "audio"
|
||||
have_multimodal_outputs = True
|
||||
enable_update_additional_information = True
|
||||
requires_raw_input_tokens = True
|
||||
requires_request_ids = True
|
||||
has_preprocess = False
|
||||
has_postprocess = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
del prefix
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = str(vllm_config.model_config.model)
|
||||
self.backend: BatchedToken2Wav | None = None
|
||||
self._states: dict[str, _RequestState] = {}
|
||||
self._owned_prompt_wavs: dict[str, tuple[str, str]] = {}
|
||||
extra = self._extra_config()
|
||||
self._min_batch_size = int(extra.get("code2wav_min_batch_size", 1))
|
||||
if self._min_batch_size < 1:
|
||||
raise ValueError("MiniCPM-o Code2Wav code2wav_min_batch_size must be >= 1")
|
||||
self._default_prompt_id = str(extra.get("prompt_cache_id", "HT_ref_audio"))
|
||||
self._default_prompt_wav = str(
|
||||
extra.get(
|
||||
"prompt_wav",
|
||||
Path(self.model_path) / "assets" / "HT_ref_audio.wav",
|
||||
)
|
||||
)
|
||||
|
||||
def _extra_config(self) -> dict[str, Any]:
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
connector = getattr(model_config, "stage_connector_config", None)
|
||||
if isinstance(connector, Mapping):
|
||||
extra = connector.get("extra", connector)
|
||||
else:
|
||||
extra = getattr(connector, "extra", None)
|
||||
return dict(extra) if isinstance(extra, Mapping) else {}
|
||||
|
||||
def _release_prompt(self, state_id: str) -> None:
|
||||
owned = self._owned_prompt_wavs.pop(state_id, None)
|
||||
if owned is None:
|
||||
return
|
||||
prompt_cache_id, prompt_wav = owned
|
||||
if self.backend is not None:
|
||||
self.backend.evict_prompt(prompt_cache_id, prompt_wav)
|
||||
try:
|
||||
os.unlink(prompt_wav)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def _write_reference_prompt(
|
||||
self,
|
||||
state_id: str,
|
||||
reference: object,
|
||||
sample_rate: object,
|
||||
) -> tuple[str, str]:
|
||||
waveform = torch.as_tensor(reference, dtype=torch.float32).reshape(-1).cpu()
|
||||
if waveform.numel() == 0:
|
||||
return self._default_prompt_id, self._default_prompt_wav
|
||||
sample_rate_hz = int(_scalar(sample_rate, 24000))
|
||||
if sample_rate_hz <= 0:
|
||||
raise _batch_error(
|
||||
"invalid_ref_audio_sample_rate",
|
||||
request_id=state_id,
|
||||
sample_rate=sample_rate_hz,
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
digest.update(str(sample_rate_hz).encode("ascii"))
|
||||
digest.update(waveform.numpy().tobytes())
|
||||
prompt_cache_id = f"{state_id}:{digest.hexdigest()}"
|
||||
current = self._owned_prompt_wavs.get(state_id)
|
||||
if current is not None and current[0] == prompt_cache_id and os.path.exists(current[1]):
|
||||
return current
|
||||
|
||||
self._release_prompt(state_id)
|
||||
handle = tempfile.NamedTemporaryFile(prefix="minicpmo45-ref-", suffix=".wav", delete=False)
|
||||
prompt_wav = handle.name
|
||||
handle.close()
|
||||
try:
|
||||
sf.write(prompt_wav, waveform.numpy(), sample_rate_hz, format="WAV")
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(prompt_wav)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
self._owned_prompt_wavs[state_id] = (prompt_cache_id, prompt_wav)
|
||||
return prompt_cache_id, prompt_wav
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
return torch.zeros((input_ids.numel(), 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def compute_logits(self, hidden_states: Any, sampling_metadata: Any = None) -> None:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _split_segments(input_ids: torch.Tensor, counts: Any) -> list[torch.Tensor]:
|
||||
flat = input_ids.reshape(-1)
|
||||
if counts is None:
|
||||
return [flat]
|
||||
if not isinstance(counts, Sequence) or isinstance(counts, (str, bytes, bytearray)):
|
||||
raise _batch_error("invalid_seq_token_counts", value_type=type(counts).__name__)
|
||||
normalized = [int(value) for value in counts]
|
||||
if any(value < 0 for value in normalized):
|
||||
raise _batch_error("negative_seq_token_count", counts=normalized)
|
||||
if sum(normalized) != int(flat.numel()):
|
||||
raise _batch_error(
|
||||
"seq_token_count_mismatch",
|
||||
counts=normalized,
|
||||
total=int(flat.numel()),
|
||||
)
|
||||
return list(torch.split(flat, normalized))
|
||||
|
||||
def _parse_item(
|
||||
self,
|
||||
index: int,
|
||||
state_id: str,
|
||||
segment: torch.Tensor,
|
||||
info: Mapping[str, Any],
|
||||
) -> _WorkItem:
|
||||
meta = info.get("meta")
|
||||
if not isinstance(meta, Mapping):
|
||||
meta = info
|
||||
request_id = str(_scalar(meta.get("request_id"), _scalar(info.get("request_id"), "")))
|
||||
if not request_id:
|
||||
raise _batch_error("missing_request_id", output_index=index)
|
||||
cache_epoch = int(_scalar(meta.get("cache_epoch"), 0))
|
||||
chunk_seq = int(_scalar(meta.get("chunk_seq"), 0))
|
||||
if cache_epoch < 0 or chunk_seq < 0:
|
||||
raise _batch_error(
|
||||
"negative_stream_position",
|
||||
request_id=request_id,
|
||||
cache_epoch=cache_epoch,
|
||||
chunk_seq=chunk_seq,
|
||||
)
|
||||
last_chunk = bool(_scalar(meta.get("last_chunk"), False))
|
||||
codes = info.get("codes")
|
||||
audio = codes.get("audio") if isinstance(codes, Mapping) else None
|
||||
reference = codes.get("ref") if isinstance(codes, Mapping) else None
|
||||
tokens = _codec_tensor(audio, segment)
|
||||
if last_chunk and int(_scalar(meta.get("code_flat_numel"), tokens.numel())) == 0:
|
||||
# The generation scheduler reserves one placeholder token for an
|
||||
# empty terminal chunk. The producer's explicit length is the
|
||||
# authority, so do not decode that placeholder as codec data.
|
||||
tokens = segment.new_empty(0, dtype=torch.long)
|
||||
previous = self._states.get(state_id)
|
||||
if previous is None:
|
||||
if chunk_seq != 0:
|
||||
raise _batch_error(
|
||||
"missing_state_for_chunk",
|
||||
request_id=request_id,
|
||||
cache_epoch=cache_epoch,
|
||||
chunk_seq=chunk_seq,
|
||||
)
|
||||
elif cache_epoch < previous.cache_epoch:
|
||||
raise _batch_error(
|
||||
"stale_cache_epoch",
|
||||
request_id=request_id,
|
||||
expected=previous.cache_epoch,
|
||||
actual=cache_epoch,
|
||||
)
|
||||
elif cache_epoch > previous.cache_epoch:
|
||||
if chunk_seq != 0:
|
||||
raise _batch_error(
|
||||
"new_epoch_requires_first_chunk",
|
||||
request_id=request_id,
|
||||
cache_epoch=cache_epoch,
|
||||
chunk_seq=chunk_seq,
|
||||
)
|
||||
self._release_prompt(state_id)
|
||||
previous = None
|
||||
elif chunk_seq != previous.chunk_seq + 1:
|
||||
raise _batch_error(
|
||||
"stale_or_reordered_chunk",
|
||||
request_id=request_id,
|
||||
expected=previous.chunk_seq + 1,
|
||||
actual=chunk_seq,
|
||||
)
|
||||
|
||||
explicit_prompt_id = _scalar(meta.get("prompt_cache_id"))
|
||||
explicit_prompt_wav = _scalar(meta.get("prompt_wav"))
|
||||
if previous is not None and explicit_prompt_id is None and explicit_prompt_wav is None:
|
||||
prompt_cache_id = previous.prompt_cache_id
|
||||
prompt_wav = previous.prompt_wav
|
||||
elif reference is not None:
|
||||
prompt_cache_id, prompt_wav = self._write_reference_prompt(
|
||||
state_id,
|
||||
reference,
|
||||
meta.get("ref_audio_sr"),
|
||||
)
|
||||
else:
|
||||
prompt_cache_id = str(explicit_prompt_id or self._default_prompt_id)
|
||||
prompt_wav = str(explicit_prompt_wav or self._default_prompt_wav)
|
||||
|
||||
if previous is not None and prompt_cache_id != previous.prompt_cache_id:
|
||||
raise _batch_error(
|
||||
"prompt_changed_midstream",
|
||||
request_id=request_id,
|
||||
expected=previous.prompt_cache_id,
|
||||
actual=prompt_cache_id,
|
||||
)
|
||||
if previous is not None and prompt_wav != previous.prompt_wav:
|
||||
raise _batch_error(
|
||||
"prompt_changed_midstream",
|
||||
request_id=request_id,
|
||||
expected=previous.prompt_wav,
|
||||
actual=prompt_wav,
|
||||
)
|
||||
duplex_turn_id = _scalar(meta.get("duplex_turn_id"))
|
||||
duplex_epoch = _scalar(meta.get("duplex_epoch"))
|
||||
return _WorkItem(
|
||||
output_index=index,
|
||||
state_id=state_id,
|
||||
request_id=request_id,
|
||||
cache_epoch=cache_epoch,
|
||||
chunk_seq=chunk_seq,
|
||||
prompt_cache_id=prompt_cache_id,
|
||||
prompt_wav=prompt_wav,
|
||||
last_chunk=last_chunk,
|
||||
tokens=tokens,
|
||||
previous=previous,
|
||||
segment_text=str(_scalar(meta.get("native_duplex_segment_text"), "")),
|
||||
duplex_turn_id=int(duplex_turn_id) if duplex_turn_id is not None else None,
|
||||
duplex_epoch=int(duplex_epoch) if duplex_epoch is not None else None,
|
||||
segment_end=bool(_scalar(meta.get("segment_end"), False)),
|
||||
turn_end=bool(_scalar(meta.get("turn_end"), False)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _bucket_key(item: _WorkItem) -> tuple[Any, ...]:
|
||||
cache_signature: Any
|
||||
if item.previous is None:
|
||||
cache_signature = ("uninitialized",)
|
||||
else:
|
||||
cache_signature = state_shape_signature(item.previous.token2wav)
|
||||
return (
|
||||
item.prompt_cache_id,
|
||||
item.prompt_wav,
|
||||
int(item.tokens.numel()),
|
||||
cache_signature,
|
||||
item.last_chunk,
|
||||
item.cache_epoch,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
del positions, intermediate_tensors, inputs_embeds
|
||||
ids = input_ids if isinstance(input_ids, torch.Tensor) else torch.empty(0, dtype=torch.long)
|
||||
segments = self._split_segments(ids, kwargs.get("seq_token_counts"))
|
||||
empty = torch.empty(0, dtype=torch.float32, device=ids.device)
|
||||
sample_rate = torch.tensor(24000, dtype=torch.int32)
|
||||
if not runtime_additional_information:
|
||||
count = len(segments)
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={
|
||||
"model_outputs": [empty for _ in range(count)],
|
||||
"sr": [sample_rate for _ in range(count)],
|
||||
},
|
||||
)
|
||||
if len(runtime_additional_information) != len(segments):
|
||||
raise _batch_error(
|
||||
"runtime_info_count_mismatch",
|
||||
segments=len(segments),
|
||||
runtime_infos=len(runtime_additional_information),
|
||||
)
|
||||
if self.backend is None:
|
||||
raise _batch_error("backend_not_loaded")
|
||||
|
||||
state_ids = kwargs.get("request_ids")
|
||||
if state_ids is None:
|
||||
state_ids = []
|
||||
for index, info in enumerate(runtime_additional_information):
|
||||
if not isinstance(info, Mapping):
|
||||
state_ids.append(str(index))
|
||||
continue
|
||||
meta = info.get("meta")
|
||||
source = meta if isinstance(meta, Mapping) else info
|
||||
state_ids.append(str(_scalar(source.get("request_id"), index)))
|
||||
if len(state_ids) != len(segments):
|
||||
raise _batch_error(
|
||||
"request_id_count_mismatch",
|
||||
segments=len(segments),
|
||||
request_ids=len(state_ids),
|
||||
)
|
||||
items: list[_WorkItem] = []
|
||||
for index, (state_id, segment, info) in enumerate(
|
||||
zip(state_ids, segments, runtime_additional_information, strict=True)
|
||||
):
|
||||
if not isinstance(info, Mapping):
|
||||
raise _batch_error(
|
||||
"invalid_runtime_info",
|
||||
output_index=index,
|
||||
value_type=type(info).__name__,
|
||||
)
|
||||
items.append(self._parse_item(index, str(state_id), segment, info))
|
||||
state_ids = [item.state_id for item in items]
|
||||
if len(state_ids) != len(set(state_ids)):
|
||||
raise _batch_error("duplicate_request_in_forward", request_ids=state_ids)
|
||||
outputs = [empty for _ in segments]
|
||||
sentinels = [item for item in items if item.last_chunk and item.tokens.numel() == 0]
|
||||
compute_items = [item for item in items if item.tokens.numel() > 0]
|
||||
invalid_empty = [item.request_id for item in items if not item.last_chunk and item.tokens.numel() == 0]
|
||||
if invalid_empty:
|
||||
raise _batch_error("empty_nonfinal_chunk", request_ids=invalid_empty)
|
||||
|
||||
buckets: dict[tuple[Any, ...], list[_WorkItem]] = {}
|
||||
for item in compute_items:
|
||||
buckets.setdefault(self._bucket_key(item), []).append(item)
|
||||
undersized = [
|
||||
{
|
||||
"size": len(bucket),
|
||||
"request_ids": [item.request_id for item in bucket],
|
||||
"codec_len": int(bucket[0].tokens.numel()),
|
||||
}
|
||||
for bucket in buckets.values()
|
||||
if len(bucket) < self._min_batch_size
|
||||
]
|
||||
if undersized:
|
||||
raise _batch_error(
|
||||
"exact_shape_bucket_below_minimum",
|
||||
minimum=self._min_batch_size,
|
||||
buckets=undersized,
|
||||
)
|
||||
|
||||
pending: dict[str, _RequestState | None] = {item.state_id: None for item in sentinels}
|
||||
for bucket in buckets.values():
|
||||
batch_size = len(bucket)
|
||||
try:
|
||||
features = self.backend.prepare_prompt(
|
||||
bucket[0].prompt_cache_id,
|
||||
bucket[0].prompt_wav,
|
||||
)
|
||||
if bucket[0].previous is None:
|
||||
states = self.backend.setup_batch(features, batch_size)
|
||||
else:
|
||||
states = [item.previous.token2wav for item in bucket if item.previous is not None]
|
||||
tokens = torch.stack([item.tokens for item in bucket], dim=0)
|
||||
audios, next_states = self.backend.decode_batch(
|
||||
tokens,
|
||||
features,
|
||||
states,
|
||||
last_chunk=bucket[0].last_chunk,
|
||||
)
|
||||
except Exception as exc:
|
||||
if isinstance(exc, RuntimeError) and str(exc).startswith("MiniCPMO45Code2WavBatchError "):
|
||||
raise
|
||||
raise _batch_error(
|
||||
"backend_unsupported_or_failed",
|
||||
request_ids=[item.request_id for item in bucket],
|
||||
error_type=type(exc).__name__,
|
||||
error=str(exc),
|
||||
) from exc
|
||||
if len(audios) != batch_size or len(next_states) != batch_size:
|
||||
raise _batch_error(
|
||||
"backend_result_size_mismatch",
|
||||
expected=batch_size,
|
||||
audios=len(audios),
|
||||
states=len(next_states),
|
||||
)
|
||||
for item, audio, next_state in zip(bucket, audios, next_states, strict=True):
|
||||
outputs[item.output_index] = audio.reshape(-1).to(dtype=torch.float32)
|
||||
pending[item.state_id] = (
|
||||
None
|
||||
if item.last_chunk
|
||||
else _RequestState(
|
||||
cache_epoch=item.cache_epoch,
|
||||
chunk_seq=item.chunk_seq,
|
||||
prompt_cache_id=item.prompt_cache_id,
|
||||
prompt_wav=item.prompt_wav,
|
||||
token2wav=next_state,
|
||||
)
|
||||
)
|
||||
|
||||
for request_id, state in pending.items():
|
||||
if state is None:
|
||||
self._states.pop(request_id, None)
|
||||
self._release_prompt(request_id)
|
||||
else:
|
||||
self._states[request_id] = state
|
||||
sample_rate_tensor = torch.as_tensor(sample_rate, dtype=torch.int32)
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={
|
||||
"model_outputs": outputs,
|
||||
"sr": [sample_rate_tensor.clone() for _ in outputs],
|
||||
"meta.llm_output_text_utf8": [
|
||||
torch.tensor(list(item.segment_text.encode("utf-8")), dtype=torch.uint8) for item in items
|
||||
],
|
||||
"meta.tts_is_last_chunk": [torch.tensor(item.last_chunk, dtype=torch.bool) for item in items],
|
||||
"meta.segment_end": [torch.tensor(item.segment_end, dtype=torch.bool) for item in items],
|
||||
"meta.turn_end": [torch.tensor(item.turn_end, dtype=torch.bool) for item in items],
|
||||
"meta.duplex_turn_id": [
|
||||
torch.tensor(item.duplex_turn_id, dtype=torch.int64)
|
||||
if item.duplex_turn_id is not None
|
||||
else torch.empty(0, dtype=torch.int64)
|
||||
for item in items
|
||||
],
|
||||
"meta.duplex_epoch": [
|
||||
torch.tensor(item.duplex_epoch, dtype=torch.int64)
|
||||
if item.duplex_epoch is not None
|
||||
else torch.empty(0, dtype=torch.int64)
|
||||
for item in items
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
for request_id in finished_req_ids:
|
||||
state_id = str(request_id)
|
||||
self._states.pop(state_id, None)
|
||||
self._release_prompt(state_id)
|
||||
|
||||
def make_omni_output(self, model_outputs: Any, **_: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
|
||||
return OmniOutput(*model_outputs)
|
||||
raise TypeError(f"MiniCPMO45Code2Wav expected OmniOutput, got {type(model_outputs).__name__}")
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
for _ in weights:
|
||||
pass
|
||||
if self.backend is not None:
|
||||
return {name for name, _ in self.named_parameters()}
|
||||
from stepaudio2.token2wav import Token2wav
|
||||
|
||||
extra = self._extra_config()
|
||||
prompt_path = Path(self._default_prompt_wav)
|
||||
if not prompt_path.is_file():
|
||||
raise FileNotFoundError(f"MiniCPM-o Code2Wav prompt audio not found: {prompt_path}")
|
||||
token2wav_path = Path(self.model_path) / "assets" / "token2wav"
|
||||
if not token2wav_path.is_dir():
|
||||
raise FileNotFoundError(f"MiniCPM-o Code2Wav assets not found: {token2wav_path}")
|
||||
use_float16 = bool(extra.get("token2wav_float16", False))
|
||||
previous_dtype = torch.get_default_dtype()
|
||||
try:
|
||||
# vLLM constructs bf16 models under a bf16 default-dtype context.
|
||||
# Token2wav contains fp32-only S3Tokenizer/HiFT modules, so build
|
||||
# its independent assets in their native precision.
|
||||
torch.set_default_dtype(torch.float32)
|
||||
token2wav = Token2wav(
|
||||
str(token2wav_path),
|
||||
float16=use_float16,
|
||||
n_timesteps=int(extra.get("token2wav_n_timesteps", 10)),
|
||||
)
|
||||
finally:
|
||||
torch.set_default_dtype(previous_dtype)
|
||||
self.backend = BatchedToken2Wav(token2wav)
|
||||
# Token2wav loads flow.pt and hift.pt inside its constructor instead of
|
||||
# from the parent MiniCPM checkpoint iterator. Report those registered
|
||||
# parameters as initialized so vLLM's strict loader audit does not
|
||||
# misclassify the independently loaded Stage-2 weights as missing.
|
||||
return {name for name, _ in self.named_parameters()}
|
||||
@@ -34,7 +34,6 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy
|
||||
from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
|
||||
@@ -58,9 +57,11 @@ logger = init_logger(__name__)
|
||||
class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE):
|
||||
"""MiniCPM-o 4.5 Omni model for conditional generation.
|
||||
|
||||
This model has two pipeline stages:
|
||||
- llm: multimodal thinker and text generation
|
||||
- tts: talker generation followed by its built-in Token2Wav vocoder
|
||||
Three-stage pipeline:
|
||||
- thinker (model_stage="llm"): image / video / audio encoders + 3D
|
||||
resampler + the omni LLM that emits text + hidden states.
|
||||
- talker (model_stage="tts"): native continuous MiniCPMTTS AR that emits
|
||||
codec-token deltas for the separate Code2Wav stage.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
@@ -106,7 +107,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
|
||||
elif self.model_stage == "tts":
|
||||
self.thinker = None
|
||||
# Initialize talker model (LLM generation)
|
||||
# The Talker is always the runner-owned continuous codec producer.
|
||||
self.talker = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "talker"),
|
||||
@@ -125,12 +126,18 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
self.make_empty_intermediate_tensors = (
|
||||
(self.thinker.make_empty_intermediate_tensors)
|
||||
if self.model_stage == "llm" and self.thinker is not None
|
||||
else self.talker.make_empty_intermediate_tensors
|
||||
if self.talker is not None
|
||||
else lambda: None
|
||||
)
|
||||
|
||||
self._language_model_names = ["model"]
|
||||
self.prefer_model_sampler = self.model_stage in {"llm", "tts"}
|
||||
self.has_preprocess = self.model_stage == "llm"
|
||||
# Both AR stages require model-specific embeddings. The Thinker uses
|
||||
# preprocess for duplex audio, while the Talker converts the
|
||||
# tts_token_ids/tts_hidden_states handoff into its conditioning
|
||||
# embeddings and initializes request-local codec generation state.
|
||||
self.has_preprocess = self.model_stage in {"llm", "tts"}
|
||||
|
||||
@cached_property
|
||||
def sampler(self):
|
||||
@@ -299,6 +306,8 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
and sampler. This hook only turns the current duplex audio append into
|
||||
the prompt embeddings consumed by the normal runner forward.
|
||||
"""
|
||||
if self.model_stage == "tts":
|
||||
return self.talker.preprocess(input_ids=input_ids, input_embeds=input_embeds, **kwargs)
|
||||
if self.model_stage != "llm":
|
||||
embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids)
|
||||
return input_ids, embeds, {}
|
||||
@@ -474,174 +483,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
"""vLLM V1 encoder profiling calls this; the inherited Protocol stub returns None."""
|
||||
return self.get_multimodal_embeddings(**kwargs)
|
||||
|
||||
def _run_tts_request(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor | None,
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
talker_info: dict[str, Any],
|
||||
device: torch.device,
|
||||
) -> tuple[dict[str, Any] | None, bool, bool]:
|
||||
meta_info = talker_info.get("meta") if isinstance(talker_info.get("meta"), dict) else {}
|
||||
if talker_info.get("native_duplex") is True:
|
||||
# ids/hidden_states may be accumulated for the talker KV stream,
|
||||
# but displayed transcript belongs to the current thinker segment.
|
||||
tts_text = meta_info.get("native_duplex_segment_text", "")
|
||||
else:
|
||||
tts_text = talker_info.get("llm_output_text", "")
|
||||
if isinstance(tts_text, list):
|
||||
tts_text = (
|
||||
tts_text[-1]
|
||||
if talker_info.get("native_duplex") is True and tts_text
|
||||
else (tts_text[0] if tts_text else "")
|
||||
)
|
||||
if not isinstance(tts_text, str):
|
||||
tts_text = ""
|
||||
|
||||
with torch.inference_mode():
|
||||
talker_result = self.talker(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
additional_information=talker_info,
|
||||
)
|
||||
chunk_flags = getattr(self.talker, "_ar_last_chunk_flags", [True])
|
||||
chunk_is_last = bool(chunk_flags[-1]) if chunk_flags else True
|
||||
turn_end_flags = getattr(self.talker, "_ar_turn_end_flags", [False])
|
||||
turn_ended = bool(turn_end_flags[-1]) if turn_end_flags else False
|
||||
if talker_info.get("native_duplex") is True:
|
||||
emitted_text = getattr(self.talker, "_ar_last_emitted_text", "")
|
||||
tts_text = emitted_text if isinstance(emitted_text, str) else ""
|
||||
|
||||
if not (isinstance(talker_result, tuple) and len(talker_result) == 2):
|
||||
return None, chunk_is_last, turn_ended
|
||||
|
||||
mel_spec, waveform = talker_result
|
||||
mm_out: dict[str, Any] = {}
|
||||
duplex_info = talker_info.get("duplex") if isinstance(talker_info.get("duplex"), dict) else {}
|
||||
if talker_info.get("native_duplex") is True:
|
||||
turn_id = duplex_info.get("turn_id")
|
||||
if isinstance(turn_id, int):
|
||||
mm_out["meta.duplex_turn_id"] = torch.tensor([turn_id], dtype=torch.int32, device=device)
|
||||
epoch = duplex_info.get("epoch")
|
||||
if isinstance(epoch, int):
|
||||
mm_out["meta.duplex_epoch"] = torch.tensor([epoch], dtype=torch.int32, device=device)
|
||||
mm_out["meta.tts_is_last_chunk"] = torch.tensor(
|
||||
[int(chunk_is_last)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
if talker_info.get("native_duplex") is True:
|
||||
mm_out["meta.turn_end"] = torch.tensor(
|
||||
[int(turn_ended)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
if tts_text or talker_info.get("native_duplex") is True:
|
||||
mm_out["meta.llm_output_text_utf8"] = torch.tensor(
|
||||
list(tts_text.encode("utf-8")),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
mm_out["meta.audio_text_total_chars"] = torch.tensor(
|
||||
[len(tts_text)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
if mel_spec is not None:
|
||||
mm_out["mel_spec"] = [mel_spec]
|
||||
if waveform is not None:
|
||||
mm_out["model_outputs"] = [waveform]
|
||||
elif mel_spec is not None:
|
||||
mm_out["model_outputs"] = [mel_spec]
|
||||
return mm_out, chunk_is_last, turn_ended
|
||||
|
||||
@staticmethod
|
||||
def _slice_tts_request_rows(value: torch.Tensor | None, start: int, end: int) -> torch.Tensor | None:
|
||||
return value[start:end] if isinstance(value, torch.Tensor) else value
|
||||
|
||||
def _run_batched_tts_requests(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor | None,
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
runtime_info: list[object],
|
||||
request_token_spans: object,
|
||||
num_tokens: int,
|
||||
hidden_dim: int,
|
||||
device: torch.device,
|
||||
) -> OmniOutput:
|
||||
if not isinstance(request_token_spans, list) or len(request_token_spans) != len(runtime_info):
|
||||
raise RuntimeError(
|
||||
"MiniCPM-o 4.5 batched TTS requires one request_token_spans entry "
|
||||
f"per request; got {request_token_spans!r} for {len(runtime_info)} requests"
|
||||
)
|
||||
|
||||
request_ids: list[str] = []
|
||||
request_outputs: list[dict[str, Any]] = []
|
||||
row_chunk_flags = [True] * num_tokens
|
||||
row_turn_end_flags = [False] * num_tokens
|
||||
previous_end = 0
|
||||
for index, raw_info in enumerate(runtime_info):
|
||||
if not isinstance(raw_info, dict):
|
||||
raise RuntimeError(f"MiniCPM-o 4.5 TTS request metadata at index {index} is not a dict")
|
||||
span = request_token_spans[index]
|
||||
if not (
|
||||
isinstance(span, (list, tuple)) and len(span) == 2 and all(isinstance(value, int) for value in span)
|
||||
):
|
||||
raise RuntimeError(f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: {span!r}")
|
||||
start, end = span
|
||||
if start != previous_end or start < 0 or end < start or end > num_tokens:
|
||||
raise RuntimeError(
|
||||
f"Invalid MiniCPM-o 4.5 TTS token span at index {index}: "
|
||||
f"expected start {previous_end} and 0 <= start <= end <= {num_tokens}, got {span!r}"
|
||||
)
|
||||
previous_end = end
|
||||
if start == end:
|
||||
continue
|
||||
mm_out, chunk_is_last, turn_ended = self._run_tts_request(
|
||||
input_ids=self._slice_tts_request_rows(input_ids, start, end),
|
||||
positions=self._slice_tts_request_rows(positions, start, end),
|
||||
inputs_embeds=self._slice_tts_request_rows(inputs_embeds, start, end),
|
||||
talker_info=raw_info,
|
||||
device=device,
|
||||
)
|
||||
row_chunk_flags[start:end] = [chunk_is_last] * (end - start)
|
||||
row_turn_end_flags[start:end] = [turn_ended] * (end - start)
|
||||
if mm_out is not None:
|
||||
request_ids.append(get_stream_request_key(raw_info))
|
||||
request_outputs.append(mm_out)
|
||||
# vLLM pads the flattened token tensor to a CUDA Graph capture size.
|
||||
# The spans describe only scheduled request rows, so an uncovered tail
|
||||
# is graph padding rather than a missing request. Contiguity checks
|
||||
# above still reject gaps between real requests.
|
||||
|
||||
self.talker._ar_last_chunk_flags = row_chunk_flags
|
||||
self.talker._ar_turn_end_flags = row_turn_end_flags
|
||||
dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
|
||||
if not request_outputs:
|
||||
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
|
||||
|
||||
sparse_output: dict[str, Any] = {
|
||||
"meta.req_id": request_ids,
|
||||
"meta.sparse_audio": ["1"],
|
||||
}
|
||||
output_keys = set().union(*(output.keys() for output in request_outputs))
|
||||
for key in output_keys:
|
||||
values: list[Any] = []
|
||||
for output in request_outputs:
|
||||
value = output.get(key)
|
||||
if isinstance(value, list) and len(value) == 1:
|
||||
value = value[0]
|
||||
if value is None:
|
||||
dtype = torch.uint8 if key == "meta.llm_output_text_utf8" else torch.float32
|
||||
value = torch.empty(0, dtype=dtype, device=device)
|
||||
values.append(value)
|
||||
sparse_output[key] = values
|
||||
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=sparse_output)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -658,8 +499,9 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
Forward pass for MiniCPM-o Omni model.
|
||||
|
||||
Workflow:
|
||||
1) LLM: multimodal thinker → hidden states and text tokens
|
||||
2) TTS: talker + Token2Wav → speech waveform
|
||||
1) Thinker (model_stage="llm"): Image / video / audio encoders +
|
||||
3D resampler + omni LLM → text + hidden states.
|
||||
2) Talker (model_stage="tts"): native MiniCPMTTS AR → codec deltas.
|
||||
"""
|
||||
if self.model_stage == "llm":
|
||||
# Normalize to batched inputs if caller provides 1D/2D unbatched tensors
|
||||
@@ -778,57 +620,24 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
)
|
||||
|
||||
# Talker stage: runs TTS generation and its built-in Token2Wav vocoder.
|
||||
# Talker stage: runner-owned native AR only. Waveform generation belongs
|
||||
# to the separate Code2Wav stage.
|
||||
if self.model_stage == "tts":
|
||||
if input_ids is not None:
|
||||
num_tokens = input_ids.shape[0]
|
||||
device = input_ids.device
|
||||
elif inputs_embeds is not None:
|
||||
num_tokens = inputs_embeds.shape[0]
|
||||
device = inputs_embeds.device
|
||||
else:
|
||||
num_tokens = 1
|
||||
device = current_omni_platform.get_torch_device()
|
||||
hidden_dim = self.config.hidden_size if hasattr(self.config, "hidden_size") else 2560
|
||||
|
||||
# Profile/dummy run: both input_ids and inputs_embeds are None.
|
||||
# Note: SupportsMultiModal preprocessing converts input_ids to
|
||||
# inputs_embeds, so input_ids=None alone does NOT indicate a dummy run.
|
||||
if input_ids is None and inputs_embeds is None:
|
||||
dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
|
||||
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
|
||||
|
||||
runtime_info = kwargs.get("runtime_additional_information")
|
||||
request_token_spans = kwargs.get("request_token_spans")
|
||||
if isinstance(runtime_info, list) and (len(runtime_info) > 1 or request_token_spans is not None):
|
||||
return self._run_batched_tts_requests(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
runtime_info=runtime_info,
|
||||
request_token_spans=request_token_spans,
|
||||
num_tokens=num_tokens,
|
||||
hidden_dim=hidden_dim,
|
||||
device=device,
|
||||
)
|
||||
talker_info = {}
|
||||
if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0:
|
||||
talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {}
|
||||
dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
|
||||
mm_out, _, _ = self._run_tts_request(
|
||||
return self.talker(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
talker_info=talker_info,
|
||||
device=device,
|
||||
**kwargs,
|
||||
)
|
||||
if mm_out is not None:
|
||||
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=mm_out)
|
||||
|
||||
return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
|
||||
|
||||
raise ValueError(f"Unsupported model stage: {self.model_stage}")
|
||||
|
||||
def make_omni_output(self, model_outputs, **kwargs):
|
||||
if self.model_stage != "tts":
|
||||
return model_outputs
|
||||
return self.talker.make_omni_output(model_outputs, **kwargs)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
|
||||
# Handle OmniOutput type
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
@@ -1261,7 +1070,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
|
||||
# MiniCPM-o checkpoint prefixes → stage mapping:
|
||||
# thinker: vpm, resampler, llm, apm, audio_projection_layer
|
||||
# talker: tts (ConditionalChatTTS)
|
||||
# talker: tts (native MiniCPMTTS AR codec producer)
|
||||
for k, v in weights:
|
||||
if k.startswith(("vpm.", "resampler.", "llm.", "apm.", "audio_projection_layer.")):
|
||||
thinker_weights.append((k, v))
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -3,12 +3,11 @@
|
||||
"""MiniCPM-o 4.5 pipeline topology (frozen).
|
||||
|
||||
Stage 0: Thinker — multimodal understanding + text generation.
|
||||
Stage 1: Talker — MiniCPMTTS + Token2Wav, emits the final audio waveform.
|
||||
Stage 1: Talker — MiniCPMTTS, emits codec tokens.
|
||||
Stage 2: Code2Wav — codec tokens to the final audio waveform.
|
||||
|
||||
The thinker -> talker bridge passes the hidden states + token ids extracted
|
||||
from the thinker output through ``minicpmo_4_5_omni.llm2tts``; the talker
|
||||
runs MiniCPMTTS and the on-device Token2wav vocoder in the same process and
|
||||
returns the waveform directly as the pipeline's final audio output.
|
||||
The thinker -> talker bridge uses ``llm2tts``. The talker -> Code2Wav bridge
|
||||
streams request-routed codec chunks.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
@@ -56,28 +55,24 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="tts",
|
||||
# Stage 1 shares the top-level wrapper class
|
||||
# (``MiniCPMO45OmniForConditionalGeneration``) inherited from
|
||||
# ``model_arch`` above. The wrapper dispatches on ``model_stage``
|
||||
# and, for ``"tts"``, instantiates the standalone TTS submodule
|
||||
# (``MiniCPMO45OmniTTSForConditionalGeneration``) internally.
|
||||
# Routing through the wrapper is required so that the runner-side
|
||||
# ``runtime_additional_information`` payload reaches the talker
|
||||
# (the standalone TTS class only reads ``additional_information``,
|
||||
# so wiring stage 1 directly to it would always trigger the dummy
|
||||
# path) and so the resulting waveform is packaged as
|
||||
# ``OmniOutput.multimodal_outputs["model_outputs"]`` instead of
|
||||
# being returned as a bare tuple that the AR runner would mistake
|
||||
# for hidden states. ``hf_config_name="tts_config"`` keeps KV
|
||||
# cache / mrope sizing scoped to the talker sub-config.
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(0,),
|
||||
hf_config_name="tts_config",
|
||||
engine_output_type="latent",
|
||||
custom_process_input_func=f"{_PROC}.llm2tts",
|
||||
async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=2,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(1,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
hf_config_name="tts_config",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.llm2tts",
|
||||
sampling_constraints={"detokenize": False},
|
||||
model_arch="MiniCPMO45Code2Wav",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -360,6 +360,11 @@ _OMNI_MODELS = {
|
||||
"minicpmo_4_5_omni_tts",
|
||||
"MiniCPMO45OmniTTSForConditionalGeneration",
|
||||
),
|
||||
"MiniCPMO45Code2Wav": (
|
||||
"minicpmo_4_5",
|
||||
"minicpmo_4_5_code2wav",
|
||||
"MiniCPMO45Code2Wav",
|
||||
),
|
||||
"AuraQwen3VLForConditionalGeneration": (
|
||||
"aura_omni",
|
||||
"qwen3_vl",
|
||||
|
||||
Reference in new issue
Block a user