[Model]Support MiniCPM-o 4.5 (#3642)

Signed-off-by: tc-mb <tianchi_cai@icloud.com>
Co-authored-by: GKangaroo <1095103651@qq.com>
Co-authored-by: GKangaroo <gqx24@mails.tsinghua.edu.cn>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

vllm_omni/model_executor/models/minicpmo_4_5/__init__.py

@@ -0,0 +1,9 @@
+from .minicpmo_4_5_omni import MiniCPMO45OmniForConditionalGeneration
+from .minicpmo_4_5_omni_llm import MiniCPMO45OmniLLMForConditionalGeneration
+from .minicpmo_4_5_omni_tts import MiniCPMO45OmniTTSForConditionalGeneration
+
+__all__ = [
+    "MiniCPMO45OmniForConditionalGeneration",
+    "MiniCPMO45OmniLLMForConditionalGeneration",
+    "MiniCPMO45OmniTTSForConditionalGeneration",
+]

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py

@@ -0,0 +1,384 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from:
+# https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py
+#
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Iterable
+from functools import cached_property
+
+import torch
+import torch.nn as nn
+from vllm.config import VllmConfig
+from vllm.logger import init_logger
+from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP
+from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.sequence import IntermediateTensors
+from vllm.v1.outputs import SamplerOutput
+from vllm.v1.sample.metadata import SamplingMetadata
+
+from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
+    MiniCPMO45OmniLLMDummyInputsBuilder,
+    MiniCPMO45OmniLLMMultiModalProcessor,
+    MiniCPMO45OmniLLMProcessingInfo,
+    MiniCPMOConfig,
+)
+from vllm_omni.model_executor.models.output_templates import OmniOutput
+from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
+from vllm_omni.platforms import current_omni_platform
+
+logger = init_logger(__name__)
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+    MiniCPMO45OmniLLMMultiModalProcessor,
+    info=MiniCPMO45OmniLLMProcessingInfo,
+    dummy_inputs=MiniCPMO45OmniLLMDummyInputsBuilder,
+)
+class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE):
+    """MiniCPM-o 4.5 Omni model for conditional generation.
+
+    Two-stage pipeline:
+    - thinker (model_stage="llm"): image / video / audio encoders + 3D
+      resampler + the omni LLM that emits text + hidden states.
+    - talker  (model_stage="tts"): MiniCPMTTS + the in-process Token2Wav
+      vocoder that emits the final audio waveform directly.
+    """
+
+    @classmethod
+    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+        if modality.startswith("image"):
+            return "(<image>./</image>)"
+        if modality.startswith("video"):
+            return "(<video>./</video>)"
+        if modality.startswith("audio"):
+            return "(<audio>./</audio>)"
+        raise ValueError("Only image, video or audio modality is supported")
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+        super().__init__()
+        self.have_multimodal_outputs = True
+        config: MiniCPMOConfig = vllm_config.model_config.hf_config
+        multimodal_config = vllm_config.model_config.multimodal_config
+        # keep vllm_config for later submodule init
+        self.vllm_config = vllm_config
+
+        # Store configs
+        self.config = config
+        self.multimodal_config = multimodal_config
+
+        self.model_stage = vllm_config.model_config.model_stage
+
+        if self.model_stage == "llm":
+            # Initialize thinker model (image preprocessing + vision encoder + 3D resampler)
+            self.thinker = init_vllm_registered_model(
+                vllm_config=vllm_config,
+                prefix=maybe_prefix(prefix, "thinker"),
+                hf_config=config,
+                # Registry key — must match the entries declared in
+                # vllm_omni/model_executor/models/registry.py::_OMNI_MODELS.
+                architectures=["MiniCPMO45OmniLLMForConditionalGeneration"],
+            )
+            self.model = self.thinker
+            self.talker = None
+
+        elif self.model_stage == "tts":
+            self.thinker = None
+            # Initialize talker model — runs MiniCPMTTS + the in-process
+            # Token2Wav vocoder and emits the final audio waveform directly.
+            self.talker = init_vllm_registered_model(
+                vllm_config=vllm_config,
+                prefix=maybe_prefix(prefix, "talker"),
+                hf_config=config,
+                # Registry key — must match the entries declared in
+                # vllm_omni/model_executor/models/registry.py::_OMNI_MODELS.
+                architectures=["MiniCPMO45OmniTTSForConditionalGeneration"],
+            )
+            # Initialize multimodal components if needed
+            if hasattr(self.talker, "init_multi_modal"):
+                self.talker.init_multi_modal(config)
+            self.model = self.talker
+
+        else:
+            raise ValueError(f"Invalid model stage: {self.model_stage}. Must be one of: 'llm', 'tts'")
+
+        # Set up intermediate tensors
+        self.make_empty_intermediate_tensors = (
+            (self.thinker.make_empty_intermediate_tensors)
+            if self.model_stage == "llm" and self.thinker is not None
+            else lambda: None
+        )
+
+        self._language_model_names = ["model"]
+
+    @cached_property
+    def sampler(self):
+        if hasattr(self.model, "sampler"):
+            return self.model.sampler
+        from vllm.v1.sample.sampler import Sampler
+
+        return Sampler()
+
+    # -------------------- Device utilities --------------------
+    @staticmethod
+    def _module_device(module: nn.Module) -> torch.device:
+        try:
+            return next(module.parameters()).device
+        except StopIteration:
+            # No parameters; fall back to buffers or cpu
+            for _, buf in module.named_buffers(recurse=True):
+                return buf.device
+            return torch.device("cpu")
+
+    def move_submodules_to_devices(
+        self,
+        *,
+        thinker_device: str | torch.device | None = None,
+        talker_device: str | torch.device | None = None,
+    ) -> None:
+        """Optionally move thinker / talker to different devices.
+
+        Example:
+            model.move_submodules_to_devices(
+                thinker_device='cuda:0',
+                talker_device='cuda:1',
+            )
+        """
+        if thinker_device is not None and self.thinker is not None:
+            self.thinker.to(thinker_device)
+        if talker_device is not None and self.talker is not None:
+            self.talker.to(talker_device)
+
+    def get_input_embeddings(
+        self,
+        input_ids: torch.Tensor,
+        multimodal_embeddings=None,
+    ) -> torch.Tensor:
+        return self.model.get_input_embeddings(input_ids, multimodal_embeddings)
+
+    def embed_input_ids(
+        self,
+        input_ids: torch.Tensor,
+        multimodal_embeddings=None,
+        *,
+        is_multimodal=None,
+    ) -> torch.Tensor:
+        if self.model_stage == "tts":
+            return self.get_input_embeddings(input_ids)
+        return super().embed_input_ids(input_ids, multimodal_embeddings, is_multimodal=is_multimodal)
+
+    def get_multimodal_embeddings(self, **kwargs):
+        # Delegate to the active stage submodule when it implements MM encoding.
+        mm_fn = getattr(self.model, "get_multimodal_embeddings", None)
+        if mm_fn is not None:
+            return mm_fn(**kwargs)
+        return []
+
+    def embed_multimodal(self, **kwargs: object):
+        """vLLM V1 encoder profiling calls this; the inherited Protocol stub returns None."""
+        return self.get_multimodal_embeddings(**kwargs)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        intermediate_tensors: IntermediateTensors | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        sampling_metadata: SamplingMetadata | None = None,
+        logits_index: int | None = None,
+        sampler=None,
+        additional_information: dict[str, object] | None = None,
+        **kwargs: object,
+    ) -> torch.Tensor | IntermediateTensors | OmniOutput:
+        """
+        Forward pass for MiniCPM-o Omni model.
+
+        Workflow:
+        1) Thinker (model_stage="llm"): Image / video / audio encoders +
+           3D resampler + omni LLM → text + hidden states.
+        2) Talker (model_stage="tts"): MiniCPMTTS + the in-process
+           Token2Wav vocoder → audio waveform (final pipeline output).
+        """
+        if self.model_stage == "llm":
+            # Normalize to batched inputs if caller provides 1D/2D unbatched tensors
+            # TODO: Remove this hack when NPU supports batched inputs properly
+            added_batch_dim = False
+            if input_ids is not None and input_ids.ndim == 1:
+                input_ids = input_ids.unsqueeze(0)
+                added_batch_dim = True
+            if positions is not None and positions.ndim == 1:
+                positions = positions.unsqueeze(0)
+                added_batch_dim = True
+            if inputs_embeds is not None and inputs_embeds.ndim == 2:
+                inputs_embeds = inputs_embeds.unsqueeze(0)
+                added_batch_dim = True
+            thinker_dev = self._module_device(self.thinker)
+
+            # if input_ids is None, set it to a zero tensor
+            if input_ids is None:
+                input_ids = torch.zeros(inputs_embeds.shape[1], dtype=torch.long, device=thinker_dev).unsqueeze(0)
+                added_batch_dim = True
+
+            # Ensure inputs on thinker's device
+            if input_ids is not None and input_ids.device != thinker_dev:
+                input_ids = input_ids.to(thinker_dev)
+            if positions is not None and positions.device != thinker_dev:
+                positions = positions.to(thinker_dev)
+            if inputs_embeds is not None and inputs_embeds.device != thinker_dev:
+                inputs_embeds = inputs_embeds.to(thinker_dev)
+
+            if current_omni_platform.is_npu():
+                # TODO: remove this hack when NPU supports batched inputs properly
+                thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
+                thinker_positions = positions[0] if positions.ndim > 1 else positions
+                thinker_inputs_embeds = (
+                    inputs_embeds[0] if inputs_embeds is not None and added_batch_dim else inputs_embeds
+                )
+            else:
+                thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
+                thinker_positions = positions[0] if positions is not None and added_batch_dim else positions
+                thinker_inputs_embeds = (
+                    inputs_embeds[0] if inputs_embeds is not None and added_batch_dim else inputs_embeds
+                )
+
+            # Run thinker
+            thinker_output = self.thinker(
+                input_ids=thinker_input_ids,
+                positions=thinker_positions,
+                intermediate_tensors=intermediate_tensors,
+                inputs_embeds=thinker_inputs_embeds,
+                **kwargs,
+            )
+
+            if isinstance(thinker_output, tuple):
+                embeds, text_hidden_states = thinker_output
+            else:
+                text_hidden_states = thinker_output
+
+            # Prepare hidden states for downstream stages
+            # Ensure correct shape: (batch_size, seq_len, hidden_dim)
+            if added_batch_dim:
+                text_hidden_states = text_hidden_states.squeeze(0)
+
+            # Return hidden states with latent in multimodal_outputs for stage_input_processors
+            return OmniOutput(
+                text_hidden_states=text_hidden_states,
+                multimodal_outputs={"latent": text_hidden_states},
+            )
+
+        # Talker stage: runs MiniCPMTTS + the in-process Token2Wav vocoder and
+        # emits the final audio waveform directly.
+        if self.model_stage == "tts":
+            if input_ids is not None:
+                num_tokens = input_ids.shape[0]
+                device = input_ids.device
+            elif inputs_embeds is not None:
+                num_tokens = inputs_embeds.shape[0]
+                device = inputs_embeds.device
+            else:
+                num_tokens = 1
+                device = torch.device("cuda")
+            hidden_dim = self.config.hidden_size if hasattr(self.config, "hidden_size") else 2560
+
+            # Profile/dummy run: both input_ids and inputs_embeds are None.
+            # Note: SupportsMultiModal preprocessing converts input_ids to
+            # inputs_embeds, so input_ids=None alone does NOT indicate a dummy run.
+            if input_ids is None and inputs_embeds is None:
+                dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
+                return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
+
+            runtime_info = kwargs.get("runtime_additional_information")
+            talker_info = {}
+            if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0:
+                talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {}
+
+            with torch.inference_mode():
+                talker_result = self.talker(
+                    input_ids=input_ids,
+                    positions=positions,
+                    inputs_embeds=inputs_embeds,
+                    additional_information=talker_info,
+                )
+
+            dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
+
+            # Talker returns a (mel_spec, waveform_or_None) tuple. MiniCPM-o
+            # 4.5 emits only the waveform (mel_spec is always None); keep the
+            # 2-slot unpack so a future mel-emitting variant can plug in
+            # without changing the wrapper.
+            mm_out: dict = {}
+            if isinstance(talker_result, tuple) and len(talker_result) == 2:
+                _, waveform = talker_result
+                if waveform is not None:
+                    mm_out["model_outputs"] = [waveform]
+
+            return OmniOutput(
+                text_hidden_states=dummy_hidden,
+                multimodal_outputs=mm_out if mm_out else None,
+            )
+
+        raise ValueError(f"Unsupported model stage: {self.model_stage}")
+
+    def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
+        # Handle OmniOutput type
+        if isinstance(hidden_states, OmniOutput):
+            hidden_states = hidden_states.text_hidden_states
+
+        # Use model for logits computation
+        return self.model.compute_logits(hidden_states)
+
+    def sample(
+        self,
+        logits: torch.Tensor,
+        sampling_metadata: SamplingMetadata,
+    ) -> SamplerOutput | None:
+        # Use model for sampling
+        return self.model.sample(logits, sampling_metadata)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+        """Load weights for the active stage of the omni model."""
+        loaded_weights = set()
+        thinker_weights = []
+        talker_weights = []
+
+        # MiniCPM-o checkpoint prefixes → stage mapping:
+        #   thinker: vpm, resampler, llm, apm, audio_projection_layer
+        #   talker:  tts (MiniCPMTTS); the Token2Wav vocoder weights load
+        #            separately inside the talker module from the
+        #            ``assets/token2wav`` subdirectory and do not appear
+        #            in this iterator.
+        for k, v in weights:
+            if k.startswith(("vpm.", "resampler.", "llm.", "apm.", "audio_projection_layer.")):
+                thinker_weights.append((k, v))
+            elif k.startswith("tts."):
+                talker_weights.append((k, v))
+            else:
+                logger.warning("Unknown weight prefix: %s, skipping", k)
+
+        # Load thinker weights
+        if self.thinker is not None and thinker_weights:
+            thinker_loaded = self.thinker.load_weights(thinker_weights)
+            thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
+            loaded_weights.update(thinker_loaded)
+
+        # Load talker weights
+        if self.talker is not None and talker_weights:
+            talker_loaded = self.talker.load_weights(talker_weights)
+            talker_loaded = add_prefix_to_loaded_weights(talker_loaded, "talker")
+            loaded_weights.update(talker_loaded)
+
+        return loaded_weights

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py

@@ -0,0 +1,456 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from:
+# https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py
+"""MiniCPM-o 4.5 Talker + Token2Wav: MiniCPMTTS with hidden_text_merge condition.
+
+Pipeline:
+  1. Receive thinker hidden_states + full token IDs via additional_information
+  2. Extract tts_bos..tts_eos region
+  3. Build condition: emb_text(tokens) + projector_semantic(hidden) (hidden_text_merge)
+  4. Run MiniCPMTTS.generate() -> discrete audio tokens
+  5. Run Token2wav(tokens) -> waveform bytes -> numpy array
+"""
+
+import io
+import logging
+import os
+from collections.abc import Iterable
+
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from vllm.config import VllmConfig
+from vllm.model_executor.models.interfaces import SupportsPP
+
+try:
+    from stepaudio2 import Token2wav as _Token2wav
+
+    _stepaudio2_available = True
+except ImportError:
+    _Token2wav = None
+    _stepaudio2_available = False
+
+logger = logging.getLogger(__name__)
+
+
+def _install_torchaudio_soundfile_shim() -> None:
+    """Monkey-patch torchaudio.load to use soundfile instead of the default
+    torchcodec backend, which requires libtorchcodec/ffmpeg shared libs that
+    may be missing on the deployment machine."""
+    try:
+        import torchaudio
+
+        if getattr(torchaudio, "_soundfile_shim_installed", False):
+            return
+        _orig_load = torchaudio.load
+
+        def _patched_load(uri, *args, **kwargs):
+            try:
+                return _orig_load(uri, *args, **kwargs)
+            except Exception:
+                import numpy as _np
+                import soundfile as _sf
+
+                data, sr = _sf.read(uri, dtype="float32", always_2d=True)
+                wav = torch.from_numpy(_np.ascontiguousarray(data.T))
+                return wav, sr
+
+        torchaudio.load = _patched_load
+        torchaudio._soundfile_shim_installed = True
+        logger.info("Installed torchaudio.load soundfile shim")
+    except Exception as _e:
+        logger.warning("Could not install torchaudio shim: %s", _e)
+
+
+_install_torchaudio_soundfile_shim()
+
+
+class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
+    """MiniCPM-o 4.5 Talker: MiniCPMTTS + Token2wav in a single forward pass."""
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+        super().__init__()
+        from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import MiniCPMOConfig
+
+        config: MiniCPMOConfig = vllm_config.model_config.hf_config
+        self.config = config
+        self.vllm_config = vllm_config
+
+        self.tts = None
+        self.audio_tokenizer = None
+        self._assets_loaded = False
+
+        tts_config = getattr(config, "tts_config", None)
+        if tts_config is not None:
+            self._tts_config = tts_config
+            self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687)
+            self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692)
+            self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562)
+            self._hidden_size = getattr(tts_config, "hidden_size", 768)
+            self._normalize = getattr(tts_config, "normalize_projected_hidden", True)
+        else:
+            self._tts_config = None
+
+    def _lazy_init_tts(self):
+        if self._assets_loaded or self._tts_config is None:
+            return
+        try:
+            model_path = self.vllm_config.model_config.model
+            import os
+            import sys
+
+            if model_path not in sys.path:
+                sys.path.insert(0, model_path)
+            from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+            MiniCPMTTS = get_class_from_dynamic_module("modeling_minicpmo.MiniCPMTTS", model_path)
+
+            # MiniCPMTTS.__init__ reads `config.top_p / top_k / repetition_penalty`
+            # directly (modeling_minicpmo.py L4112-4114), but the model repo's
+            # config.json `tts_config` block does not declare these fields and
+            # PretrainedConfig in recent transformers no longer surfaces
+            # generation-style params on `self.config`. Inject the defaults the
+            # upstream code itself ships with (modeling_minicpmo.py L2212-2214,
+            # L3132-3133) so attribute access does not raise.
+            for _attr, _default in (("top_p", 0.8), ("top_k", 100), ("repetition_penalty", 1.02)):
+                if not hasattr(self._tts_config, _attr):
+                    setattr(self._tts_config, _attr, _default)
+
+            prev_dtype = torch.get_default_dtype()
+            torch.set_default_dtype(torch.float32)
+            try:
+                self.tts_obj = MiniCPMTTS(config=self._tts_config, audio_tokenizer=None)
+            finally:
+                torch.set_default_dtype(prev_dtype)
+            self.emb_text = self.tts_obj.emb_text
+            self.projector_semantic = self.tts_obj.projector_semantic
+
+            token2wav_dir = os.path.join(model_path, "assets", "token2wav")
+            if os.path.isdir(token2wav_dir):
+                if not _stepaudio2_available:
+                    raise ImportError(
+                        "MiniCPM-o 4.5 token2wav stage requires the `stepaudio2` Python "
+                        "module (a MiniCPM-o-flavored Token2wav vocoder, NOT the upstream "
+                        "stepfun-ai/Step-Audio2 — the upstream signature does not accept "
+                        "n_timesteps and will fail at __init__). Install via:\n"
+                        "    pip install 'vllm-omni[minicpmo]'   # recommended, declared as PR extra\n"
+                        "Equivalent direct installs of the same `from stepaudio2 import Token2wav`\n"
+                        "entry point used by openbmb/MiniCPM-o-4_5/modeling_minicpmo.py:\n"
+                        "    pip install stepaudio2-minicpmo     # bare token2wav package\n"
+                        "    pip install 'minicpmo-utils[all]'   # MiniCPM-o umbrella (also brings image/video deps)"
+                    )
+                prev_dtype2 = torch.get_default_dtype()
+                torch.set_default_dtype(torch.float32)
+                try:
+                    # NB: this must be the MiniCPM-o-flavored Token2wav from
+                    # the `stepaudio2-minicpmo` PyPI package (or the
+                    # `minicpmo-utils[all]` umbrella), not the upstream
+                    # `stepfun-ai/Step-Audio2` repo. The MiniCPM-o variant's
+                    # __init__ accepts n_timesteps; the upstream signature is
+                    # (model_path, float16=False) and will raise
+                    # TypeError on n_timesteps. See ImportError message below
+                    # for installation guidance.
+                    self.audio_tokenizer = _Token2wav(token2wav_dir, float16=False, n_timesteps=10)
+                finally:
+                    torch.set_default_dtype(prev_dtype2)
+                self.tts_obj.audio_tokenizer = self.audio_tokenizer
+                logger.info("Loaded Token2wav from %s", token2wav_dir)
+            # Only mark init as complete after every step succeeds, so a
+            # partial failure leaves the next call free to retry the full
+            # init instead of short-circuiting back to a silent empty path.
+            self._assets_loaded = True
+        except ImportError:
+            # Surface missing dependencies directly so users can act on them
+            # instead of getting a silent None waveform downstream.
+            raise
+        except Exception:
+            # Re-raise non-import init failures (bad token2wav assets, missing
+            # weights, OOM during Token2wav construction, etc.) so the server
+            # fails loudly at startup / first request instead of returning
+            # silent empty audio for every subsequent request.
+            logger.error("Failed to init 4.5 TTS", exc_info=True)
+            raise
+
+    def generate_speech(
+        self,
+        tts_token_ids: torch.Tensor,
+        tts_hidden_states: torch.Tensor,
+    ) -> np.ndarray | None:
+        """Run full 4.5 TTS pipeline using original MiniCPMTTS.generate."""
+        self._lazy_init_tts()
+        if not hasattr(self, "tts_obj") or self.tts_obj is None:
+            logger.warning("generate_speech: tts_obj not initialized")
+            return None
+
+        tts = self.tts_obj
+        device = tts.emb_text.weight.device
+        dtype = tts.emb_text.weight.dtype
+
+        llm_embeds = tts.emb_text(tts_token_ids.to(device))
+        hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype))
+        if getattr(tts.config, "normalize_projected_hidden", False):
+            hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1)
+        tts_embeds = llm_embeds + hidden_embeds
+
+        text_eos = tts.emb_text(torch.tensor([tts.config.text_eos_token_id], device=device, dtype=torch.long))
+        audio_bos = tts.emb_text(torch.tensor([tts.audio_bos_token_id], device=device, dtype=torch.long))
+        spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=tts_embeds.dtype)
+
+        inputs_embeds = torch.cat([spk_embeds, tts_embeds, text_eos, audio_bos], dim=0).unsqueeze(0)
+        logger.info("generate_speech: inputs_embeds shape=%s", list(inputs_embeds.shape))
+
+        # Scale max_new_token with input text length to avoid mid-stream truncation on long
+        # responses (default 2048 can only cover ~300 text tokens at ~6x audio/text ratio).
+        # Empirically 511 text tokens → 1951 audio tokens (~3.8x) finishes cleanly, so use 10x
+        # as a safe upper bound with a floor of 2048 and a hard cap of 16384 to bound latency/mem.
+        num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0
+        max_new_token = max(2048, min(16384, num_text * 10))
+
+        eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=device)
+        outputs = tts.generate(
+            inputs_embeds=inputs_embeds,
+            eos_token=eos_token,
+            max_new_token=max_new_token,
+            show_tqdm=False,
+        )
+        generated_tokens = outputs.new_ids.squeeze(-1)
+        logger.info(
+            "generate_speech: generated %d audio tokens (cap=%d, text_tokens=%d)",
+            generated_tokens.shape[-1],
+            max_new_token,
+            num_text,
+        )
+
+        if self.audio_tokenizer is None:
+            logger.warning("No audio_tokenizer")
+            return None
+
+        import torchaudio
+
+        model_path = self.vllm_config.model_config.model
+        default_ref = os.path.join(model_path, "assets", "HT_ref_audio.wav")
+        prompt_wav_path = default_ref if os.path.exists(default_ref) else None
+
+        _orig_save = torchaudio.save
+
+        def _patched_save(uri, src, sample_rate, **kw):
+            kw.pop("backend", None)
+            if hasattr(uri, "write"):
+                sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV")
+                return
+            return _orig_save(uri, src, sample_rate, backend="soundfile", **kw)
+
+        torchaudio.save = _patched_save
+        prev_dtype = torch.get_default_dtype()
+        torch.set_default_dtype(torch.float32)
+        try:
+            with torch.amp.autocast("cuda", enabled=False):
+                token_list = generated_tokens.squeeze(0).tolist()
+                num_tokens = len(token_list)
+
+                # For long outputs, the one-shot vocoder path
+                # (Token2wav.__call__ -> flow.inference) runs full O(N^2) self-
+                # attention over all audio tokens and OOMs on a 24GB card once
+                # N exceeds a few thousand (e.g. 4964 tokens needs ~3GiB for a
+                # single attention matmul). Switch to the chunked / streaming
+                # vocoder (set_stream_cache + stream) which truncates the flow
+                # attention caches to prompt_len + 100 steps on every chunk,
+                # keeping peak memory bounded regardless of total length.
+                STREAM_THRESHOLD = int(os.environ.get("MINICPMO45_TTS_STREAM_THRESHOLD", "2500"))  # ~100s @ 25Hz
+                CHUNK_SIZE = int(os.environ.get("MINICPMO45_TTS_STREAM_CHUNK", "50"))  # ~2s per chunk
+                MIN_TAIL = 6  # must exceed flow.pre_lookahead_len (typically 3)
+
+                if num_tokens <= STREAM_THRESHOLD:
+                    wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path)
+                    waveform, sr = sf.read(io.BytesIO(wav_bytes))
+                    waveform = waveform.astype(np.float32)
+                else:
+                    # Build chunk boundaries, merging a too-small tail into the
+                    # previous chunk so every chunk satisfies MIN_TAIL.
+                    boundaries = []
+                    i = 0
+                    while i < num_tokens:
+                        end = min(i + CHUNK_SIZE, num_tokens)
+                        if 0 < num_tokens - end < MIN_TAIL:
+                            end = num_tokens
+                        boundaries.append((i, end))
+                        i = end
+
+                    logger.info(
+                        "generate_speech: streaming vocoder, %d tokens -> %d chunks (chunk=%d)",
+                        num_tokens,
+                        len(boundaries),
+                        CHUNK_SIZE,
+                    )
+
+                    stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path)
+                    self.audio_tokenizer.stream_cache = stream_cache
+                    self.audio_tokenizer.hift_cache_dict = hift_cache_dict
+
+                    try:
+                        pieces = []
+                        for idx, (s, e) in enumerate(boundaries):
+                            is_last = idx == len(boundaries) - 1
+                            wav_np = self.audio_tokenizer.stream(
+                                token_list[s:e],
+                                prompt_wav_path,
+                                last_chunk=is_last,
+                                return_waveform=True,
+                            )
+                            pieces.append(np.asarray(wav_np).reshape(-1))
+                        waveform = np.concatenate(pieces, axis=0).astype(np.float32)
+                        sr = 24000
+                    finally:
+                        # Free per-request streaming state so the next request starts clean
+                        self.audio_tokenizer.stream_cache = None
+                        self.audio_tokenizer.hift_cache_dict = {}
+        finally:
+            torch.set_default_dtype(prev_dtype)
+            torchaudio.save = _orig_save
+
+        logger.info("generate_speech: waveform %d samples, sr=%d", waveform.shape[0], sr)
+        return waveform
+
+    def _generate_tokens(self, inputs_embeds: torch.Tensor, max_new_token: int = 2048) -> torch.Tensor | None:
+        """Autoregressive generation of audio tokens using the TTS LlamaModel."""
+        device = inputs_embeds.device
+        eos_token = self._num_audio_tokens - 1
+        condition_length = inputs_embeds.shape[1]
+        num_vq = len(self.emb_code)
+
+        new_tokens = torch.zeros(1, max_new_token, num_vq, device=device, dtype=torch.long)
+        past_key_values = None
+        finished = False
+
+        for t in range(max_new_token):
+            if t == 0:
+                emb = inputs_embeds
+                position_ids = torch.arange(condition_length, device=device).unsqueeze(0)
+            else:
+                code_emb = [self.emb_code[q](new_tokens[:, t - 1 : t, q]) for q in range(num_vq)]
+                emb = torch.stack(code_emb, -1).sum(-1)
+                position_ids = torch.tensor([[condition_length + t - 1]], device=device)
+
+            outputs = self.tts_model(
+                inputs_embeds=emb,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                use_cache=True,
+            )
+            hidden = outputs.last_hidden_state
+            past_key_values = outputs.past_key_values
+
+            logits = torch.stack([self.head_code[q](hidden[:, -1]) for q in range(num_vq)], dim=-1)
+            logits = logits.float() / 0.8
+
+            if t < 50:
+                logits[:, eos_token, :] = -float("inf")
+
+            probs = F.softmax(logits, dim=1)
+            idx = torch.multinomial(probs.view(-1, probs.shape[1]), 1).view(1, num_vq)
+            new_tokens[:, t] = idx
+
+            if (idx == eos_token).any():
+                finished = True
+                break
+
+        return new_tokens[:, : t + 1 if finished else t, :]
+
+    def _dummy_hidden_states(
+        self,
+        input_ids: torch.Tensor | None,
+        positions: torch.Tensor | None,
+        inputs_embeds: torch.Tensor | None,
+    ) -> torch.Tensor:
+        """Shape-correct zero tensor for vllm KV cache profiling.
+
+        vllm's gpu_model_runner._dummy_run takes forward()'s return value as
+        ``hidden_states`` and does ``hidden_states[logit_indices_device]``;
+        returning None on the dummy path crashes with
+        ``TypeError: 'NoneType' object is not subscriptable``.
+        """
+        for ref in (input_ids, positions, inputs_embeds):
+            if isinstance(ref, torch.Tensor):
+                num_tokens = int(ref.shape[0]) if ref.ndim >= 1 else 1
+                device = ref.device
+                break
+        else:
+            num_tokens = 1
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        hidden_size = int(getattr(self, "_hidden_size", 768) or 768)
+        return torch.zeros((num_tokens, hidden_size), device=device, dtype=torch.bfloat16)
+
+    def forward(
+        self,
+        input_ids=None,
+        positions=None,
+        intermediate_tensors=None,
+        inputs_embeds=None,
+        additional_information=None,
+        **kwargs,
+    ):
+        if additional_information is None:
+            additional_information = {}
+
+        tts_token_ids = additional_information.get("tts_token_ids")
+        tts_hidden_states = additional_information.get("tts_hidden_states")
+        tts_text = additional_information.get("llm_output_text", [""])
+        if isinstance(tts_text, list):
+            tts_text = tts_text[0] if tts_text else ""
+
+        if tts_token_ids is None or tts_hidden_states is None:
+            # KV cache profiling / dummy run path — no real TTS input yet.
+            logger.debug("4.5 Talker: dummy forward (missing tts_token_ids/tts_hidden_states)")
+            return self._dummy_hidden_states(input_ids, positions, inputs_embeds)
+
+        logger.info("4.5 Talker: generating speech for %d tokens", tts_token_ids.shape[0])
+        waveform = self.generate_speech(tts_token_ids, tts_hidden_states)
+        # Tuple layout: (mel_spec, waveform). 4.5 talker emits only waveform,
+        # so mel_spec stays None; the wrapper unpacks in this order and
+        # packages the waveform into ``multimodal_outputs["model_outputs"]``.
+        if waveform is not None:
+            return None, torch.tensor(waveform, dtype=torch.float32)
+        return None, None
+
+    def compute_logits(self, hidden_states, *args, **kwargs):
+        return torch.zeros(1, 2, device=hidden_states.device if isinstance(hidden_states, torch.Tensor) else "cuda")
+
+    def sample(self, logits, sampling_metadata):
+        return None
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+        loaded = set()
+        tts_weights = {}
+        for k, v in weights:
+            if k.startswith("tts."):
+                tts_weights[k.replace("tts.", "", 1)] = v
+                # vllm sanity-checks `loaded` against `named_parameters()`.
+                # The submodule is attached at `self.tts_obj`, not `self.tts`,
+                # so report the loaded name under the on-module path.
+                loaded.add(k.replace("tts.", "tts_obj.", 1))
+
+        if tts_weights and self._tts_config is not None:
+            self._lazy_init_tts()
+            if hasattr(self, "tts_obj") and self.tts_obj is not None:
+                missing, unexpected = self.tts_obj.load_state_dict(tts_weights, strict=False)
+                if missing:
+                    logger.warning("TTS missing keys (%d): %s", len(missing), missing[:5])
+                if unexpected:
+                    logger.warning("TTS unexpected keys (%d): %s", len(unexpected), unexpected[:5])
+                self.tts_obj = self.tts_obj.to("cuda")
+                self.emb_text = self.tts_obj.emb_text
+                self.projector_semantic = self.tts_obj.projector_semantic
+                logger.info("Loaded %d TTS weights, moved to cuda", len(tts_weights))
+
+        return loaded
+
+    def get_input_embeddings(self, input_ids, multimodal_embeddings=None, **kwargs):
+        if hasattr(self, "emb_text") and self.emb_text is not None:
+            return self.emb_text(input_ids)
+        return torch.zeros(input_ids.shape[0], 1)
+
+    def embed_input_ids(self, input_ids, **kwargs):
+        return self.get_input_embeddings(input_ids, **kwargs)

vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""MiniCPM-o 4.5 pipeline topology (frozen).
+
+Stage 0: Thinker — multimodal understanding + text generation.
+Stage 1: Talker  — MiniCPMTTS + Token2Wav, emits the final audio waveform.
+
+The thinker -> talker bridge passes the hidden states + token ids extracted
+from the thinker output through ``minicpmo_4_5_omni.llm2tts``; the talker
+runs MiniCPMTTS and the on-device Token2wav vocoder in the same process and
+returns the waveform directly as the pipeline's final audio output.
+"""
+
+from vllm_omni.config.stage_config import (
+    PipelineConfig,
+    StageExecutionType,
+    StagePipelineConfig,
+)
+
+_PROC = "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni"
+
+
+MINICPMO_4_5_PIPELINE = PipelineConfig(
+    model_type="minicpmo_4_5",
+    model_arch="MiniCPMO45OmniForConditionalGeneration",
+    # MiniCPM-o 4.5's HF config.json reports `model_type="minicpmo"` and
+    # `architectures=["MiniCPMO"]` — both shared verbatim with older MiniCPM-o
+    # 1.0 / 2.6 checkpoints. The only field distinguishing the generations is
+    # the top-level ``version`` string, so we register both the shared
+    # ``MiniCPMO`` arch (for auto-detection) and the 4.5-specific arch (for
+    # repos that opt into the explicit name later), then pin the routing to
+    # 4.5 via ``hf_config_predicate``. Without the predicate, loading a 2.6
+    # checkpoint would also intersect ``["MiniCPMO"]`` here and get routed
+    # into the 4.5 pipeline, which would then fail at load time.
+    hf_architectures=("MiniCPMO", "MiniCPMO45OmniForConditionalGeneration"),
+    hf_config_predicate=lambda c: str(getattr(c, "version", "")) == "4.5",
+    stages=(
+        StagePipelineConfig(
+            stage_id=0,
+            model_stage="llm",
+            execution_type=StageExecutionType.LLM_AR,
+            input_sources=(),
+            final_output=True,
+            final_output_type="text",
+            owns_tokenizer=True,
+            requires_multimodal_data=True,
+            engine_output_type="latent",
+            sampling_constraints={"detokenize": True},
+        ),
+        StagePipelineConfig(
+            stage_id=1,
+            model_stage="tts",
+            # Stage 1 shares the top-level wrapper class
+            # (``MiniCPMO45OmniForConditionalGeneration``) inherited from
+            # ``model_arch`` above. The wrapper dispatches on ``model_stage``
+            # and, for ``"tts"``, instantiates the standalone TTS submodule
+            # (``MiniCPMO45OmniTTSForConditionalGeneration``) internally.
+            # Routing through the wrapper is required so that the runner-side
+            # ``runtime_additional_information`` payload reaches the talker
+            # (the standalone TTS class only reads ``additional_information``,
+            # so wiring stage 1 directly to it would always trigger the dummy
+            # path) and so the resulting waveform is packaged as
+            # ``OmniOutput.multimodal_outputs["model_outputs"]`` instead of
+            # being returned as a bare tuple that the AR runner would mistake
+            # for hidden states. ``hf_config_name="tts_config"`` keeps KV
+            # cache / mrope sizing scoped to the talker sub-config.
+            execution_type=StageExecutionType.LLM_AR,
+            input_sources=(0,),
+            final_output=True,
+            final_output_type="audio",
+            hf_config_name="tts_config",
+            engine_output_type="audio",
+            custom_process_input_func=f"{_PROC}.llm2tts",
+            sampling_constraints={"detokenize": False},
+        ),
+    ),
+)

vllm_omni/model_executor/models/registry.py

@@ -250,6 +250,22 @@ _OMNI_MODELS = {
         "ming_flash_omni",
         "MingFlashOmniForConditionalGeneration",
     ),
+    # MiniCPM-o 4.5 Omni models
+    "MiniCPMO45OmniForConditionalGeneration": (
+        "minicpmo_4_5",
+        "minicpmo_4_5_omni",
+        "MiniCPMO45OmniForConditionalGeneration",
+    ),
+    "MiniCPMO45OmniLLMForConditionalGeneration": (
+        "minicpmo_4_5",
+        "minicpmo_4_5_omni_llm",
+        "MiniCPMO45OmniLLMForConditionalGeneration",
+    ),
+    "MiniCPMO45OmniTTSForConditionalGeneration": (
+        "minicpmo_4_5",
+        "minicpmo_4_5_omni_tts",
+        "MiniCPMO45OmniTTSForConditionalGeneration",
+    ),
 }