Merge branch 'main' into feat/moss-tts-nano

Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-04-21 17:50:21 -04:00
commit 3bd77d3f8f
611 files changed
+48775 -14783

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,654 @@
"""Qwen3 Code Predictor -- optimized re-prefill, no KV cache.
Shared by Qwen3-Omni and Qwen3-TTS talker models.
* SDPA attention (F.scaled_dot_product_attention) with native GQA support
* HF-compatible numerics (float32 RMSNorm, float32 RoPE, separate linear layers)
* Per-call embedding buffer to avoid cross-request aliasing
* Pre-allocated position_ids (read-only, safe to persist)
* torch.compile (epilogue_fusion=False) on inner transformer by default
* Optional manual CUDA graph capture per batch-size bucket
* Inline sampling (top-k + top-p) -- no custom op overhead
"""
from __future__ import annotations
import dataclasses
from collections.abc import Iterable
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
# ===================================================================
# HF-numerics-compatible layers for code predictor
# ===================================================================
#
# These use plain PyTorch ops (nn.Linear, manual RMSNorm in float32,
# rotate_half RoPE) to produce outputs numerically identical to the
# HuggingFace reference. vLLM's fused kernels (RMSNorm, QKVParallel,
# get_rope) introduce small precision differences that compound across
# the autoregressive steps of the code predictor, causing severe
# audio quality degradation.
#
# See: https://github.com/vllm-project/vllm-omni/issues/2274
class _RMSNorm(nn.Module):
"""RMSNorm matching HuggingFace's implementation exactly.
Computes variance in float32 to avoid bfloat16 precision loss.
"""
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
class _RotaryEmbedding(nn.Module):
"""RoPE matching HuggingFace's implementation exactly.
Forces float32 computation for cos/sin, matching HF's torch.autocast(enabled=False).
"""
def __init__(self, config) -> None:
super().__init__()
head_dim = getattr(
config,
"head_dim",
config.hidden_size // config.num_attention_heads,
)
rope_theta = getattr(config, "rope_theta", 10000.0)
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# position_ids: [batch, seq_len]
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
position_ids_expanded = position_ids[:, None, :].float()
# Force float32 (matching HF)
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos()
sin = emb.sin()
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
# ===================================================================
# Attention
# ===================================================================
class CodePredictorAttention(nn.Module):
"""Multi-head self-attention for code predictor.
Uses ``F.scaled_dot_product_attention`` with HF-compatible RoPE and RMSNorm.
No KV cache -- the code predictor always re-prefills the full (short)
sequence each AR step.
Input : [B, seq_len, hidden_size]
Output: [B, seq_len, hidden_size]
"""
def __init__(self, config, *, prefix: str = "") -> None:
super().__init__()
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.head_dim = getattr(
config,
"head_dim",
config.hidden_size // config.num_attention_heads,
)
self.hidden_size = config.hidden_size
self.scaling = self.head_dim**-0.5
self._use_gqa = self.num_kv_heads != self.num_heads
# Separate q/k/v projections matching HF (no fused packing)
bias = getattr(config, "attention_bias", False)
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=bias)
self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=bias)
self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=bias)
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
self.q_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim)
hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim)
q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)
k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2)
v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2)
cos, sin = position_embeddings
# cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads
cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]
sin = sin.unsqueeze(1)
q = (q * cos) + (_rotate_half(q) * sin)
k = (k * cos) + (_rotate_half(k) * sin)
attn_out = F.scaled_dot_product_attention(
q,
k,
v,
scale=self.scaling,
is_causal=True,
enable_gqa=self._use_gqa,
)
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
return self.o_proj(attn_out)
# ===================================================================
# MLP
# ===================================================================
class CodePredictorMLP(nn.Module):
"""SiLU-gated MLP for code predictor, matching HF's implementation."""
def __init__(self, config, *, prefix: str = "") -> None:
super().__init__()
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
# ===================================================================
# Decoder Layer
# ===================================================================
class CodePredictorDecoderLayer(nn.Module):
"""Transformer decoder layer (SDPA, no KV cache)."""
def __init__(self, config, *, prefix: str = "") -> None:
super().__init__()
self.self_attn = CodePredictorAttention(config, prefix=f"{prefix}.self_attn")
self.mlp = CodePredictorMLP(config, prefix=f"{prefix}.mlp")
self.input_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(hidden_states, position_embeddings)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
# ===================================================================
# Base Transformer Model (re-prefill, no KV cache)
# ===================================================================
class CodePredictorBaseModel(nn.Module):
"""Inner transformer for code predictor.
Signature: ``forward(inputs_embeds, position_ids) -> hidden_states``
"""
def __init__(
self,
config,
*,
embedding_dim: int | None = None,
use_parallel_embedding: bool = False,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
emb_dim = int(embedding_dim) if embedding_dim is not None else int(config.hidden_size)
if use_parallel_embedding:
self.codec_embedding = nn.ModuleList(
[VocabParallelEmbedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
)
else:
self.codec_embedding = nn.ModuleList(
[nn.Embedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
)
self.layers = nn.ModuleList(
[
CodePredictorDecoderLayer(config, prefix=f"{prefix}.layers.{idx}")
for idx in range(config.num_hidden_layers)
]
)
self.norm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = _RotaryEmbedding(config)
def get_input_embeddings(self) -> nn.ModuleList:
return self.codec_embedding
def forward(
self,
inputs_embeds: torch.Tensor,
position_ids: torch.Tensor,
) -> torch.Tensor:
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids)
for layer in self.layers:
hidden_states = layer(hidden_states, position_embeddings)
hidden_states = self.norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
param = params_dict.get(name)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
# ===================================================================
# Wrapper Configuration
# ===================================================================
@dataclasses.dataclass
class CodePredictorWrapperConfig:
"""Controls behavioral differences between model-specific code predictors."""
use_cuda_graphs: bool = False
use_parallel_embedding: bool = False
use_projection: bool = False
return_proj_buf: bool = False
sampling_mode: str = "stored"
# ===================================================================
# Code Predictor Wrapper (optimized re-prefill, persistent buffers)
# ===================================================================
class CodePredictorWrapper(nn.Module):
"""Optimized code predictor -- re-prefill approach, no KV cache.
Each AR step forwards the full growing sequence (len 2 -> num_code_groups+1)
through the transformer. The extra O(T^2) FLOPs are negligible for
short sequences, and this avoids all KV-cache management overhead.
Optimizations:
1. Per-call embedding buffer -- avoids cross-request aliasing.
2. Pre-allocated position_ids -- no torch.arange per step.
3. Cached module references -- bypass ModuleList indexing.
4. torch.compile on inner transformer.
5. Inline sampling (top-k + top-p) -- no custom op overhead.
6. Optional manual CUDA graph capture per batch-size bucket.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
cp_config,
wrapper_config: CodePredictorWrapperConfig,
talker_hidden_size: int | None = None,
prefix: str = "",
) -> None:
super().__init__()
self._vllm_config = vllm_config
self.config = cp_config
self._wrapper_config = wrapper_config
self.prefix = prefix
self._num_groups = int(cp_config.num_code_groups)
self._cp_hidden = int(cp_config.hidden_size)
# For Omni backward compat (accessed by the talker)
self.num_code_groups = self._num_groups
# Determine embedding dimension
_talker_hidden = int(talker_hidden_size) if talker_hidden_size is not None else self._cp_hidden
self.model = CodePredictorBaseModel(
cp_config,
embedding_dim=_talker_hidden,
use_parallel_embedding=wrapper_config.use_parallel_embedding,
prefix=f"{prefix}.model" if prefix else "model",
)
self.lm_head = nn.ModuleList(
[nn.Linear(cp_config.hidden_size, cp_config.vocab_size, bias=False) for _ in range(self._num_groups - 1)]
)
# Projection: Identity when hidden sizes match or not needed
if wrapper_config.use_projection and _talker_hidden != self._cp_hidden:
self.small_to_mtp_projection = nn.Linear(_talker_hidden, self._cp_hidden, bias=True)
else:
self.small_to_mtp_projection = nn.Identity()
# Sampling defaults for "stored" mode
self._top_k: int = 50
self._top_p: float = 0.8
# Lazily initialised state
self._proj_buf: torch.Tensor | None = None
self._model_dtype: torch.dtype | None = None
self._compiled_model_fwd = None
self._bucket_sizes: list[int] = []
self._bucket_pos_ids: dict[int, torch.Tensor] = {}
self._lm_heads_list: list[nn.Module] | None = None
self._codec_embeds_list: list[nn.Module] | None = None
self._cuda_graphs: dict[int, tuple[torch.cuda.CUDAGraph, torch.Tensor]] = {}
def get_input_embeddings(self) -> nn.ModuleList:
return self.model.get_input_embeddings()
def set_sampling_params(self, top_k: int = 50, top_p: float = 0.8) -> None:
"""Configure sampling parameters to maintain consistency with previous implementation."""
self._top_k = top_k
self._top_p = top_p
logger.debug("Sampling parameters updated: top_k=%d, top_p=%.2f", top_k, top_p)
# ------------------------------------------------------------------
# Lazy-init helpers
# ------------------------------------------------------------------
def _ensure_buffers(self, device: torch.device, dtype: torch.dtype, bsz: int) -> None:
"""Ensure the projection buffer can hold at least *bsz* rows."""
max_seq = self._num_groups + 1
if (
self._proj_buf is not None
and self._proj_buf.device == device
and self._proj_buf.dtype == dtype
and self._proj_buf.shape[0] >= bsz
):
return
self._proj_buf = torch.zeros(bsz, max_seq, self._cp_hidden, dtype=dtype, device=device)
def _setup_compile(self) -> None:
"""Lazily set up torch.compile with optional CUDA graph capture."""
if self._compiled_model_fwd is not None:
return
# Cache model parameter dtype so forward() doesn't need to query it
# on every call. Also ensures warmup buffers match model precision
# even when upstream modules produce a different dtype (#2385).
self._model_dtype = next(self.model.parameters()).dtype
self._lm_heads_list = list(self.lm_head)
self._codec_embeds_list = list(self.model.codec_embedding)
if not current_omni_platform.supports_torch_inductor():
logger.warning_once("code_predictor: torch.compile disabled")
self._compiled_model_fwd = self.model.forward
return
# torch.compile fuses RMSNorm/RoPE in ways that lose float32
# precision, compounding across AR steps. Use epilogue_fusion=False
# to disable the problematic fusions while still getting kernel
# fusion benefits for the linear layers and SDPA.
self._compiled_model_fwd = torch.compile(
self.model.forward,
dynamic=False,
options={"epilogue_fusion": False},
)
self._warmup_buckets()
if self._wrapper_config.use_cuda_graphs:
self._capture_cuda_graphs()
logger.info("code_predictor: torch.compile (no epilogue fusion) + CUDA graphs")
else:
logger.info("code_predictor: torch.compile (dynamic=False, no epilogue fusion)")
def _padded_bsz(self, bsz: int) -> int:
"""Round batch size up to nearest power-of-2 bucket."""
for bucket in self._bucket_sizes:
if bsz <= bucket:
return bucket
return bsz
def _warmup_buckets(self) -> None:
"""Warmup power-of-2 batch-size buckets to front-load Inductor compilation."""
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
bucket_sizes = [1 << i for i in range(max_bsz.bit_length()) if (1 << i) <= max_bsz]
if max_bsz not in bucket_sizes:
bucket_sizes.append(max_bsz)
self._bucket_sizes = sorted(bucket_sizes)
max_seq = self._num_groups + 1
device = next(self.model.parameters()).device
# Ensure proj_buf matches model parameter dtype to avoid dtype
# mismatch during warmup compilation (see #2385).
self._ensure_buffers(device, self._model_dtype, max(self._bucket_sizes))
proj_buf = self._proj_buf
for bsz in self._bucket_sizes:
pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
self._bucket_pos_ids[bsz] = pos_ids
for _ in range(3):
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
def _capture_cuda_graphs(self) -> None:
"""Capture a CUDA graph per bucket using vLLM's global graph pool."""
from vllm.platforms import current_platform
pool = current_platform.get_global_graph_pool()
max_seq = self._num_groups + 1
proj_buf = self._proj_buf
for bsz in self._bucket_sizes:
static_input = proj_buf[:bsz, :max_seq, :]
pos_ids = self._bucket_pos_ids[bsz]
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g, pool=pool):
static_output = self._compiled_model_fwd(static_input, pos_ids)
self._cuda_graphs[bsz] = (g, static_output)
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
# ------------------------------------------------------------------
# Forward -- re-prefill + inline sampling
# ------------------------------------------------------------------
@torch.inference_mode()
def forward(
self,
layer0_code: torch.Tensor,
layer0_embed: torch.Tensor,
last_talker_hidden: torch.Tensor,
do_sample: bool = True,
temperature: float = 0.9,
top_k: int = 50,
top_p: float = 1.0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Predict residual codebooks 1..G-1 autoregressively via re-prefill."""
bsz = int(layer0_code.shape[0])
num_groups = self._num_groups
device = layer0_code.device
# _setup_compile caches _model_dtype on first call; use it for buffers
# so they always match model weight precision (#2385).
self._setup_compile()
dtype = self._model_dtype
padded_bsz = self._padded_bsz(bsz)
self._ensure_buffers(device, dtype, padded_bsz)
proj_buf = self._proj_buf
max_seq = num_groups + 1
projection = self.small_to_mtp_projection
model_fwd = self._compiled_model_fwd
lm_heads = self._lm_heads_list
codec_embeds = self._codec_embeds_list
# Zero the padded region of the buffer
proj_buf[:padded_bsz].zero_()
# Fill buffer positions 0 (talker hidden) & 1 (layer0 embed)
proj_buf[:bsz, 0, :] = projection(last_talker_hidden.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
proj_buf[:bsz, 1, :] = projection(layer0_embed.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
# Get pre-computed pos_ids for this bucket
full_pos_ids = self._bucket_pos_ids.get(padded_bsz)
if full_pos_ids is None:
full_pos_ids = (
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1).contiguous()
)
# Use captured CUDA graph if available, otherwise call compiled fn.
cuda_graph_entry = self._cuda_graphs.get(padded_bsz)
# Prepare sampling parameters
stored_mode = self._wrapper_config.sampling_mode == "stored"
if stored_mode:
s_top_k = self._top_k
s_top_p = self._top_p
else:
use_sampling = do_sample and temperature > 0
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
if use_sampling and top_p != 1.0:
raise NotImplementedError(
"top_p sampling is not implemented for the vLLM-native code predictor; please set top_p=1.0."
)
# Output codes -- shape depends on return mode
if self._wrapper_config.return_proj_buf:
all_codes = torch.empty(bsz, num_groups, 1, dtype=torch.int64, device=device)
all_codes[:, 0] = layer0_code.reshape(bsz, -1)[:, :1]
else:
all_codes = torch.empty(bsz, num_groups, dtype=torch.long, device=device)
all_codes[:, 0] = layer0_code.reshape(bsz)
# Autoregressive loop: predict layers 1..G-1
for step in range(1, num_groups):
# Run transformer (CUDA graph replay or compiled forward)
if cuda_graph_entry is not None:
cuda_graph_entry[0].replay()
hidden_out = cuda_graph_entry[1]
else:
hidden_out = model_fwd(proj_buf[:padded_bsz, :max_seq, :], full_pos_ids)
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
# Sample next code
if stored_mode:
# "stored" mode: top-k -> top-p -> softmax -> multinomial
if s_top_k > 0:
topk_vals, _ = logits.topk(s_top_k, dim=-1)
logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf"))
if s_top_p < 1.0:
sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
sorted_probs = F.softmax(sorted_logits, dim=-1)
cumulative_probs = sorted_probs.cumsum(dim=-1)
remove_mask = (cumulative_probs - sorted_probs) >= s_top_p
sorted_logits[remove_mask] = float("-inf")
logits = sorted_logits.scatter(1, sorted_idx, sorted_logits)
probs = F.softmax(logits, dim=-1)
code = torch.multinomial(probs, num_samples=1)
else:
# "per_call" mode: temperature-scaled + top-k
if use_sampling:
scaled = logits * inv_temperature
if top_k > 0:
topk_vals, _ = scaled.topk(top_k, dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
probs = F.softmax(scaled, dim=-1)
code = torch.multinomial(probs, num_samples=1)
else:
code = logits.argmax(dim=-1, keepdim=True)
# Store code
if self._wrapper_config.return_proj_buf:
all_codes[:, step] = code
else:
all_codes[:, step] = code.reshape(bsz)
# Embed predicted code -> project -> next buffer position
if step < num_groups - 1 or self._wrapper_config.return_proj_buf:
new_embed = codec_embeds[step - 1](code)
proj_buf[:bsz, step + 1, :] = projection(new_embed.reshape(bsz, 1, -1)).reshape(bsz, -1)
if self._wrapper_config.return_proj_buf:
return all_codes, proj_buf[:bsz].clone()
return all_codes
# ------------------------------------------------------------------
# Weight loading
# ------------------------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights directly (no fused projection remapping needed)."""
loaded: set[str] = set()
model_weights: list[tuple[str, torch.Tensor]] = []
other_weights: list[tuple[str, torch.Tensor]] = []
for name, w in weights:
if "rotary_emb.inv_freq" in name:
continue
if name.startswith("model."):
model_weights.append((name[len("model.") :], w))
else:
other_weights.append((name, w))
loaded_model = self.model.load_weights(model_weights)
loaded |= {f"model.{n}" for n in loaded_model}
params = dict(self.named_parameters(remove_duplicate=False))
for name, w in other_weights:
param = params.get(name)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, w)
loaded.add(name)
return loaded
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CosyVoice3 pipeline topology (frozen).
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
* ``sync_process_input_func`` runs when ``deploy.async_chunk=false``:
stage 1 builds full-sequence flow input via ``text2flow``.
* ``async_chunk_process_next_stage_input_func`` runs when
``deploy.async_chunk=true``: stage 0 streams codec chunks to stage 1
through the shared-memory connector.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
COSYVOICE3_PIPELINE = PipelineConfig(
model_type="cosyvoice3",
model_arch="CosyVoice3Model",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="cosyvoice3_talker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
sampling_constraints={
# merged speech stop token (logsumexp of all 200 stop logits)
"stop_token_ids": [6562],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="cosyvoice3_code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="latent",
sync_process_input_func=f"{_PROC}.text2flow",
),
),
)
@@ -649,6 +649,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
input_embeds: torch.Tensor,
last_talker_hidden: torch.Tensor,
text_step: torch.Tensor,
**kwargs: Any,
) -> tuple[torch.Tensor, torch.Tensor]:
"""GPU fast-path: run Fast AR to predict residual codebook codes.
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fish Speech S2 Pro pipeline topology (frozen).
Stage 0: slow_ar — text → RVQ codec tokens (LLM autoregressive).
Stage 1: dac_decoder — RVQ tokens → audio waveform (LLM_GENERATION).
The HF config top-level reports ``model_type = "fish_qwen3_omni"`` (the
OmniConfig that bundles slow-AR and fast-AR sub-configs), which is why the
registry key follows the HF name rather than the human-readable "fish_speech".
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech"
FISH_SPEECH_PIPELINE = PipelineConfig(
model_type="fish_qwen3_omni",
model_arch="FishSpeechSlowARForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="fish_speech_slow_ar",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.slow_ar_to_dac_decoder_async_chunk"),
sampling_constraints={
"detokenize": False,
# <|im_end|> — stop when the model emits end-of-turn.
"stop_token_ids": [151645],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="dac_decoder",
model_arch="FishSpeechDACDecoder",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sampling_constraints={"detokenize": True},
),
),
)
@@ -38,10 +38,7 @@ def _encode_plain_text(tokenizer: Any, text: str) -> list[int]:
def _encode_control_token(tokenizer: Any, token: str) -> list[int]:
vocab = tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else {}
token_id = vocab.get(token)
if token_id is None:
token_id = tokenizer.convert_tokens_to_ids(token)
token_id = tokenizer.convert_tokens_to_ids(token)
if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None):
raise ValueError(f"Fish Speech tokenizer is missing required control token: {token}")
return [int(token_id)]
@@ -21,6 +21,7 @@
# limitations under the License.
"""Inference-only GLM-Image model compatible with HuggingFace weights."""
import math
import os
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
@@ -127,6 +128,14 @@ class GlmImageDataParser(MultiModalDataParser):
parsers["img2img"] = self._parse_image_data
return parsers
def parse_mm_data(self, mm_data, **kwargs):
# Normalize "img2img" to "image" so the rest of the pipeline
# (mm_hashes, _merge_mm_kwargs) uses a single modality key.
normalized = {}
for k, v in mm_data.items():
normalized["image" if k == "img2img" else k] = v
return super().parse_mm_data(normalized, **kwargs)
class GlmImageProcessingInfo(BaseProcessingInfo):
"""
@@ -346,6 +355,10 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
target_h = mm_kwargs.get("target_h", 1024) if mm_kwargs else 1024
target_w = mm_kwargs.get("target_w", 1024) if mm_kwargs else 1024
logger.debug(
f"_call_hf_processor: target dimensions for generation: {target_h}x{target_w}, mm_kwargs={mm_kwargs}"
)
if not mm_data or not mm_data.get("images"):
# Text-to-image mode
if processor is not None:
@@ -566,6 +579,58 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
tensor_type="pt",
)
def _apply_hf_processor_text_only(
self, prompt_text: str, hf_processor_mm_kwargs: Mapping[str, object], tokenization_kwargs: Mapping[str, object]
) -> list[int]:
prompt_ids, _, _ = super()._apply_hf_processor_text_mm(
prompt_text=prompt_text,
mm_items=MultiModalDataItems({}),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return prompt_ids
def _build_generation_grids(self, hf_processor_mm_kwargs: Mapping[str, object]) -> torch.Tensor:
"""Build generation grids for M-RoPE decode positions.
For GLM-Image generation, decode order is:
1) small preview grid
2) large target grid
3) EOS
We store grids as [large, small] to match HF processor behavior, and
decode logic consumes them in reverse order.
"""
target_h = (
hf_processor_mm_kwargs.get("target_h") if isinstance(hf_processor_mm_kwargs.get("target_h"), int) else None
)
target_w = (
hf_processor_mm_kwargs.get("target_w") if isinstance(hf_processor_mm_kwargs.get("target_w"), int) else None
)
if target_h is None or target_w is None:
target_h = (
hf_processor_mm_kwargs.get("height") if isinstance(hf_processor_mm_kwargs.get("height"), int) else 1024
)
target_w = (
hf_processor_mm_kwargs.get("width") if isinstance(hf_processor_mm_kwargs.get("width"), int) else 1024
)
factor = 32
target_h = (target_h // factor) * factor
target_w = (target_w // factor) * factor
token_h = target_h // factor
token_w = target_w // factor
ratio = token_h / token_w if token_w > 0 else 1.0
small_token_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
small_token_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
return torch.tensor(
[[1, token_h, token_w], [1, small_token_h, small_token_w]],
dtype=torch.long,
)
def _apply_hf_processor_main(
self,
prompt: str | list[int],
@@ -594,126 +659,145 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
logger.debug(f"_apply_hf_processor_main: mm_counts={mm_counts}, num_images={num_images}")
if num_images == 0 or enable_hf_prompt_update:
if num_images == 0 and isinstance(prompt, str):
# t2i mode or normal flow - use parent implementation
return super()._apply_hf_processor_main(
prompt=prompt,
prompt_ids = self._apply_hf_processor_text_only(
prompt_text=prompt,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
enable_hf_prompt_update=enable_hf_prompt_update,
)
# i2i mode with enable_hf_prompt_update=False (cache miss scenario)
# We need to build prompt_ids with image placeholders
logger.debug(f"_apply_hf_processor_main: i2i mode with enable_hf_prompt_update=False, num_images={num_images}")
# t2i has no source images, so mm features cannot provide image_grid_thw.
# Provide explicit generation grids for M-RoPE to avoid fallback token parsing
# (which can degrade high-resolution spatial positions, e.g. 1920x1920).
try:
mrope_grid_thw = self._build_generation_grids(hf_processor_mm_kwargs)
mm_processed_data["mrope_image_grid_thw"] = mrope_grid_thw
logger.info(
"_apply_hf_processor_main t2i: mrope_image_grid_thw=%s",
mrope_grid_thw.tolist(),
)
except Exception as e:
logger.warning("_apply_hf_processor_main t2i: failed to set mrope_image_grid_thw: %s", e)
# Get mm data from our overridden _apply_hf_processor_mm_only
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
return prompt_ids, mm_processed_data, False
# i2i mode: use unified HF processor path only.
# This avoids drift between duplicated manual/HF i2i implementations.
logger.debug(
"_apply_hf_processor_main: i2i mode (enable_hf_prompt_update=%s), num_images=%s",
enable_hf_prompt_update,
num_images,
)
# In this path we do NOT call HF apply_chat_template, so we must still
# provide full grids (source + target) for M-RoPE to compute decode positions.
# Keep `image_grid_thw` source-only for MM batching/validation.
try:
source_grid_thw = mm_processed_data.get("image_grid_thw")
if source_grid_thw is not None and isinstance(source_grid_thw, torch.Tensor):
# Compute target grid following HF GlmImageProcessor: factor=32.
# Prefer explicit target_h/target_w if present, otherwise fall back.
target_h = (
hf_processor_mm_kwargs.get("target_h")
if isinstance(hf_processor_mm_kwargs.get("target_h"), int)
else None
)
target_w = (
hf_processor_mm_kwargs.get("target_w")
if isinstance(hf_processor_mm_kwargs.get("target_w"), int)
else None
)
if target_h is None or target_w is None:
# Some callers pass generation size as height/width.
target_h = (
hf_processor_mm_kwargs.get("height")
if isinstance(hf_processor_mm_kwargs.get("height"), int)
else 1024
)
target_w = (
hf_processor_mm_kwargs.get("width")
if isinstance(hf_processor_mm_kwargs.get("width"), int)
else 1024
)
if not isinstance(prompt, str):
# Online OpenAI chat preprocessing can arrive here with tokenized
# prompts (list[int]) before serving_chat replaces engine prompt
# with the clean text prompt. Do not fail the whole request.
logger.warning(
"_apply_hf_processor_main i2i: got tokenized prompt type=%s; "
"using compatibility path for preprocessing",
type(prompt).__name__,
)
factor = 32
target_h = (target_h // factor) * factor
target_w = (target_w // factor) * factor
token_h = target_h // factor
token_w = target_w // factor
target_grid = torch.tensor([[1, token_h, token_w]], dtype=source_grid_thw.dtype)
prompt_ids = list(prompt)
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
mm_processed_data["mrope_image_grid_thw"] = torch.cat([source_grid_thw, target_grid], dim=0)
except Exception:
# Best-effort only; M-RoPE has additional fallbacks.
pass
# Build prompt_ids with image placeholders
# _apply_prompt_updates will replace each [image_token_id] with expanded tokens
tokenizer = self.info.get_tokenizer()
image_token_id = tokenizer.convert_tokens_to_ids("<|image|>")
if isinstance(prompt, str):
# Match HF GlmImageProcessor behavior: append target grid tokens + BOS.
# This helps M-RoPE/grid parsing and keeps i2i vs t2i behavior aligned.
# Preserve full grids for M-RoPE decode (source + target), while
# keeping image_grid_thw source-only for MM batching.
try:
grid_bos = getattr(tokenizer, "grid_bos_token", "")
grid_eos = getattr(tokenizer, "grid_eos_token", "")
bos = getattr(tokenizer, "bos_token", "")
# Use the same target sizes we used for mrope grids when available.
target_h = (
hf_processor_mm_kwargs.get("target_h")
if isinstance(hf_processor_mm_kwargs.get("target_h"), int)
else None
)
target_w = (
hf_processor_mm_kwargs.get("target_w")
if isinstance(hf_processor_mm_kwargs.get("target_w"), int)
else None
)
if target_h is None or target_w is None:
source_grid_thw = mm_processed_data.get("image_grid_thw")
if source_grid_thw is not None and isinstance(source_grid_thw, torch.Tensor):
target_h = (
hf_processor_mm_kwargs.get("height")
if isinstance(hf_processor_mm_kwargs.get("height"), int)
else 1024
hf_processor_mm_kwargs.get("target_h")
if isinstance(hf_processor_mm_kwargs.get("target_h"), int)
else None
)
target_w = (
hf_processor_mm_kwargs.get("width")
if isinstance(hf_processor_mm_kwargs.get("width"), int)
else 1024
hf_processor_mm_kwargs.get("target_w")
if isinstance(hf_processor_mm_kwargs.get("target_w"), int)
else None
)
if target_h is None or target_w is None:
target_h = (
hf_processor_mm_kwargs.get("height")
if isinstance(hf_processor_mm_kwargs.get("height"), int)
else 1024
)
target_w = (
hf_processor_mm_kwargs.get("width")
if isinstance(hf_processor_mm_kwargs.get("width"), int)
else 1024
)
factor = 32
target_h = (target_h // factor) * factor
target_w = (target_w // factor) * factor
token_h = target_h // factor
token_w = target_w // factor
expanded_prompt = f"{prompt}{grid_bos}{token_h} {token_w}{grid_eos}{bos}"
text_ids = tokenizer.encode(expanded_prompt, add_special_tokens=False)
factor = 32
token_h = max(1, target_h // factor)
token_w = max(1, target_w // factor)
target_grid = torch.tensor([[1, token_h, token_w]], dtype=source_grid_thw.dtype)
mm_processed_data["mrope_image_grid_thw"] = torch.cat([source_grid_thw, target_grid], dim=0)
except Exception:
text_ids = tokenizer.encode(prompt, add_special_tokens=False)
pass
# Prompt updates will expand image placeholders in this compatibility path.
return prompt_ids, mm_processed_data, False
images = mm_items.get_items("image", ImageProcessorItems)
image_list = [images.get(i) for i in range(images.get_count())]
if not image_list:
raise ValueError("GLM-Image i2i requires at least one source image in mm_items")
hf_inputs = self._call_hf_processor(
prompt=prompt,
mm_data={"images": image_list},
mm_kwargs=hf_processor_mm_kwargs,
tok_kwargs=tokenization_kwargs,
)
input_ids = hf_inputs.get("input_ids")
if input_ids is None:
raise ValueError("HF i2i processor returned no input_ids")
if isinstance(input_ids, torch.Tensor):
prompt_ids = input_ids[0].tolist() if input_ids.dim() > 1 else input_ids.tolist()
else:
text_ids = list(prompt)
prompt_ids = (
input_ids[0]
if isinstance(input_ids, list) and input_ids and isinstance(input_ids[0], list)
else list(input_ids)
)
# Prepend image placeholders - one per image
prompt_ids = [image_token_id] * num_images + text_ids
mm_processed_data = BatchFeature(dict(), tensor_type="pt")
for key in ("pixel_values", "image_grid_thw", "mrope_image_grid_thw"):
value = hf_inputs.get(key)
if value is not None:
mm_processed_data[key] = value
logger.debug(f"_apply_hf_processor_main: built prompt_ids with {num_images} image placeholders")
image_grid_thw = mm_processed_data.get("image_grid_thw")
mrope_grid_thw = mm_processed_data.get("mrope_image_grid_thw")
hf_config = self.info.get_hf_config()
image_token_id = getattr(hf_config, "image_token_id", 167855)
image_token_count = prompt_ids.count(image_token_id)
logger.info(
"_apply_hf_processor_main i2i(HF): num_images=%s, prompt_len=%s, image_token_count=%s, "
"source_grid_shape=%s, mrope_grid_shape=%s",
num_images,
len(prompt_ids),
image_token_count,
tuple(image_grid_thw.shape) if image_grid_thw is not None else None,
tuple(mrope_grid_thw.shape) if mrope_grid_thw is not None else None,
)
# Return is_update_applied=False so _apply_prompt_updates will expand the placeholders
return prompt_ids, mm_processed_data, False
# HF processor already expanded image placeholders in input_ids.
return prompt_ids, mm_processed_data, True
def _get_mm_fields_config(
self,
@@ -2667,9 +2751,23 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP
# Input format: "text<sop>H W<eop><sop>h w<eop><bos>" where <bos>=image_start_token_id=16384
# For 1024x1024: H=32, W=32 (large), h=16, w=16 (small preview)
if not image_grid_thw:
# Preferred path for t2i: use explicit target size propagated from
# serving/request sampling params. This avoids fragile grid parsing
# from token IDs and matches HF processor grid construction.
target_h = kwargs.get("target_h")
target_w = kwargs.get("target_w")
if isinstance(target_h, int) and isinstance(target_w, int) and target_h > 0 and target_w > 0:
factor = 32
token_h = target_h // factor
token_w = target_w // factor
ratio = token_h / token_w if token_w > 0 else 1.0
small_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
small_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
image_grid_thw = [[1, token_h, token_w], [1, small_h, small_w]]
# Try to parse from kwargs (passed from processor)
hf_config_arg = kwargs.get("hf_config")
if hf_config_arg is not None and hasattr(hf_config_arg, "image_grid_thw"):
if (not image_grid_thw) and hf_config_arg is not None and hasattr(hf_config_arg, "image_grid_thw"):
image_grid_thw = hf_config_arg.image_grid_thw
# If still empty, try to infer from input tokens
@@ -2723,19 +2821,29 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP
prompt_ends_with_start = len(input_tokens) > 0 and input_tokens[-1] == image_start_token_id
if prompt_ends_with_start and len(image_grid_thw) == num_source_images and num_source_images > 0:
# i2i mode: source grids exist but no target grids
# Parse target grids from prompt tokens or use defaults
parsed_grids = self._parse_grid_from_tokens(input_tokens, hf_config)
if parsed_grids:
# parsed_grids contains all grids mentioned in prompt
# For i2i, add only the generation target grids
if len(parsed_grids) > num_source_images:
image_grid_thw = list(image_grid_thw) + parsed_grids[num_source_images:]
else:
# Fallback: add default 1024x1024 generation grids (1 target for i2i)
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
# Prefer explicit target size propagated from request sampling params.
# This avoids fragile grid parsing from token IDs for non-1024 i2i.
target_h = kwargs.get("target_h")
target_w = kwargs.get("target_w")
if isinstance(target_h, int) and isinstance(target_w, int) and target_h > 0 and target_w > 0:
factor = 32
token_h = target_h // factor
token_w = target_w // factor
image_grid_thw = list(image_grid_thw) + [[1, token_h, token_w]]
else:
# Fallback to default 1024x1024 grids for generation
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
# Parse target grids from prompt tokens or use defaults
parsed_grids = self._parse_grid_from_tokens(input_tokens, hf_config)
if parsed_grids:
# parsed_grids contains all grids mentioned in prompt
# For i2i, add only the generation target grids
if len(parsed_grids) > num_source_images:
image_grid_thw = list(image_grid_thw) + parsed_grids[num_source_images:]
else:
# Fallback: add default 1024x1024 generation grid (1 target for i2i)
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
else:
# Fallback to default 1024x1024 grid for generation
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
llm_pos_ids_list: list[torch.Tensor] = []
@@ -77,7 +77,9 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.utils.tensor_schema import TensorSchema
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.models.hunyuan_image3.autoencoder_kl_3d import AutoencoderKLConv3D
from vllm_omni.model_executor.models.hunyuan_image3.siglip2 import LightProjector, Siglip2VisionTransformer
@@ -175,8 +177,11 @@ class HunyuanModel(HunYuanModel):
return True
return False
skipped_unexpected: set[str] = set()
for name, loaded_weight in weights:
if contains_unexpected_keyword(name, unexpected_keywords):
skipped_unexpected.add(name)
continue
if "rotary_emb.inv_freq" in name:
@@ -362,6 +367,17 @@ class HunyuanModel(HunYuanModel):
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if skipped_unexpected:
logger.warning_once(
"Skipped %d weights matching unexpected_keywords "
"(e.g. vae, vision_model, patch_embed, timestep_emb). "
"If upstream renamed components, these may be silently "
"lost. Skipped names: %s",
len(skipped_unexpected),
sorted(skipped_unexpected)[:10],
)
return loaded_params
@@ -1149,6 +1165,8 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
HunyuanImage3Inputs: TypeAlias = HunyuanImage3PixelInputs
prefer_model_sampler = True
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -1199,6 +1217,10 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
else:
self.lm_head = PPMissingLayer()
# --- AR-stage components ---
# These are needed for image encoding in the AR stage.
# If a future text-only stage is added, gate on vllm_config.model_config.model_stage.
# vae
self.vae = AutoencoderKLConv3D.from_config(config.vae)
self.patch_embed = UNetDown(
@@ -1226,6 +1248,63 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
self._mrope_joint_img_sep_token_id = tokenizer.convert_tokens_to_ids("<joint_img_sep>")
self._mrope_max_num_patches = config.vit_processor.get("max_num_patches", 729)
# Special token IDs for logits processors (stage transitions).
# These mirror the official tokenization_hunyuan_image_3.py setup.
self._end_of_think_id = tokenizer.convert_tokens_to_ids("</think>")
self._recaption_id = tokenizer.convert_tokens_to_ids("<recaption>")
self._end_of_recaption_id = tokenizer.convert_tokens_to_ids("</recaption>")
self._answer_id = tokenizer.convert_tokens_to_ids("<answer>")
self._end_of_answer_id = tokenizer.convert_tokens_to_ids("</answer>")
image_base_size = getattr(config, "image_base_size", 1024)
self._size_token_id = tokenizer.convert_tokens_to_ids(f"<img_size_{image_base_size}>")
self._start_ratio_id = tokenizer.convert_tokens_to_ids("<img_ratio_0>")
self._end_ratio_id = tokenizer.convert_tokens_to_ids("<img_ratio_32>")
ratio_33 = tokenizer.convert_tokens_to_ids("<img_ratio_33>")
ratio_36 = tokenizer.convert_tokens_to_ids("<img_ratio_36>")
self._ratio_other_slices = [(ratio_33, ratio_36 + 1)]
# Build the full set of ratio token IDs for use as stop tokens.
self._all_ratio_ids = set(range(self._start_ratio_id, self._end_ratio_id + 1))
for s, e in self._ratio_other_slices:
self._all_ratio_ids.update(range(s, e))
# Determine mode: comprehension (I2T/T2T) vs generation (IT2I/T2I).
engine_output_type = getattr(vllm_config.model_config, "engine_output_type", None)
self._is_comprehension = engine_output_type in (None, "text")
# For comprehension mode, block image generation tokens but allow
# text structure tokens (<think>, <answer>, etc.) so the model can
# follow its natural generation pattern. Stop tokens in YAML will
# terminate at </answer> or EOS.
self._blocked_token_ids: set[int] = set()
if self._is_comprehension:
self._blocked_token_ids.update(
[
self._mrope_boi_token_id, # <boi>
self._mrope_eoi_token_id, # <eoi>
self._size_token_id, # <img_size_*>
]
)
self._blocked_token_ids.update(self._all_ratio_ids)
# For generation mode, build stage transition map.
# Official logic: </think> → [<recaption>],
# </recaption> → [<answer>, <boi>, <img_size_*>]
# After <img_size_*>, restrict vocab to ratio tokens only.
# Stage-transition forced sequences, keyed by trigger token.
self._stage_transitions: dict[int, list[int]] = {}
if not self._is_comprehension:
self._stage_transitions[self._end_of_think_id] = [
self._recaption_id,
]
self._stage_transitions[self._end_of_recaption_id] = [
self._answer_id,
self._mrope_boi_token_id,
self._size_token_id,
]
self._sampler: Sampler | None = None
self._eos_token_id: int = tokenizer.eos_token_id
self._replace_rotary_embeddings()
def _replace_rotary_embeddings(self):
@@ -1257,6 +1336,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
head_dim,
rope_theta,
)
if replaced == 0:
raise RuntimeError(
"HunyuanImage3: _replace_rotary_embeddings replaced 0 layers. "
"The custom interleaved 2D mRoPE is not active — model outputs "
"will be incorrect. Check that model.layers[*].self_attn.rotary_emb exists."
)
def _parse_and_validate_image_input(
self,
@@ -1274,6 +1359,10 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
if vit_pixel_values is None or vae_pixel_values is None:
return None
# Handle empty batch (e.g., during profiling with 0 images / T2T mode)
if vit_pixel_values.numel() == 0 or vae_pixel_values.numel() == 0:
return None
return HunyuanImage3PixelInputs(
type="pixel_values",
pixel_values={
@@ -1472,6 +1561,112 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
# ------------------------------------------------------------------
# Custom sampler — applies HunyuanImage3-specific logits processors
# before the standard sampling step.
#
# Comprehension (I2T / T2T):
# Block generation-specific special tokens so sampling can't
# accidentally produce <answer>, <boi>, ratio tokens, etc.
#
# Generation (IT2I / T2I think):
# 1. _StageTransitionLogitsProcessor — force token sequences at
# transition boundaries (</think> → <recaption>, etc.)
# 2. _ConditionalSliceVocabLogitsProcessor — after <img_size_*>,
# restrict vocab to ratio tokens only (greedy).
# ------------------------------------------------------------------
def sample(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> SamplerOutput | None:
if logits is None or logits.numel() == 0:
return None
if self._sampler is None:
self._sampler = Sampler()
min_score = torch.finfo(logits.dtype).min
assert logits.shape[0] == 1, f"HunyuanImage3 sampler requires max_num_seqs=1, got batch size {logits.shape[0]}"
for req_idx in range(logits.shape[0]):
decoded_tokens: list[int] = (
sampling_metadata.output_token_ids[req_idx] if req_idx < len(sampling_metadata.output_token_ids) else []
)
last_token = decoded_tokens[-1] if decoded_tokens else -1
if self._is_comprehension:
for tid in self._blocked_token_ids:
logits[req_idx, tid] = min_score
else:
forced = self._get_forced_token(decoded_tokens)
if forced is not None:
logits[req_idx].fill_(min_score)
logits[req_idx, forced] = 0
elif last_token == self._size_token_id:
self._apply_ratio_restriction(logits, req_idx, min_score)
elif last_token in self._all_ratio_ids:
logits[req_idx].fill_(min_score)
logits[req_idx, self._eos_token_id] = 0
return self._sampler(logits=logits, sampling_metadata=sampling_metadata)
def _get_forced_token(self, decoded_tokens: list[int]) -> int | None:
"""Derive the next forced token from output history (stateless).
Scans decoded_tokens backwards for the most recent trigger token,
then prefix-matches the forced sequence against what followed.
Returns the next token to force, or None if the sequence is complete
or history has diverged from the expected forced sequence.
"""
for i in range(len(decoded_tokens) - 1, -1, -1):
trigger = decoded_tokens[i]
if trigger not in self._stage_transitions:
continue
forced_seq = self._stage_transitions[trigger]
emitted = decoded_tokens[i + 1 :]
matched = 0
for expected, actual in zip(forced_seq, emitted):
if actual != expected:
# History diverged from the expected forced sequence.
# Stop applying transition forcing for safety.
return None
matched += 1
if matched < len(forced_seq):
return forced_seq[matched]
return None
return None
def _apply_ratio_restriction(
self,
logits: torch.Tensor,
req_idx: int,
min_score: float,
) -> None:
"""Port of official _ConditionalSliceVocabLogitsProcessor.__call__.
After the size token, only allow ratio tokens and pick greedily.
"""
original = logits[req_idx].clone()
logits[req_idx].fill_(min_score)
# Allow primary ratio range.
logits[req_idx, self._start_ratio_id : self._end_ratio_id + 1] = original[
self._start_ratio_id : self._end_ratio_id + 1
]
# Allow extra ratio slices.
for s, e in self._ratio_other_slices:
logits[req_idx, s:e] = original[s:e]
# Force greedy: keep only the argmax.
max_id = logits[req_idx].argmax().item()
logits[req_idx].fill_(min_score)
logits[req_idx, max_id] = 0
def make_empty_intermediate_tensors(
self, batch_size: int, dtype: torch.dtype, device: torch.device
) -> IntermediateTensors:
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiMo Audio pipeline topology (frozen).
Stage 0: fused thinker+talker — multimodal understanding + text + RVQ codes.
Stage 1: Code2Wav — RVQ codes → waveform.
MiMoAudioConfig inherits from Qwen2Config, so the HF ``model_type`` field
reports ``qwen2`` — the registry's model_type-based auto-detect can't
disambiguate. ``hf_architectures`` lets ``StageConfigFactory`` fall back to
matching ``hf_config.architectures`` instead.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.mimo_audio"
MIMO_AUDIO_PIPELINE = PipelineConfig(
model_type="mimo_audio",
# HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in
# ``model_executor/models/registry.py``; it resolves to the internal
# class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``.
model_arch="MiMoAudioModel",
hf_architectures=("MiMoAudioModel",),
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="fused_thinker_talker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sync_process_input_func=f"{_PROC}.llm2code2wav",
sampling_constraints={"detokenize": False},
),
),
)
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
from .ming_flash_omni import MingFlashOmniForConditionalGeneration
from .ming_flash_omni_thinker import (
MingFlashOmniThinkerDummyInputsBuilder,
MingFlashOmniThinkerForConditionalGeneration,
MingFlashOmniThinkerMultiModalProcessor,
MingFlashOmniThinkerProcessingInfo,
)
__all__ = [
"MingFlashOmniForConditionalGeneration",
"MingFlashOmniThinkerForConditionalGeneration",
"MingFlashOmniThinkerProcessingInfo",
"MingFlashOmniThinkerMultiModalProcessor",
"MingFlashOmniThinkerDummyInputsBuilder",
]
@@ -0,0 +1,246 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright 2024 ANT Group and the HuggingFace Inc. team.
# Copyright (c) 2022 OpenAI
# Adapted from Ming repository modeling_whisper_encoder.py
# https://github.com/inclusionAI/Ming
import operator
from collections.abc import Iterable
from itertools import accumulate
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func
from vllm_omni.model_executor.models.whisper_utils import Conv1d, Linear, sinusoids
logger = init_logger(__name__)
class MultiHeadAttention(nn.Module):
"""Multi-head attention with packed sequence support.
Adapted from Qwen3-TTS WhisperEncoder.
"""
def __init__(self, n_state: int, n_head: int, use_flash_attn: bool = True):
super().__init__()
self.n_head = n_head
self.query = Linear(n_state, n_state)
self.key = Linear(n_state, n_state, bias=False)
self.value = Linear(n_state, n_state)
self.out = Linear(n_state, n_state)
if use_flash_attn and not HAS_FLASH_ATTN:
logger.warning("flash-attn is not available. Fallback to manual PyTorch version")
self.use_flash_attn = use_flash_attn and HAS_FLASH_ATTN
def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
"""Forward pass with packed sequence support.
Args:
x: [total_tokens, n_state] packed sequence
cu_seqlens: [num_seqs + 1] cumulative sequence lengths, e.g. [0, len1, len1+len2, ...]
Returns:
[total_tokens, n_state] attention output
"""
q = self.query(x)
k = self.key(x)
v = self.value(x)
n_ctx, n_state = q.shape
head_dim = n_state // self.n_head
q = q.view(n_ctx, self.n_head, head_dim)
k = k.view(n_ctx, self.n_head, head_dim)
v = v.view(n_ctx, self.n_head, head_dim)
# Try flash attention varlen
if self.use_flash_attn and cu_seqlens is not None and q.dtype in [torch.float16, torch.bfloat16]:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
attn_output = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen)
else:
attn_output = self._manual_attention(q, k, v, cu_seqlens)
# Reshape back: [T, H, D] -> [T, H*D]
attn_output = attn_output.contiguous().view(n_ctx, n_state)
return self.out(attn_output)
def _manual_attention(
self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, cu_seqlens: torch.Tensor
) -> torch.Tensor:
"""Manual attention for variable-length sequences (fallback)."""
_, n_head, head_dim = q.shape
scale = head_dim**-0.5
# Unpack sequences and pad to max length
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
batch_size = len(seqlens)
max_seqlen = max(seqlens)
# Create padded tensors
q_padded = torch.zeros(batch_size, max_seqlen, n_head, head_dim, dtype=q.dtype, device=q.device)
k_padded = torch.zeros_like(q_padded)
v_padded = torch.zeros_like(q_padded)
# Fill with actual sequences
for i in range(batch_size):
start_idx = cu_seqlens[i]
end_idx = cu_seqlens[i + 1]
seq_len = seqlens[i]
q_padded[i, :seq_len] = q[start_idx:end_idx]
k_padded[i, :seq_len] = k[start_idx:end_idx]
v_padded[i, :seq_len] = v[start_idx:end_idx]
# Transpose for attention: [B, H, T, D]
q_padded = q_padded.transpose(1, 2)
k_padded = k_padded.transpose(1, 2)
v_padded = v_padded.transpose(1, 2)
# Create attention mask for variable lengths: 0 for valid positions, -inf for padding
padding_mask = (
torch.arange(max_seqlen, device=q.device)[None, :] >= torch.tensor(seqlens, device=q.device)[:, None]
)
attn_mask = torch.zeros(batch_size, 1, 1, max_seqlen, dtype=q.dtype, device=q.device)
attn_mask = attn_mask.masked_fill(padding_mask.unsqueeze(1).unsqueeze(2), -torch.finfo(q.dtype).max)
# Compute attention
attn_scores = torch.matmul(q_padded, k_padded.transpose(-2, -1)) * scale
attn_scores = attn_scores + attn_mask
attn_weights = F.softmax(attn_scores, dim=-1)
context = torch.matmul(attn_weights, v_padded)
# Transpose back: [B, H, T, D] -> [B, T, H, D]
context = context.transpose(1, 2).contiguous()
output_packed = torch.cat([context[i, : seqlens[i]] for i in range(batch_size)], dim=0)
return output_packed
class ResidualAttentionBlock(nn.Module):
"""Whisper-style residual attention block with packed sequence support.
Adapted from
https://github.com/openai/whisper/blob/v20250625/whisper/model.py
vllm_omni/model_executor/models/qwen3_tts/tokenizer_25hz/vq/whisper_encoder.py
"""
def __init__(self, n_state: int, n_head: int, use_flash_attn: bool = True):
super().__init__()
self.attn = MultiHeadAttention(n_state, n_head, use_flash_attn=use_flash_attn)
self.attn_ln = nn.LayerNorm(n_state)
n_mlp = n_state * 4
self.mlp = nn.Sequential(
Linear(n_state, n_mlp),
nn.GELU(),
Linear(n_mlp, n_state),
)
self.mlp_ln = nn.LayerNorm(n_state)
def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
x = x + self.attn(self.attn_ln(x), cu_seqlens=cu_seqlens)
x = x + self.mlp(self.mlp_ln(x))
return x
class WhisperAudioEncoder(nn.Module):
"""Whisper audio encoder for Ming with packed sequence support.
Adapted from
https://github.com/openai/whisper/blob/v20250625/whisper/model.py
vllm_omni/model_executor/models/qwen3_tts/tokenizer_25hz/vq/whisper_encoder.py
"""
def __init__(
self,
n_mels: int = 128,
n_ctx: int = 15000,
n_state: int = 1280,
n_head: int = 20,
n_layer: int = 32,
use_flash_attn: bool = True,
):
super().__init__()
self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
# self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state))
self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state))
self.blocks = nn.ModuleList(
[ResidualAttentionBlock(n_state, n_head, use_flash_attn=use_flash_attn) for _ in range(n_layer)]
)
self.ln_post = nn.LayerNorm(n_state)
self.audio_emb_dim = n_state
self.n_layer = n_layer
self.n_mels = n_mels
self.use_flash_attn = use_flash_attn
def forward(
self,
x_list: list[torch.Tensor],
audio_lens: list[int],
) -> torch.Tensor:
"""Forward pass with packed sequence format for variable-length inputs.
Args:
x_list: List of [n_mels, T_i] mel spectrogram features for each audio
audio_lens: List of original audio lengths in frames
Returns:
[total_T', n_state] packed encoded audio features, where
total_T' is the sum of all encoded sequence lengths
"""
# Cast inputs to model dtype
target_dtype = self.conv1.weight.dtype
x_list = [x.to(target_dtype) for x in x_list]
encoded_list = []
encoded_lens = []
for mel_spec in x_list:
# mel_spec: [n_mels, T] - process through conv layers
x = mel_spec.unsqueeze(0) # [1, n_mels, T]
x = F.gelu(self.conv1(x))
x = F.gelu(self.conv2(x))
x = x.squeeze(0).transpose(0, 1) # [T', n_state]
# Add positional embedding
seq_len = x.shape[0]
positional_embedding = self.positional_embedding[:seq_len, :]
x = (x + positional_embedding).to(x.dtype)
encoded_list.append(x)
encoded_lens.append(seq_len)
x_packed = torch.cat(encoded_list, dim=0) # [total_T', n_state]
cu_seqlens = list(accumulate(encoded_lens, func=operator.add, initial=0))
cu_seqlens = torch.tensor(cu_seqlens, device=x_packed.device, dtype=torch.int32)
for block in self.blocks:
x_packed = block(x_packed, cu_seqlens=cu_seqlens)
x_packed = self.ln_post(x_packed)
return x_packed
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict: dict[str, torch.Tensor] = {
**dict(self.named_parameters(remove_duplicate=False)),
**dict(self.named_buffers()),
}
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if name not in params_dict:
logger.warning("Skipping unknown audio encoder weight: %s", name)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -0,0 +1,223 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright 2024 ANT Group and the HuggingFace Inc. team. All rights reserved.
# Adapted from Ming repository modeling_bailingmm2.py
# https://github.com/inclusionAI/Ming
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Ming-flash-omni-2.0 unified model (thinker + imagegen + talker)."""
from collections.abc import Iterable
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.utils import (
init_vllm_registered_model,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config, MingFlashOmniConfig
from .ming_flash_omni_thinker import (
MingFlashOmniThinkerDummyInputsBuilder,
MingFlashOmniThinkerMultiModalProcessor,
MingFlashOmniThinkerProcessingInfo,
)
logger = init_logger(__name__)
@MULTIMODAL_REGISTRY.register_processor(
MingFlashOmniThinkerMultiModalProcessor,
info=MingFlashOmniThinkerProcessingInfo,
dummy_inputs=MingFlashOmniThinkerDummyInputsBuilder,
)
class MingFlashOmniForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsPP,
SupportsMRoPE,
CustomProcessMixin,
):
"""Unified Ming-flash-omni-2.0 model combining thinker, imagegen, and talker."""
supports_multimodal = True
requires_raw_input_tokens: bool = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
self.config = config
if isinstance(config, MingFlashOmniConfig):
thinker_config = config.thinker_config
else:
thinker_config = config
self.thinker_config: BailingMM2Config = thinker_config
self.model_stage = vllm_config.model_config.model_stage
if self.model_stage == "thinker":
thinker_vllm_config = vllm_config.with_hf_config(
thinker_config, architectures=["MingFlashOmniThinkerForConditionalGeneration"]
)
self.thinker = init_vllm_registered_model(
vllm_config=thinker_vllm_config,
prefix=maybe_prefix(prefix, "thinker"),
architectures=["MingFlashOmniThinkerForConditionalGeneration"],
)
self.model = self.thinker
self.imagegen = None
self.talker = None
elif self.model_stage == "imagegen":
# TODO: Implement image generator stage
raise NotImplementedError(
"Image generation stage is not yet implemented. Please use model_stage='thinker' for now."
)
elif self.model_stage == "talker":
# TODO: Implement talker (TTS) stage
raise NotImplementedError(
"Talker (TTS) stage is not yet implemented. Please use model_stage='thinker' for now."
)
else:
raise ValueError(
f"Invalid model_stage: {self.model_stage}. Must be one of: 'thinker', 'imagegen', 'talker'"
)
# Set up intermediate tensors
self.make_empty_intermediate_tensors = (
self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> OmniOutput:
return self.model.forward(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
**kwargs,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
sampling_metadata=None,
) -> torch.Tensor | None:
if hasattr(self.model, "compute_logits"):
return self.model.compute_logits(hidden_states, sampling_metadata)
return None
def sample(
self,
logits: torch.Tensor,
sampling_metadata,
):
if hasattr(self.model, "sample"):
return self.model.sample(logits, sampling_metadata)
raise NotImplementedError("sample method not available on current stage")
def get_mrope_input_positions(self, *args, **kwargs):
if hasattr(self.model, "get_mrope_input_positions"):
return self.model.get_mrope_input_positions(*args, **kwargs)
raise NotImplementedError("get_mrope_input_positions not available on current stage")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loaded_weights = set()
thinker_weights = []
imagegen_weights = []
talker_weights = []
for name, value in weights:
if name.startswith("thinker."):
thinker_weights.append((name, value))
elif name.startswith("imagegen."):
imagegen_weights.append((name, value))
elif name.startswith("talker."):
talker_weights.append((name, value))
else:
# Weights without prefix go to thinker by default
thinker_weights.append((name, value))
if self.model_stage == "thinker" and thinker_weights:
# Remove "thinker." prefix before loading
thinker_weights_stripped = [
(name.replace("thinker.", "", 1) if name.startswith("thinker.") else name, value)
for name, value in thinker_weights
]
thinker_loaded = self.thinker.load_weights(thinker_weights_stripped)
thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
loaded_weights.update(thinker_loaded)
# TODO: Load imagegen weights when implemented
# TODO: Load talker weights when implemented
return loaded_weights
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
language_model="thinker.language_model",
connector=["thinker.linear_proj.", "thinker.linear_proj_audio."],
tower_model=["thinker.vision.", "thinker.audio."],
)
@property
def sampler(self):
if hasattr(self.model, "sampler"):
return self.model.sampler
return None
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings=None,
*,
is_multimodal=None,
) -> torch.Tensor:
return self.model.embed_input_ids(
input_ids,
multimodal_embeddings,
is_multimodal=is_multimodal,
)
def embed_multimodal(self, **kwargs):
return self.model.embed_multimodal(**kwargs)
@@ -0,0 +1,893 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright 2024 ANT Group and the HuggingFace Inc. team.
# Adapted from Ming repository modeling_bailingmm2.py and processing_bailingmm2.py
# https://github.com/inclusionAI/Ming
"""Ming-flash-omni-2.0 Thinker stage implementation (multimodal understanding)."""
from collections.abc import Iterable, Iterator, Mapping, Sequence
from typing import Annotated, Any
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers.feature_extraction_utils import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.qwen2_5_vl import (
Qwen2_5_VLImageInputs,
Qwen2_5_VLImagePixelInputs,
Qwen2_5_VLVideoInputs,
Qwen2_5_VLVideoPixelInputs,
)
from vllm.model_executor.models.qwen2_vl import (
Qwen2VLProcessingInfo,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
_merge_multimodal_embeddings,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
AudioProcessorItems,
ImageProcessorItems,
MultiModalDataItems,
MultiModalDataParser,
VideoProcessorItems,
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config
from vllm_omni.transformers_utils.processors.ming import (
PLACEHOLDER_AUDIO_TOKEN_IN_TEXT,
PLACEHOLDER_IMAGE_TOKEN_IN_TEXT,
PLACEHOLDER_VIDEO_TOKEN_IN_TEXT,
MingFlashOmniProcessor,
MingWhisperFeatureExtractor,
)
from .audio_encoder import WhisperAudioEncoder
from .modeling_bailing_moe_v2 import BailingMoeV2ForCausalLM
from .projectors import AudioProjector, VisionProjector
from .vision_encoder import MingVisionEncoder
logger = init_logger(__name__)
class MingAudioInput(TensorSchema):
"""
Dimensions:
- b: Batch size
- l: Total audio frames (clips concatenated along the time axis)
- nm: Number of mel bins
- N: Max number of audio clips per batch item
"""
audio_feats: Annotated[
torch.Tensor,
TensorShape("b", "l", "nm"),
]
audio_feats_lengths: Annotated[
torch.Tensor,
TensorShape("b", "N"),
]
class MingFlashOmniThinkerProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self) -> BailingMM2Config:
return self.ctx.get_hf_config(BailingMM2Config)
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(MingFlashOmniProcessor, **kwargs)
def get_target_channels(self) -> int:
# See `_normalize_audio_tensor` in vllm_omni/transformers_utils/processors/ming.py
return 1
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None, "audio": None}
def get_mm_max_tokens_per_item(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
mm_counts = mm_counts or {}
requested_modalities = {m for m, c in mm_counts.items() if c > 0}
mm_max_tokens: dict[str, int] = {}
if requested_modalities & {"image", "video"}:
vl_tokens = super().get_mm_max_tokens_per_item(
seq_len=seq_len,
mm_counts=mm_counts,
)
mm_max_tokens.update({m: vl_tokens[m] for m in ["image", "video"] if m in requested_modalities})
if "audio" in requested_modalities:
# TODO: consider computing from audio config
mm_max_tokens["audio"] = 3000
return mm_max_tokens
def get_feature_extractor(self, **kwargs: object) -> MingWhisperFeatureExtractor:
hf_processor = self.get_hf_processor(**kwargs)
feature_extractor = hf_processor.audio_processor
assert isinstance(feature_extractor, MingWhisperFeatureExtractor)
return feature_extractor
def get_data_parser(self):
feature_extractor = self.get_feature_extractor()
return MultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
)
class MingFlashOmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[MingFlashOmniThinkerProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
num_audios = mm_counts.get("audio", 0)
hf_processor = self.info.get_hf_processor()
audio_token: str = hf_processor.audio_token
image_token: str = hf_processor.image_token
video_token: str = hf_processor.video_token
return image_token * num_images + video_token * num_videos + audio_token * num_audios
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
num_audios = mm_counts.get("audio", 0)
# Default dimensions for dummy data
image_width, image_height = 448, 448
video_width, video_height = 448, 448
num_frames = 8
audio_duration = 3.0 # seconds
sample_rate = 16000
audio_length = int(audio_duration * sample_rate)
mm_data: MultiModalDataDict = {
"image": self._get_dummy_images(
width=image_width,
height=image_height,
num_images=num_images,
),
"video": self._get_dummy_videos(
width=video_width,
height=video_height,
num_frames=num_frames,
num_videos=num_videos,
),
"audio": [(np.random.randn(audio_length).astype(np.float32), sample_rate) for _ in range(num_audios)],
}
return mm_data
class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashOmniThinkerProcessingInfo]):
"""Multimodal processor for Ming-flash-omni Thinker stage.
Handles preprocessing of 1) image, 2) video, and 3) audio inputs,
and expands placeholder tokens to the correct number of patch tokens.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
# might want to add a fallback to resolve token ids
# vocab = tokenizer.get_vocab()
thinker_config = self.info.get_hf_config()
# patch/delimiter token IDs (used in replacement sequences)
image_start_token_id = thinker_config.llm_config.image_start_token
image_patch_token_id = thinker_config.llm_config.image_patch_token
image_end_token_id = thinker_config.llm_config.image_end_token
video_start_token_id = thinker_config.llm_config.video_start_token
frame_patch_token_id = thinker_config.llm_config.video_patch_token
video_end_token_id = thinker_config.llm_config.video_end_token
audio_start_token_id = thinker_config.llm_config.audio_start_token
audio_patch_token_id = thinker_config.llm_config.audio_patch_token
audio_end_token_id = thinker_config.llm_config.audio_end_token
vision_config = thinker_config.vision_config
spatial_merge_size = vision_config.spatial_merge_size if vision_config else 2
newline_token_ids: list[int] = tokenizer.encode("\n", add_special_tokens=False)
out_mm_data = out_mm_kwargs.get_data()
def get_replacement_image(item_idx: int) -> PromptUpdateDetails:
"""Generate token sequence for an image."""
grid_thw = out_mm_data.get("image_grid_thw")
if grid_thw is None:
raise ValueError(
"image_grid_thw missing from processor output; "
"cannot determine image patch count for prompt replacement."
)
if isinstance(grid_thw, torch.Tensor):
thw = grid_thw[item_idx]
num_patches = int(thw.prod().item()) // (spatial_merge_size**2)
else:
thw = grid_thw[item_idx]
num_patches = (thw[0] * thw[1] * thw[2]) // (spatial_merge_size**2)
# Build token sequence: <image> <imagePatch>*N </image> \n
# the newline token is added in purpose from original model processing
tokens: list[int] = []
tokens.append(image_start_token_id)
tokens.extend([image_patch_token_id] * num_patches)
tokens.append(image_end_token_id)
# Refer to Ming's BailingMM2Processor._expand_image_tokens
# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/processing_bailingmm2.py
tokens.extend(newline_token_ids)
# Only <imagePatch> tokens receive multimodal embeddings
return PromptUpdateDetails.select_token_id(tokens, image_patch_token_id)
def get_replacement_video(item_idx: int) -> PromptUpdateDetails:
"""Generate token sequence for a video."""
grid_thw = out_mm_data.get("video_grid_thw", None)
if grid_thw is None:
raise ValueError(
"video_grid_thw missing from processor output; "
"cannot determine video patch count for prompt replacement."
)
if isinstance(grid_thw, torch.Tensor):
thw = grid_thw[item_idx]
num_patches = int(thw.prod().item()) // (spatial_merge_size**2)
else:
thw = grid_thw[item_idx]
num_patches = (thw[0] * thw[1] * thw[2]) // (spatial_merge_size**2)
# Build token sequence: <video> <framePatch>*N </video> \n
# the newline token is added in purpose from original model processing
tokens: list[int] = []
tokens.append(video_start_token_id)
tokens.extend([frame_patch_token_id] * num_patches)
tokens.append(video_end_token_id)
tokens.extend(newline_token_ids)
# Only <framePatch> tokens receive multimodal embeddings
return PromptUpdateDetails.select_token_id(tokens, frame_patch_token_id)
def get_replacement_audio(item_idx: int) -> PromptUpdateDetails:
"""Generate token sequence for an audio."""
encoder_feats_lengths = out_mm_data.get("encoder_feats_lengths", None)
if encoder_feats_lengths is None:
raise ValueError(
"encoder_feats_lengths missing from processor output; "
"cannot determine audio patch count for prompt replacement."
)
if isinstance(encoder_feats_lengths, torch.Tensor):
num_patches = int(encoder_feats_lengths[item_idx].item())
else:
num_patches = encoder_feats_lengths[item_idx]
# Build token sequence: <audio> <audioPatch>*N </audio>
tokens: list[int] = []
tokens.append(audio_start_token_id)
tokens.extend([audio_patch_token_id] * num_patches)
tokens.append(audio_end_token_id)
# Only <audioPatch> tokens receive multimodal embeddings
return PromptUpdateDetails.select_token_id(tokens, audio_patch_token_id)
# Build prompt updates and process replacement
updates: list[PromptUpdate] = []
if "image" in mm_items and mm_items.get_items("image", ImageProcessorItems):
updates.append(
PromptReplacement(
modality="image",
target=PLACEHOLDER_IMAGE_TOKEN_IN_TEXT,
replacement=get_replacement_image,
)
)
if "video" in mm_items and mm_items.get_items("video", VideoProcessorItems):
updates.append(
PromptReplacement(
modality="video",
target=PLACEHOLDER_VIDEO_TOKEN_IN_TEXT,
replacement=get_replacement_video,
)
)
if "audio" in mm_items and mm_items.get_items("audio", AudioProcessorItems):
updates.append(
PromptReplacement(
modality="audio",
target=PLACEHOLDER_AUDIO_TOKEN_IN_TEXT,
replacement=get_replacement_audio,
)
)
return updates
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
config: dict[str, MultiModalFieldConfig] = {}
# Image fields, pixel_values is flat (concatenated patches from all images)
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
if "pixel_values" in hf_inputs:
image_sizes = image_grid_thw.prod(-1)
config["pixel_values"] = MultiModalFieldConfig.flat_from_sizes(
"image",
image_sizes,
)
if "image_grid_thw" in hf_inputs:
config["image_grid_thw"] = MultiModalFieldConfig.batched("image")
# Video fields, same flat layout as images
video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
if "pixel_values_videos" in hf_inputs:
video_sizes = video_grid_thw.prod(-1)
config["pixel_values_videos"] = MultiModalFieldConfig.flat_from_sizes(
"video",
video_sizes,
)
if "video_grid_thw" in hf_inputs:
config["video_grid_thw"] = MultiModalFieldConfig.batched("video")
# Audio fields
if "audio_feats" in hf_inputs:
config["audio_feats"] = MultiModalFieldConfig.batched("audio")
if "audio_feats_lengths" in hf_inputs:
config["audio_feats_lengths"] = MultiModalFieldConfig.batched("audio")
if "encoder_feats_lengths" in hf_inputs:
config["encoder_feats_lengths"] = MultiModalFieldConfig.batched("audio")
if "placeholder_audio_loc_lens" in hf_inputs:
config["placeholder_audio_loc_lens"] = MultiModalFieldConfig.batched("audio")
return config
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call sub-processors for multimodal inputs and tokenize.
We call the image/audio sub-processors directly (instead of going
through `MingFlashOmniProcessor.__call__`) so that the high-level
placeholder tokens remain **unexpanded** in the tokenized output.
"""
hf_processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
data: dict[str, object] = {}
images = mm_data.get("images", None)
if images is not None:
image_outputs = hf_processor.image_processor(
images=images,
videos=None,
return_tensors="pt",
)
data.update(image_outputs)
videos = mm_data.get("videos", None)
if videos is not None:
video_outputs = hf_processor.image_processor(
images=None,
videos=videos,
return_tensors="pt",
)
# Rename keys to distinguish from images
if "pixel_values" in video_outputs:
video_outputs["pixel_values_videos"] = video_outputs.pop("pixel_values")
if "image_grid_thw" in video_outputs:
video_outputs["video_grid_thw"] = video_outputs.pop("image_grid_thw")
data.update(video_outputs)
audios = mm_data.get("audios", None)
if audios is not None:
# vLLM's AudioProcessorItems provides raw numpy arrays (already resampled).
# MingWhisperAudioProcessor expects (waveform, sr) tuples,
# so wrap them with the target sample rate.
target_sr = hf_processor.audio_processor.sampling_rate
audio_tuples = [(a, target_sr) if not isinstance(a, tuple) else a for a in audios]
audio_outputs = hf_processor.audio_processor(
audio_tuples,
return_tensors="pt",
)
data.update(audio_outputs)
# Tokenize text with placeholders still intact
text_outputs = tokenizer(prompt, return_tensors="pt", **tok_kwargs)
data.update(text_outputs)
return BatchFeature(data=data)
@MULTIMODAL_REGISTRY.register_processor(
MingFlashOmniThinkerMultiModalProcessor,
info=MingFlashOmniThinkerProcessingInfo,
dummy_inputs=MingFlashOmniThinkerDummyInputsBuilder,
)
class MingFlashOmniThinkerForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsPP,
SupportsMRoPE,
CustomProcessMixin,
):
"""Ming Thinker stage: multimodal understanding
(text + image + video + audio) -> text generation.
"""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={"model.": "language_model."},
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
# vllm_omni/transformers_utils/processors/ming.py
if modality.startswith("image"):
return "<IMAGE>"
elif modality.startswith("video"):
return "<VIDEO>"
elif modality.startswith("audio"):
return "<AUDIO>"
raise ValueError("Only image, video, or audio modality is supported")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
thinker_config: BailingMM2Config = config
if (
thinker_config.llm_config is None
or thinker_config.vision_config is None
or thinker_config.audio_config is None
):
raise ValueError(
"MingFlashOmniThinker requires `llm_config`, `vision_config`, and `audio_config` in `thinker_config`."
)
llm_config = thinker_config.llm_config
self.config = llm_config
self.thinker_config = thinker_config
self.have_multimodal_outputs = True
# Initialize LLM as a component
with self._mark_language_model(vllm_config):
llm_vllm_config = vllm_config.with_hf_config(llm_config)
self.language_model = BailingMoeV2ForCausalLM(
vllm_config=llm_vllm_config, prefix=maybe_prefix(prefix, "llm")
)
# Ming thinker is inherently multimodal; initialize both towers eagerly.
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.vision = MingVisionEncoder(
vision_config=thinker_config.vision_config,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "vision"),
)
self.linear_proj = VisionProjector(
vision_dim=self.vision.image_emb_dim,
llm_dim=llm_config.hidden_size,
mlp_depth=getattr(thinker_config, "mlp_depth", 2),
)
logger.info("Initialized MingVisionEncoder and VisionProjector")
audio_cfg = thinker_config.audio_config
whisper_cfg = getattr(audio_cfg, "whisper_encoder_config", {}) or {}
with self._mark_tower_model(vllm_config, "audio"):
self.audio = WhisperAudioEncoder(
**whisper_cfg,
use_flash_attn=True,
)
self.linear_proj_audio = AudioProjector(
audio_dim=self.audio.audio_emb_dim,
llm_dim=llm_config.hidden_size,
ds_kernel_size=getattr(audio_cfg, "ds_kernel_size", 3),
ds_stride=getattr(audio_cfg, "ds_stride", 2),
mlp_depth=getattr(thinker_config, "mlp_depth", 1),
)
logger.info("Initialized WhisperAudioEncoder and AudioProjector")
# Expose interfaces
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
logger.info("MingFlashOmniThinker initialized with vision and audio towers")
def extract_image_feature(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor:
"""Extract and project image features.
Args:
pixel_values: Flattened pixel values from vision processor.
grid_thw: [num_images, 3] tensor of (t, h, w) grid dimensions.
Returns:
[seq_len, hidden_size] L2-normalized image embeddings.
"""
if self.vision is None:
raise ValueError("Vision encoder not initialized")
with torch.amp.autocast(pixel_values.device.type, dtype=torch.bfloat16):
image_embeds = self.vision(pixel_values, grid_thw=grid_thw)
if self.vision.use_deepstack:
image_embeds = image_embeds[:, : self.vision.image_emb_dim]
image_embeds = self.linear_proj(image_embeds)
image_embeds = F.normalize(image_embeds, dim=-1)
return image_embeds
def extract_audio_feature(
self, audio_feats: torch.Tensor, audio_feats_lengths: torch.Tensor
) -> tuple[torch.Tensor, ...]:
"""Extract and project audio features.
Args:
audio_feats: [B, L_total, n_mels] wrapped mel features — multiple audio
clips per batch item are concatenated along the time dimension
(time-first, as produced by MingWhisperFeatureExtractor).
audio_feats_lengths: [B, N] lengths of each audio clip per batch item.
N is the max number of clips per item; zero-padded entries are skipped.
Returns:
Tuple of per-clip [T'_i, hidden_size] projected audio embeddings.
"""
if self.audio is None:
raise ValueError("Audio encoder not initialized")
# Unwrap packed [B, L_total, n_mels] into a list of [n_mels, T'_i] tensors,
# one per audio clip, as expected by WhisperAudioEncoder.
x_list: list[torch.Tensor] = []
audio_lens: list[int] = []
for i in range(audio_feats_lengths.shape[0]):
feat_index = 0
for j in range(audio_feats_lengths.shape[1]):
feat_len = int(audio_feats_lengths[i, j].item())
if feat_len == 0:
break
mel_seg = audio_feats[i, feat_index : feat_index + feat_len].transpose(0, 1)
x_list.append(mel_seg)
audio_lens.append(feat_len)
feat_index += feat_len
audio_packed = self.audio(x_list, audio_lens)
# Compute per-clip lengths after Whisper Conv1d (kernel=3, stride=2, pad=1)
encoded_lens = [(audio_len - 3 + 2) // 2 + 1 for audio_len in audio_lens]
# Project packed
proj_packed, proj_lens = self.linear_proj_audio.forward_packed(audio_packed, encoded_lens)
normalize = getattr(self.thinker_config.audio_config, "norm_query_embeds", False)
if normalize:
proj_packed = F.normalize(proj_packed, dim=-1)
proj_packed = proj_packed.to(audio_feats.dtype)
# Split into per-clip tensors
return proj_packed.split(proj_lens)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
"""Parse and validate multimodal kwargs into per-modality dicts."""
mm_input_by_modality: dict[str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | MingAudioInput] = {}
for key in kwargs:
if key == "pixel_values" and "image" not in mm_input_by_modality:
pixel_values = kwargs.get("pixel_values")
image_grid_thw = kwargs.get("image_grid_thw")
if pixel_values is not None and image_grid_thw is not None:
mm_input_by_modality["image"] = Qwen2_5_VLImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values, # type: ignore[arg-type]
image_grid_thw=image_grid_thw, # type: ignore[arg-type]
)
elif key == "pixel_values_videos" and "video" not in mm_input_by_modality:
pixel_values_videos = kwargs.get("pixel_values_videos")
video_grid_thw = kwargs.get("video_grid_thw")
second_per_grid_ts = kwargs.get("second_per_grid_ts")
if pixel_values_videos is not None and video_grid_thw is not None:
mm_input_by_modality["video"] = Qwen2_5_VLVideoPixelInputs(
type="pixel_values_videos",
pixel_values_videos=pixel_values_videos, # type: ignore[arg-type]
video_grid_thw=video_grid_thw, # type: ignore[arg-type]
second_per_grid_ts=second_per_grid_ts, # type: ignore[arg-type]
)
elif key == "audio_feats" and "audio" not in mm_input_by_modality:
audio_feats = kwargs.get("audio_feats")
audio_feats_lengths = kwargs.get("audio_feats_lengths")
if audio_feats is not None and audio_feats_lengths is not None:
mm_input_by_modality["audio"] = MingAudioInput(
audio_feats=audio_feats, # type: ignore[arg-type]
audio_feats_lengths=audio_feats_lengths, # type: ignore[arg-type]
)
return mm_input_by_modality
def _process_image_input(self, image_input: Qwen2_5_VLImageInputs) -> list[torch.Tensor]:
# Splits the flat [total_tokens, D] output of extract_image_feature
# into one tensor per image.
pixel_values = image_input["pixel_values"]
image_grid_thw = image_input["image_grid_thw"]
image_embeds = self.extract_image_feature(pixel_values, image_grid_thw)
merge_unit = self.thinker_config.vision_config.spatial_merge_size**2
sizes = (image_grid_thw.prod(dim=-1) // merge_unit).tolist()
return list(image_embeds.split([int(s) for s in sizes], dim=0))
def _process_video_input(self, video_input: Qwen2_5_VLVideoInputs) -> list[torch.Tensor]:
pixel_values_videos = video_input["pixel_values_videos"]
video_grid_thw = video_input["video_grid_thw"]
video_embeds = self.extract_image_feature(pixel_values_videos, video_grid_thw)
merge_unit = self.thinker_config.vision_config.spatial_merge_size**2
sizes = (video_grid_thw.prod(dim=-1) // merge_unit).tolist()
return list(video_embeds.split([int(s) for s in sizes], dim=0))
def _process_audio_input(self, audio_input: MingAudioInput) -> list[torch.Tensor]:
return list(self.extract_audio_feature(audio_input["audio_feats"], audio_input["audio_feats_lengths"]))
def _compute_modality_masks(self, input_ids: torch.Tensor) -> tuple[torch.Tensor | None, torch.Tensor | None]:
"""Compute vision and audio MoE-routing masks from input_ids.
Returns:
Tuple of (vision_mask, audio_mask), each shape [seq_len] bool.
"""
llm_config = self.config
# vision mask
vision_mask = torch.zeros_like(input_ids, dtype=torch.bool)
image_token = llm_config.image_patch_token
video_token = llm_config.video_patch_token
vision_mask = vision_mask | (input_ids == image_token)
vision_mask = vision_mask | (input_ids == video_token)
# audio mask
audio_token = llm_config.audio_patch_token
audio_mask = input_ids == audio_token
return vision_mask, audio_mask
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return []
# preserve the order of modalities
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
for modality, mm_input in mm_input_by_modality.items():
if modality == "image":
multimodal_embeddings += tuple(self._process_image_input(mm_input)) # type: ignore[arg-type]
elif modality == "video":
multimodal_embeddings += tuple(self._process_video_input(mm_input)) # type: ignore[arg-type]
elif modality == "audio":
multimodal_embeddings += tuple(self._process_audio_input(mm_input)) # type: ignore[arg-type]
return multimodal_embeddings
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
inputs_embeds = self.language_model.model.word_embeddings(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
assert is_multimodal is not None, "`is_multimodal` mask required when `multimodal_embeddings` provided"
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> OmniOutput:
# Compute MoE modality masks on every device
image_mask, audio_mask = self._compute_modality_masks(input_ids)
hidden_states = self.language_model.forward(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
image_mask=image_mask,
audio_mask=audio_mask,
)
# Capture embeddings for downstream stages
multimodal_outputs = {
"final_hidden_states": hidden_states,
}
return OmniOutput(
text_hidden_states=hidden_states,
multimodal_outputs=multimodal_outputs,
)
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states, sampling_metadata)
def sample(self, logits: torch.Tensor, sampling_metadata):
return self.language_model.sample(logits, sampling_metadata)
@property
def sampler(self):
return self.language_model.sampler
def iter_mm_features(
self,
mm_features: list[MultiModalFeatureSpec],
) -> Iterator[tuple[int, str, dict[str, Any]]]:
"""Iterate over image/video features sorted by token position.
Yields: (offset, modality, feature_data) where feature_data contains:
- image: {"grid_t", "grid_h", "grid_w", "second_per_grid_t"}
- video: {"grid_t", "grid_h", "grid_w", "second_per_grid_t"}
Audio features are not yielded: Ming assigns them sequential
text positions (same T/H/W value) rather than 3D grid positions.
"""
spatial_merge_size = self.config.spatial_merge_size
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
if mm_feature.data is None:
continue
offset = mm_feature.mm_position.offset
modality = mm_feature.modality
if modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
yield (
offset,
"image",
{
"grid_t": int(t),
"grid_h": int(h) // spatial_merge_size,
"grid_w": int(w) // spatial_merge_size,
"second_per_grid_t": 0.0,
},
)
elif modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
second_per_grid_t = 1.0
spgt_field = mm_feature.data.get("second_per_grid_ts")
if spgt_field is not None:
second_per_grid_t = float(spgt_field.data.item())
yield (
offset,
"video",
{
"grid_t": int(t),
"grid_h": int(h) // spatial_merge_size,
"grid_w": int(w) // spatial_merge_size,
"second_per_grid_t": second_per_grid_t,
},
)
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec] | None = None,
**kwargs: object,
) -> tuple[torch.Tensor, int]:
"""Compute M-RoPE input positions using mm_features directly."""
llm_config = self.config
tokens_per_second: int = getattr(llm_config, "tokens_per_second", 2)
seq_len = len(input_tokens)
llm_pos_ids_list: list[np.ndarray] = []
st = 0 # index of next unprocessed token
for patch_offset, _modality, data in self.iter_mm_features(mm_features or []):
text_len = patch_offset - st
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
if text_len > 0:
llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx)
st_idx += text_len
# 3-D grid positions for patch tokens
grid_t: int = data["grid_t"]
grid_h: int = data["grid_h"]
grid_w: int = data["grid_w"]
second_per_grid_t: float = data["second_per_grid_t"]
t_raw = np.arange(grid_t)
if second_per_grid_t > 0:
t_index = (t_raw * second_per_grid_t * tokens_per_second).astype(np.int64)
else:
t_index = t_raw.astype(np.int64)
t_index = np.repeat(t_index, grid_h * grid_w)
h_index = np.tile(np.arange(grid_h).repeat(grid_w), grid_t)
w_index = np.tile(np.arange(grid_w), grid_t * grid_h)
llm_pos_ids_list.append(np.stack([t_index, h_index, w_index]) + st_idx)
num_patches = grid_t * grid_h * grid_w
st = patch_offset + num_patches
if st < seq_len:
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
tail_len = seq_len - st
llm_pos_ids_list.append(np.broadcast_to(np.arange(tail_len), (3, tail_len)) + st_idx)
if llm_pos_ids_list:
position_ids = torch.from_numpy(np.concatenate(llm_pos_ids_list, axis=1).astype(np.int64)) # (3, seq_len)
else:
# text-only, simple sequential positions
position_ids = torch.arange(seq_len, dtype=torch.long).unsqueeze(0).expand(3, -1)
mrope_position_delta = int(position_ids.max().item()) + 1 - seq_len
return position_ids, mrope_position_delta
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@@ -0,0 +1,896 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright 2023 Antgroup and The HuggingFace Inc. team. All rights reserved.
# Adapted from Ming
# https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections.abc import Iterable
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.config.cache import CacheConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE, SharedFusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead, VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.utils import (
PPMissingLayer,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMoeV2Config
logger = init_logger(__name__)
class MingVideoRopeMRotaryEmbedding(MRotaryEmbedding):
"""MRotaryEmbedding with Ming's video_rope cos/sin interleaving.
Unlike standard mrope which maps contiguous frequency sections to T/H/W,
video_rope alternates H/W frequencies element-wise in the spatial section
and places temporal frequencies at the end:
Standard mrope: [T T T ... H H H ... W W W ...]
Video rope: [H W H W ... H W ... T T T ...]
Refer to Ming's BailingMoeV2RotaryEmbedding3D
https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py#L174
"""
def _remap_video_rope(
self,
cos: torch.Tensor,
sin: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Remap 3D cos/sin to video_rope interleaved layout.
Args:
cos, sin: [3, num_tokens, rotary_dim // 2]
Returns:
cos, sin: [num_tokens, rotary_dim // 2]
Refer to Ming's apply_3d_rotary_pos_emb
https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py#L226
"""
assert self.mrope_section is not None
hw_size = self.mrope_section[1] + self.mrope_section[2]
result_cos = torch.empty_like(cos[0])
result_sin = torch.empty_like(sin[0])
# Spatial frequencies: even indices from H (dim 1), odd from W (dim 2)
result_cos[:, 0:hw_size:2] = cos[1, :, 0:hw_size:2]
result_cos[:, 1:hw_size:2] = cos[2, :, 1:hw_size:2]
result_sin[:, 0:hw_size:2] = sin[1, :, 0:hw_size:2]
result_sin[:, 1:hw_size:2] = sin[2, :, 1:hw_size:2]
# Temporal frequencies at the end
result_cos[:, hw_size:] = cos[0, :, hw_size:]
result_sin[:, hw_size:] = sin[0, :, hw_size:]
return result_cos, result_sin
def forward_native(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert positions.ndim == 1 or positions.ndim == 2
assert key is not None
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
num_tokens = positions.shape[-1]
cos_sin = cos_sin_cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 2:
cos, sin = self._remap_video_rope(cos, sin)
query_shape = query.shape
query = query.view(num_tokens, -1, self.head_size)
query_rot = query[..., : self.rotary_dim]
query_pass = query[..., self.rotary_dim :]
query_rot = self.apply_rotary_emb.forward_native(query_rot, cos, sin)
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
key_shape = key.shape
key = key.view(num_tokens, -1, self.head_size)
key_rot = key[..., : self.rotary_dim]
key_pass = key[..., self.rotary_dim :]
key_rot = self.apply_rotary_emb.forward_native(key_rot, cos, sin)
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
return query, key
def forward_cuda(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# No custom Triton kernel for video_rope; fall back to native for 3D
# TODO: Consider custom optimization
if positions.ndim == 2:
return self.forward_native(positions, query, key, offsets)
return super().forward_cuda(positions, query, key, offsets)
def forward_cpu(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
return self.forward_native(positions, query, key, offsets)
class BailingMoeV2MLP(nn.Module):
def __init__(
self,
config: BailingMoeV2Config,
intermediate_size: int,
hidden_act: str = "silu",
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
self.intermediate_size = intermediate_size
self.gate_up_proj = MergedColumnParallelLinear(
self.hidden_size,
[self.intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
self.intermediate_size,
self.hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
raise ValueError(f"Unsupported activation: {hidden_act}")
self.act_fn = SiluAndMul()
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class BailingMoeV2Gate(nn.Module):
"""MoE routing gate with grouped expert selection."""
def __init__(
self,
config: BailingMoeV2Config,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.top_k = config.num_experts_per_tok
self.num_experts = config.num_experts
self.n_group = config.n_group
self.topk_group = config.topk_group
self.gating_dim = config.hidden_size
self.gate = ReplicatedLinear(
self.gating_dim,
self.num_experts,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate",
)
self.routed_scaling_factor = config.routed_scaling_factor
self.expert_bias = nn.Parameter(torch.zeros(self.num_experts), requires_grad=False)
def group_limited_topk(self, scores: torch.Tensor):
"""Group-limited top-k selection for expert routing."""
num_tokens, _ = scores.size()
# Organize experts into groups
group_scores = scores.view(num_tokens, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
group_mask = torch.zeros_like(group_scores)
group_mask.scatter_(1, group_idx, 1)
# Mask experts based on selected groups
score_mask = (
group_mask.unsqueeze(-1)
.expand(num_tokens, self.n_group, self.num_experts // self.n_group)
.reshape(num_tokens, -1)
)
masked_scores = scores.masked_fill(~score_mask.bool(), float("-inf"))
probs, top_indices = torch.topk(masked_scores, k=self.top_k, dim=-1, sorted=False)
return probs, top_indices
def forward(self, hidden_states):
# compute gating score
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
logits, _ = self.gate(hidden_states)
logits = logits.float()
scores = torch.sigmoid(logits)
scores_for_routing = scores + self.expert_bias
_, topk_idx = self.group_limited_topk(scores_for_routing)
scores = torch.gather(scores, dim=1, index=topk_idx).type_as(logits)
topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores
topk_weight = topk_weight * self.routed_scaling_factor
return topk_idx, topk_weight, logits
def _unpack_multi_routing(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Stateless routing function that unpacks pre-computed routing results.
Used as `custom_routing_function` for `FusedMoE`. The caller is expected
to pack (topk_weight, topk_idx) into `gating_output` before
calling FusedMoE.forward(), and this function unpacks them.
Args:
gating_output: [num_tokens, top_k * 2]
- [:, :top_k], topk_weight (float)
- [:, top_k:], topk_idx (float, cast back to int)
"""
topk_weight = gating_output[:, :topk].contiguous()
topk_idx = gating_output[:, topk:]
return topk_weight.to(torch.float32), topk_idx.to(torch.int32)
class BailingMoeV2SparseMoeBlock(nn.Module):
"""Sparse MoE block with MultiRouter support for multimodal routing.
Keep the custom multi-router gating logic external.
"""
def __init__(
self,
config: BailingMoeV2Config,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.tp_size = get_tensor_model_parallel_world_size()
self.num_experts_per_tok = config.num_experts_per_tok
if isinstance(self.config.num_shared_experts, int) and self.config.num_shared_experts > 0:
self.shared_experts = BailingMoeV2MLP(
config=self.config,
intermediate_size=self.config.moe_intermediate_size * self.config.num_shared_experts,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.shared_experts",
)
else:
self.shared_experts = None
self.experts = SharedFusedMoE(
shared_experts=self.shared_experts,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
custom_routing_function=_unpack_multi_routing,
renormalize=False, # we handle normalization in the gate
reduce_results=True,
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
self.experts.expert_mapping = FusedMoE.make_expert_params_mapping(
self.experts,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=config.num_experts,
)
self.router_type = self.config.router_type
if self.router_type == "topN":
self.gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.gate")
elif self.router_type == "MultiRouter":
self.gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.gate")
self.image_gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.image_gate")
self.audio_gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.audio_gate")
else:
raise ValueError(f"Unsupported router_type: {self.router_type}")
@staticmethod
def _normalize_mask(
mask: torch.Tensor,
bsz: int,
seq_len: int,
name: str,
) -> torch.Tensor:
"""Validate and reshape a modality mask to [bsz*seq_len, 1] bool."""
N = bsz * seq_len
if mask.ndim == 1:
# vLLM path: flat tokens [N]
assert mask.shape[0] == N, f"{name} length {mask.shape[0]} != N={N}"
elif mask.ndim == 2:
assert mask.shape == (bsz, seq_len), f"{name} shape {mask.shape} != ({bsz}, {seq_len})"
elif mask.ndim == 3:
assert mask.shape == (bsz, seq_len, 1), f"{name} shape {mask.shape} != ({bsz}, {seq_len}, 1)"
else:
raise ValueError(f"Unsupported {name} shape: {mask.shape}")
return mask.reshape(N, 1).bool()
def forward(self, hidden_states, image_mask: torch.Tensor, audio_mask: torch.Tensor):
# TODO(yuanheng-zhao): revise the shapes in the flow
assert 2 <= hidden_states.dim() <= 3, f"{self.__class__.__name__} only supports 2D or 3D inputs"
input_is_2d = hidden_states.ndim == 2
if input_is_2d:
hidden_states = hidden_states.unsqueeze(0)
bsz, seq_len, h = hidden_states.shape
if self.router_type == "MultiRouter":
image_mask = self._normalize_mask(image_mask, bsz, seq_len, "image_mask").to(hidden_states.device)
audio_mask = self._normalize_mask(audio_mask, bsz, seq_len, "audio_mask").to(hidden_states.device)
# if image_mask is not None and audio_mask is not None:
# assert torch.logical_and(image_mask, audio_mask).sum() == 0
image_topk_idx, image_topk_weight, _ = self.image_gate(hidden_states)
audio_topk_idx, audio_topk_weight, _ = self.audio_gate(hidden_states)
topk_idx, topk_weight, _ = self.gate(hidden_states)
topk_idx = torch.where(image_mask, image_topk_idx, topk_idx)
topk_weight = torch.where(image_mask, image_topk_weight, topk_weight)
topk_idx = torch.where(audio_mask, audio_topk_idx, topk_idx)
topk_weight = torch.where(audio_mask, audio_topk_weight, topk_weight)
else:
topk_idx, topk_weight, _ = self.gate(hidden_states)
# Pack pre-computed routing into a single tensor
packed_routing = torch.cat(
[
topk_weight.to(hidden_states.dtype),
topk_idx.to(hidden_states.dtype),
],
dim=-1,
)
# SharedFusedMoE expects 2D hidden_states
hidden_states_2d = hidden_states.view(-1, h)
result = self.experts(hidden_states_2d, packed_routing)
if self.shared_experts is not None:
shared_output, fused_out = result
else:
shared_output, fused_out = None, result
final_hidden_states = fused_out + shared_output if shared_output is not None else fused_out
final_hidden_states = final_hidden_states.view(bsz, seq_len, h)
return final_hidden_states.squeeze(0) if input_is_2d else final_hidden_states
class BailingMoeV2Attention(nn.Module):
"""Multi-headed attention using vLLM's Attention layer with 3D RoPE support."""
def __init__(
self,
config: BailingMoeV2Config,
layer_idx: int,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.head_dim = config.head_dim
tp_size = get_tensor_model_parallel_world_size()
assert self.num_heads % tp_size == 0
self.num_heads = self.num_heads // tp_size
self.num_kv_heads = max(1, self.num_kv_heads // tp_size)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
partial_rotary_factor = config.partial_rotary_factor
self.rope_dim = int(self.head_dim * partial_rotary_factor)
total_num_heads = config.num_attention_heads
total_num_kv_heads = config.num_key_value_heads
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
self.head_dim,
total_num_heads,
total_num_kv_heads,
bias=config.use_qkv_bias,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.dense = RowParallelLinear(
total_num_heads * self.head_dim,
self.hidden_size,
bias=config.use_bias,
quant_config=quant_config,
prefix=f"{prefix}.dense",
)
# apply vLLM RMSNorm here rather than BailingMoeV2RMSNorm, diff might exist
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
# 3D Rotary embeddings for multimodal
if config.rope_scaling is None:
raise ValueError("rope_scaling must not be None")
rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
mrope_section = config.rope_scaling.get("mrope_section", [8, 12, 12])
if rope_type == "video_rope":
# Ming-specific video_rope with custom H/W interleaving
self.rotary_emb = MingVideoRopeMRotaryEmbedding(
head_size=self.head_dim,
rotary_dim=self.rope_dim,
max_position_embeddings=config.max_position_embeddings,
base=config.rope_theta,
is_neox_style=True,
dtype=torch.get_default_dtype(),
mrope_section=mrope_section,
)
else:
# Standard m_rope (rope_type "3D", "default", or None)
rope_scaling = dict(config.rope_scaling)
rope_scaling["rope_type"] = "default" # normalize for get_rope dispatch
rope_scaling["mrope_section"] = mrope_section
self.rotary_emb = get_rope(
head_size=self.head_dim,
max_position=config.max_position_embeddings,
is_neox_style=True,
rope_parameters={
"rope_theta": config.rope_theta,
"partial_rotary_factor": config.partial_rotary_factor,
**rope_scaling,
},
)
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""Forward pass for attention with 3D RoPE.
Args:
positions: Position IDs, shape (3, num_tokens) for 3D rope
or (num_tokens,) for text-only
hidden_states: Input hidden states, shape (num_tokens, hidden_size)
Returns:
Attention output tensor, shape (num_tokens, hidden_size)
"""
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
num_tokens = q.shape[0]
q = self.q_norm(q.view(num_tokens, self.num_heads, self.head_dim)).view(num_tokens, self.q_size)
k = self.k_norm(k.view(num_tokens, self.num_kv_heads, self.head_dim)).view(num_tokens, self.kv_size)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.dense(attn_output)
return output
class BailingMoeV2DecoderLayer(nn.Module):
"""Decoder layer with attention and MoE MLP."""
def __init__(
self,
config: BailingMoeV2Config,
layer_idx: int,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.hidden_size = config.hidden_size
self.layer_idx = layer_idx
self.attention = BailingMoeV2Attention(
config=config,
layer_idx=layer_idx,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attention",
)
# MLP or MoE based on layer index
if config.num_experts is not None and layer_idx >= config.first_k_dense_replace:
self.mlp = BailingMoeV2SparseMoeBlock(
config=config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.is_moe = True
else:
self.mlp = BailingMoeV2MLP(
config=config,
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.is_moe = False
# apply vLLM RMSNorm to replace BailingMoeV2RMSNorm, diff might exist
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
image_mask: torch.Tensor | None = None,
audio_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Forward pass for decoder layer.
Args:
positions: Position IDs
hidden_states: Input hidden states
residual: Residual connection from previous layer
image_mask: Mask for image tokens (for MultiRouter MoE)
audio_mask: Mask for audio tokens (for MultiRouter MoE)
Returns:
Tuple of (hidden_states, residual)
"""
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states = self.attention(
positions=positions,
hidden_states=hidden_states,
)
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
if self.is_moe:
hidden_states = self.mlp(hidden_states, image_mask, audio_mask)
else:
# Dense MLP only takes hidden_states (no routing masks)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"image_mask": 0,
"audio_mask": 0,
}
)
class BailingMoeV2Model(nn.Module):
"""BailingMoeV2 Model adapted from:
Ming repo BailingMoeV2Model
https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py
vLLM repo BailingMoeModel
https://github.com/vllm-project/vllm/blob/7291d1b288558d48508e1a17c37b0aa170332264/vllm/model_executor/models/bailing_moe.py
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__()
# BailingMoeV2Config
config = vllm_config.model_config.hf_text_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False)
if get_pp_group().is_first_rank or (self.tie_word_embeddings and get_pp_group().is_last_rank):
self.word_embeddings = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.word_embeddings",
)
else:
self.word_embeddings = PPMissingLayer()
# Decoder layers with later pipeline parallelism support
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: BailingMoeV2DecoderLayer(
config=config,
layer_idx=int(prefix.split(".")[-1]),
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
),
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
# apply vLLM RMSNorm to replace BailingMoeV2RMSNorm, diff might exist
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
def get_input_embeddings(self):
return self.word_embeddings
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
image_mask: torch.Tensor | None = None,
audio_mask: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.word_embeddings(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
for layer in self.layers[self.start_layer : self.end_layer]:
hidden_states, residual = layer(
positions,
hidden_states,
residual,
image_mask=image_mask,
audio_mask=audio_mask,
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states, "residual": residual})
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
class BailingMoeV2ForCausalLM(nn.Module, CustomProcessMixin):
"""BailingMoeV2 model for causal language modeling, adapted for vLLM.
Inherits from CustomProcessMixin to support custom preprocessing and postprocessing
for integration with omni model pipelines.
"""
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
# BailingMoeV2Config
config = vllm_config.model_config.hf_text_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.model = BailingMoeV2Model(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
)
self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if self.tie_word_embeddings:
self.lm_head.weight = self.model.word_embeddings.weight
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.sampler = Sampler()
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
image_mask: torch.Tensor | None = None,
audio_mask: torch.Tensor | None = None,
):
hidden_states = self.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
image_mask=image_mask,
audio_mask=audio_mask,
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
sampling_metadata,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states, sampling_metadata)
return logits
def sample(
self,
logits: torch.Tensor,
sampling_metadata,
) -> SamplerOutput | None:
next_tokens = self.sampler(logits, sampling_metadata)
return next_tokens
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, weight_name, shard_id)
# BailingMoE stores fused QKV in checkpoint as query_key_value
("qkv_proj", "query_key_value", None),
# Dense MLP and shared_experts gate/up are stored separately
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Gate router linear layers: checkpoint `{r}.weight` -> model `{r}.gate.weight`
gate_name_mapper = WeightsMapper(
orig_to_new_substr={f".{r}.weight": f".{r}.gate.weight" for r in ("gate", "image_gate", "audio_gate")}
)
# FusedMoE expert params mapping is identical across all MoE layers
expert_params_mapping: list[tuple[str, str, int, str]] = []
for layer in self.model.layers:
if hasattr(layer, "mlp") and hasattr(layer.mlp, "experts"):
expert_params_mapping = layer.mlp.experts.expert_mapping
break
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in gate_name_mapper.apply(weights):
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name or "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict.get(name)
if param is not None:
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name)
break
else:
for param_name, weight_name, expert_id, shard_id in expert_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict.get(name)
if param is not None:
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, name, shard_id=shard_id, expert_id=expert_id)
loaded_params.add(name)
break
else:
param = params_dict.get(name)
if param is not None:
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from Ming repository modeling_bailingmm2.py
# https://github.com/inclusionAI/Ming
from collections.abc import Iterable
import torch
import torch.nn as nn
from vllm.logger import init_logger
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
logger = init_logger(__name__)
class Transpose(nn.Module):
"""Used in nn.Sequential pipelines."""
def __init__(self, dim0: int, dim1: int):
super().__init__()
self.dim0 = dim0
self.dim1 = dim1
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x.transpose(self.dim0, self.dim1)
class VisionProjector(nn.Module):
"""MLP projector from vision encoder output to LLM hidden space.
Args:
vision_dim: Vision encoder output dimension (out_hidden_size).
llm_dim: LLM hidden dimension.
mlp_depth: Number of linear layers (>= 1).
"""
def __init__(self, vision_dim: int, llm_dim: int, mlp_depth: int = 1):
super().__init__()
layers: list[nn.Module] = [nn.Linear(vision_dim, llm_dim)]
for _ in range(1, mlp_depth):
layers.append(nn.GELU())
layers.append(nn.Linear(llm_dim, llm_dim))
self.proj = nn.Sequential(*layers)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Project vision features.
Args:
x: [seq_len, vision_dim] or [B, seq_len, vision_dim]
Returns:
Projected features with last dim = llm_dim.
"""
return self.proj(x)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if not name.startswith("proj."):
name = f"proj.{name}"
if name not in params_dict:
logger.warning("Skipping unknown vision projector weight: %s", name)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class AudioProjector(nn.Module):
"""Projector for audio features.
Args:
audio_dim: Audio encoder output dimension (n_state).
llm_dim: LLM hidden dimension.
ds_kernel_size: Conv1d kernel size for downsampling.
ds_stride: Conv1d stride for downsampling.
mlp_depth: Total number of projection layers (>= 1).
"""
def __init__(
self,
audio_dim: int,
llm_dim: int,
ds_kernel_size: int = 3,
ds_stride: int = 2,
mlp_depth: int = 1,
):
super().__init__()
self.ds_kernel_size = ds_kernel_size
self.ds_stride = ds_stride
layers: list[nn.Module] = [
nn.Conv1d(
audio_dim,
llm_dim,
kernel_size=ds_kernel_size,
stride=ds_stride,
padding=ds_kernel_size // 2,
),
Transpose(-1, -2), # [B, llm_dim, T'] -> [B, T', llm_dim]
]
for _ in range(1, mlp_depth):
layers.append(nn.GELU())
layers.append(nn.Linear(llm_dim, llm_dim))
self.proj = nn.Sequential(*layers)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Project audio features with temporal downsampling.
Args:
x: [B, T, audio_dim] audio encoder output (channel-last).
Returns:
[B, T', llm_dim] projected features (channel-last),
where T' = (T - ds_kernel_size + 2*(ds_kernel_size//2)) // ds_stride + 1.
"""
# Conv1d expects [B, C, T], so transpose input
x = x.transpose(-1, -2) # [B, audio_dim, T]
return self.proj(x)
def forward_packed(
self,
packed: torch.Tensor,
encoded_lens: list[int],
) -> tuple[torch.Tensor, list[int]]:
"""Project packed audio features from the Whisper encoder.
Args:
packed: [total_T', audio_dim] packed encoder output.
encoded_lens: Length of each clip after Whisper encoding.
Returns:
Tuple of:
- [total_T'', llm_dim] packed projected features.
- List of projected lengths per clip.
"""
conv1d = self.proj[0]
mlp = self.proj[2:]
# Split packed tensor per clip for Conv1d
segments = packed.split(encoded_lens)
conv_segments = []
proj_lens: list[int] = []
for seg in segments:
out = conv1d(seg.transpose(0, 1).unsqueeze(0)) # [1, llm_dim, T'_i]
out = out.squeeze(0).transpose(0, 1) # [T'_i, llm_dim]
conv_segments.append(out)
proj_lens.append(out.shape[0])
packed_proj = torch.cat(conv_segments, dim=0) # [total_T'', llm_dim]
packed_proj = mlp(packed_proj)
return packed_proj, proj_lens
def compute_output_length(self, input_length: torch.Tensor) -> torch.Tensor:
"""Compute output sequence length after Conv1d downsampling.
Args:
input_length: Original mel spectrogram lengths.
Returns:
Output lengths after both convolutions.
"""
length = (input_length - 3 + 2 * 1) // 2 + 1
length = (length - self.ds_kernel_size + 2 * (self.ds_kernel_size // 2)) // self.ds_stride + 1
return length
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if not name.startswith("proj."):
name = f"proj.{name}"
if name not in params_dict:
logger.warning("Skipping unknown audio projector weight: %s", name)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -0,0 +1,125 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Adapted from Ming repository qwen3_moe_vit.py
# https://github.com/inclusionAI/Ming
from collections.abc import Iterable
import torch
import torch.nn as nn
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3Omni_VisionTransformer,
)
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
def _adapt_vision_config(vision_config):
# Adapt Ming's Qwen3VLMoeVisionConfig to be compatible with vLLM's
# Qwen3Omni_VisionTransformer expectations.
if not hasattr(vision_config, "image_size") or vision_config.image_size is None:
if hasattr(vision_config, "num_position_embeddings") and vision_config.num_position_embeddings:
import math
num_grid = int(math.sqrt(vision_config.num_position_embeddings))
vision_config.image_size = num_grid * vision_config.patch_size
else:
vision_config.image_size = vision_config.patch_size * 14 # fallback
if not hasattr(vision_config, "apply_vit_abs_pos_embed"):
vision_config.apply_vit_abs_pos_embed = True
return vision_config
class MingVisionEncoder(nn.Module):
"""**Wrapper** around vLLM's Qwen3Omni_VisionTransformer for Ming."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"deepstack_merger_list.": "merger_list.",
"merger.norm.": "merger.ln_q.",
"merger.linear_fc1.": "merger.mlp.0.",
"merger.linear_fc2.": "merger.mlp.2.",
}
)
def __init__(
self,
vision_config,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
adapted_config = _adapt_vision_config(vision_config)
norm_eps = 1e-6
self.encoder = Qwen3Omni_VisionTransformer(
vision_config=adapted_config,
norm_eps=norm_eps,
quant_config=quant_config,
prefix=f"{prefix}.encoder",
)
self.image_emb_dim = vision_config.out_hidden_size
self.use_deepstack = (
hasattr(vision_config, "deepstack_visual_indexes") and vision_config.deepstack_visual_indexes is not None
)
@property
def dtype(self) -> torch.dtype:
return self.encoder.dtype
@property
def device(self) -> torch.device:
return self.encoder.device
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor,
) -> torch.Tensor:
"""forward method of the vision encoder.
Args:
pixel_values: Flattened pixel values.
grid_thw: [num_images, 3] tensor of (t, h, w) grid sizes.
Returns:
If deepstack is enabled, returns concatenated multi-scale features
along the feature dim: [seq_len, hidden_size * (1 + num_deepstack)].
Otherwise returns [seq_len, hidden_size].
"""
return self.encoder(pixel_values, grid_thw=grid_thw)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
import re
def _remap_merger_list_inner(name: str) -> str:
name = re.sub(r"(merger_list\.\d+)\.norm\.", r"\1.ln_q.", name)
name = re.sub(r"(merger_list\.\d+)\.linear_fc1\.", r"\1.mlp.0.", name)
name = re.sub(r"(merger_list\.\d+)\.linear_fc2\.", r"\1.mlp.2.", name)
return name
remapped_weights = self.hf_to_vllm_mapper.apply(weights)
remapped_weights = ((_remap_merger_list_inner(name), tensor) for name, tensor in remapped_weights)
loaded_params = self.encoder.load_weights(remapped_weights)
loaded_params = {f"encoder.{loaded_param}" for loaded_param in loaded_params}
return loaded_params
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen2.5-Omni pipeline topology (frozen).
Stage 0: Thinker — multimodal understanding + text generation
Stage 1: Talker — text embeddings → speech tokens
Stage 2: Code2Wav — speech tokens → audio waveform
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen2_5_omni"
QWEN2_5_OMNI_PIPELINE = PipelineConfig(
model_type="qwen2_5_omni",
model_arch="Qwen2_5OmniForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
engine_output_type="latent",
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="talker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(0,),
engine_output_type="latent",
custom_process_input_func=f"{_PROC}.thinker2talker",
sampling_constraints={
"detokenize": True,
"stop_token_ids": [8294],
},
),
StagePipelineConfig(
stage_id=2,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(1,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sampling_constraints={"detokenize": True},
),
),
)
# Single-stage thinker-only variant for the abort test.
QWEN2_5_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
model_type="qwen2_5_omni_thinker_only",
model_arch="Qwen2_5OmniForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
engine_output_type="latent",
sampling_constraints={"detokenize": True},
),
),
)
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen3-Omni-MoE pipeline topology (frozen).
Stage 0: Thinker — multimodal understanding + text generation
Stage 1: Talker — text embeddings → RVQ codec codes
Stage 2: Code2Wav — RVQ codes → audio waveform
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni"
QWEN3_OMNI_PIPELINE = PipelineConfig(
model_type="qwen3_omni_moe",
model_arch="Qwen3OmniMoeForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
hf_config_name="thinker_config",
engine_output_type="latent",
custom_process_next_stage_input_func=(f"{_PROC}.thinker2talker_async_chunk"),
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="talker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(0,),
hf_config_name="talker_config",
engine_output_type="latent",
custom_process_input_func=f"{_PROC}.thinker2talker",
custom_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
sampling_constraints={
"detokenize": False,
"stop_token_ids": [2150],
},
),
StagePipelineConfig(
stage_id=2,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(1,),
final_output=True,
final_output_type="audio",
hf_config_name="thinker_config",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.talker2code2wav",
sampling_constraints={"detokenize": True},
),
),
)
@@ -180,6 +180,11 @@ class Qwen3OmniMoeForConditionalGeneration(
"trailing_text_hidden",
"tts_pad_embed_projected",
}
# Keys that need to be accumulated across streaming inputs
self.streaming_accumulated_keys: set[str] = {
"thinker_prefill_embeddings",
"thinker_hidden_states",
}
elif self.model_stage == "code2wav":
self.enable_update_additional_information = True
@@ -684,6 +689,7 @@ class Qwen3OmniMoeForConditionalGeneration(
input_embeds: torch.Tensor,
last_talker_hidden: torch.Tensor,
text_step: torch.Tensor,
**kwargs: Any,
):
# TODO(Peiqi): not support intermediate_tensors now
input_ids = safe_tensor_reshape(input_ids, (input_ids.shape[0], -1))
@@ -891,10 +897,11 @@ class Qwen3OmniMoeForConditionalGeneration(
Returns:
(input_ids, input_embeds) for talker
"""
target_len = thinker_result_ids.shape[-1]
im_start_indexes = torch.cat(
(
torch.nonzero(input_ids[0] == self.config.im_start_token_id).squeeze(),
torch.tensor([thinker_result_ids.shape[-1]], device=input_ids.device, dtype=input_ids.dtype),
torch.tensor([target_len], device=input_ids.device, dtype=input_ids.dtype),
),
dim=-1,
) # Shape [n_starts + 1]; Take batch 0 since batched inference is not supported here.
@@ -1029,8 +1036,35 @@ class Qwen3OmniMoeForConditionalGeneration(
return last_talker_hidden, text_step, update_dict
def _get_talker_user_parts(self, im_start_index, segment_end_index, multimodal_mask, thinker_hidden, thinker_embed):
clamped = min(
segment_end_index,
multimodal_mask.shape[0],
thinker_hidden.shape[0],
thinker_embed.shape[0],
)
if clamped < segment_end_index:
logger.warning(
"_get_talker_user_parts: segment_end_index %d clamped to %d "
"(embed=%d, hidden=%d, mask=%d). "
"This usually means _merge_pd_embeddings failed to merge "
"prefill embeddings – check PD prefill_mm keys.",
segment_end_index,
clamped,
thinker_embed.shape[0],
thinker_hidden.shape[0],
multimodal_mask.shape[0],
)
segment_end_index = clamped
seg_len = segment_end_index - im_start_index
if seg_len <= 0:
return torch.empty(
(0, self.config.talker_config.text_config.hidden_size),
device=thinker_hidden.device,
dtype=torch.bfloat16,
)
user_talker_part = torch.empty(
(segment_end_index - im_start_index, self.config.talker_config.text_config.hidden_size),
(seg_len, self.config.talker_config.text_config.hidden_size),
device=thinker_hidden.device,
dtype=torch.bfloat16,
)
@@ -1,510 +1,28 @@
"""Qwen3-Omni Code Predictor -- optimized re-prefill, no KV cache.
"""Qwen3-Omni Code Predictor -- thin wrapper over CodePredictorWrapper."""
* SDPA attention (F.scaled_dot_product_attention) with native GQA support
* HF-compatible numerics (float32 RMSNorm, float32 RoPE, separate linear layers)
* Per-call embedding buffer to avoid cross-request aliasing
* Pre-allocated position_ids (read-only, safe to persist)
* torch.compile (epilogue_fusion=False) on inner transformer by default
* Inline sampling (top-k + top-p) -- no custom op overhead
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
CodePredictorWrapper,
CodePredictorWrapperConfig,
)
# ===================================================================
# HF-numerics-compatible layers for code predictor
# ===================================================================
#
# These use plain PyTorch ops (nn.Linear, manual RMSNorm in float32,
# rotate_half RoPE) to produce outputs numerically identical to the
# HuggingFace reference. vLLM's fused kernels (RMSNorm, QKVParallel,
# get_rope) introduce small precision differences that compound across
# the autoregressive steps of the code predictor, causing severe
# audio quality degradation.
#
# See: https://github.com/vllm-project/vllm-omni/issues/2274
class Qwen3OmniMoeTalkerCodePredictor(CodePredictorWrapper):
"""Qwen3-Omni code predictor (no CUDA graphs, VocabParallelEmbedding)."""
class _RMSNorm(nn.Module):
"""RMSNorm matching HuggingFace's implementation exactly.
Computes variance in float32 to avoid bfloat16 precision loss.
"""
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
class _RotaryEmbedding(nn.Module):
"""RoPE matching HuggingFace's implementation exactly.
Forces float32 computation for cos/sin, matching HF's torch.autocast(enabled=False).
"""
def __init__(self, config) -> None:
super().__init__()
head_dim = getattr(
config,
"head_dim",
config.hidden_size // config.num_attention_heads,
)
rope_theta = getattr(config, "rope_theta", 10000.0)
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# position_ids: [batch, seq_len]
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
position_ids_expanded = position_ids[:, None, :].float()
# Force float32 (matching HF)
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos()
sin = emb.sin()
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
class Qwen3OmniCodePredictorAttention(nn.Module):
"""Multi-head self-attention for code predictor.
Uses ``F.scaled_dot_product_attention`` with HF-compatible RoPE and RMSNorm.
No KV cache -- the code predictor always re-prefills the full (short)
sequence each AR step.
Input : [B, seq_len, hidden_size]
Output: [B, seq_len, hidden_size]
"""
def __init__(
self,
config,
prefix: str = "",
):
super().__init__()
cp_cfg = config.code_predictor_config
self.num_heads = cp_cfg.num_attention_heads
self.num_kv_heads = cp_cfg.num_key_value_heads
self.head_dim = getattr(
cp_cfg,
"head_dim",
cp_cfg.hidden_size // cp_cfg.num_attention_heads,
)
self.hidden_size = cp_cfg.hidden_size
self.scaling = self.head_dim**-0.5
self._use_gqa = self.num_kv_heads != self.num_heads
# Separate q/k/v projections matching HF (no fused packing)
self.q_proj = nn.Linear(
self.hidden_size,
self.num_heads * self.head_dim,
bias=False,
)
self.k_proj = nn.Linear(
self.hidden_size,
self.num_kv_heads * self.head_dim,
bias=False,
)
self.v_proj = nn.Linear(
self.hidden_size,
self.num_kv_heads * self.head_dim,
bias=False,
)
self.o_proj = nn.Linear(
self.num_heads * self.head_dim,
self.hidden_size,
bias=False,
)
self.q_norm = _RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps)
self.k_norm = _RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim)
hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim)
q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)
k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2)
v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2)
cos, sin = position_embeddings
# cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads
cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]
sin = sin.unsqueeze(1)
q = (q * cos) + (_rotate_half(q) * sin)
k = (k * cos) + (_rotate_half(k) * sin)
attn_out = F.scaled_dot_product_attention(
q,
k,
v,
scale=self.scaling,
is_causal=True,
enable_gqa=self._use_gqa,
)
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
output = self.o_proj(attn_out)
return output
# ===================================================================
# MLP
# ===================================================================
class Qwen3OmniCodePredictorMLP(nn.Module):
"""SiLU-gated MLP for code predictor, matching HF's implementation."""
def __init__(
self,
config,
prefix: str = "",
):
super().__init__()
hidden_size = config.code_predictor_config.hidden_size
intermediate_size = config.code_predictor_config.intermediate_size
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
# ===================================================================
# Decoder Layer
# ===================================================================
class Qwen3OmniCodePredictorDecoderLayer(nn.Module):
"""Transformer decoder layer (SDPA, no KV cache)."""
def __init__(
self,
config,
prefix: str = "",
) -> None:
super().__init__()
self.self_attn = Qwen3OmniCodePredictorAttention(
config,
prefix=f"{prefix}.self_attn",
)
self.mlp = Qwen3OmniCodePredictorMLP(
config,
prefix=f"{prefix}.mlp",
)
cp_cfg = config.code_predictor_config
self.input_layernorm = _RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps)
self.post_attention_layernorm = _RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(hidden_states, position_embeddings)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
# ===================================================================
# Base Transformer Model (re-prefill, no KV cache)
# ===================================================================
class Qwen3OmniCodePredictorBaseModel(nn.Module):
"""Inner transformer for code predictor.
Signature: ``forward(inputs_embeds, position_ids) -> hidden_states``
-- plain Tensor in, plain Tensor out (no namedtuple).
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config.code_predictor_config
self.config = config
self.codec_embedding = nn.ModuleList(
[VocabParallelEmbedding(config.vocab_size, config.hidden_size) for _ in range(config.num_code_groups - 1)]
)
self.layers = nn.ModuleList(
[
Qwen3OmniCodePredictorDecoderLayer(
vllm_config.model_config.hf_config,
prefix=f"{prefix}.layers.{idx}",
)
for idx in range(config.num_hidden_layers)
]
)
self.norm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = _RotaryEmbedding(config)
def forward(
self,
inputs_embeds: torch.Tensor,
position_ids: torch.Tensor,
) -> torch.Tensor:
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids)
for layer in self.layers:
hidden_states = layer(hidden_states, position_embeddings)
hidden_states = self.norm(hidden_states)
return hidden_states
# ===================================================================
# Code Predictor Wrapper (optimized re-prefill, persistent buffers)
# ===================================================================
class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
"""Optimized code predictor -- re-prefill approach, no KV cache.
Each AR step forwards the full growing sequence (len 2 -> num_code_groups+1)
through the transformer. The extra O(T^2) FLOPs are negligible for
short sequences, and this avoids all KV-cache management overhead.
Optimizations:
1. Per-call embedding buffer -- avoids cross-request aliasing.
2. Pre-allocated position_ids -- no torch.arange per step.
3. Cached module references -- bypass ModuleList indexing.
4. torch.compile on inner transformer.
5. Inline sampling (top-k + top-p) -- no custom op overhead.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
self.quant_config = vllm_config.quant_config
self.prefix = prefix
self.num_code_groups = config.code_predictor_config.num_code_groups
self._hidden_size = config.code_predictor_config.hidden_size
self.model = Qwen3OmniCodePredictorBaseModel(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
cp_config = vllm_config.model_config.hf_config.code_predictor_config
super().__init__(
vllm_config=vllm_config,
cp_config=cp_config,
wrapper_config=CodePredictorWrapperConfig(
use_cuda_graphs=False,
use_parallel_embedding=True,
use_projection=False,
return_proj_buf=True,
sampling_mode="stored",
),
talker_hidden_size=cp_config.hidden_size,
prefix=prefix,
)
# One lm_head per residual layer (layers 1 .. G-1)
self.lm_head = nn.ModuleList(
[
nn.Linear(
config.code_predictor_config.hidden_size,
config.code_predictor_config.vocab_size,
bias=False,
)
for _ in range(self.num_code_groups - 1)
]
)
self.set_sampling_params()
# Lazily initialised position ids (read-only, safe to persist)
self._pos_ids: torch.Tensor | None = None
# Cached plain-list refs (set once)
self._lm_heads: list | None = None
self._codec_embeds: list | None = None
# Model forward (optionally compiled)
self._model_fwd: object | None = None
def set_sampling_params(self, top_k: int = 50, top_p: float = 0.8):
"""Configure sampling parameters to maintain consistency with previous implementation."""
self._top_k = top_k
self._top_p = top_p
logger.debug(f"Sampling parameters updated: top_k={top_k}, top_p={top_p}s")
# ------------------------------------------------------------------
# Lazy-init helpers
# ------------------------------------------------------------------
def _ensure_pos_ids(self, device: torch.device) -> None:
if self._pos_ids is not None and self._pos_ids.device == device:
return
max_seq = self.num_code_groups + 1
# [1, max_seq] for HF-style RoPE (will be expanded to [bsz, seq_len] at use)
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device).unsqueeze(0)
def _ensure_cached_refs(self) -> None:
if self._lm_heads is not None:
return
self._lm_heads = list(self.lm_head)
self._codec_embeds = list(self.model.codec_embedding)
def _ensure_model_fwd(self) -> None:
if self._model_fwd is not None:
return
if current_omni_platform.supports_torch_inductor():
# torch.compile fuses RMSNorm/RoPE in ways that lose float32
# precision, compounding across AR steps. Use epilogue_fusion=False
# to disable the problematic fusions while still getting kernel
# fusion benefits for the linear layers and SDPA.
self._model_fwd = torch.compile(
self.model.forward,
dynamic=True,
options={
"epilogue_fusion": False,
},
)
logger.info("code_predictor: torch.compile enabled (no epilogue fusion)")
else:
self._model_fwd = self.model.forward
logger.info("code_predictor: using eager mode (no torch.compile)")
# ------------------------------------------------------------------
# Forward -- re-prefill + inline sampling
# ------------------------------------------------------------------
@torch.inference_mode()
def forward(
self,
layer0_code: torch.Tensor,
layer0_embed: torch.Tensor,
last_talker_hidden: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Predict residual codebooks 1..G-1 autoregressively via re-prefill.
Args:
layer0_code: [bsz, 1] int64
layer0_embed: [bsz, 1, hidden_size]
last_talker_hidden: [bsz, 1, hidden_size]
Returns:
all_codes: [bsz, num_code_groups, 1]
proj_buf: [bsz, num_code_groups + 1, hidden_size]
pos 0 = last_talker_hidden (NOT a codec embed)
pos 1 = layer0_embed
pos 2.. = `codec_embedding[i](predicted_code_i)`
"""
bsz = int(layer0_code.shape[0])
device = layer0_code.device
dtype = last_talker_hidden.dtype
num_groups = self.num_code_groups
# Lazy init (read-only caches only)
self._ensure_pos_ids(device)
self._ensure_model_fwd()
self._ensure_cached_refs()
# Allocate proj_buf locally each call to avoid cross-call aliasing
max_seq = num_groups + 1
proj_buf = torch.zeros(bsz, max_seq, self._hidden_size, dtype=dtype, device=device)
pos_ids = self._pos_ids
model_fwd = self._model_fwd
lm_heads = self._lm_heads
codec_embeds = self._codec_embeds
# Output codes
all_codes = torch.empty(bsz, num_groups, 1, dtype=torch.int64, device=device)
all_codes[:, 0] = layer0_code
# Fill buffer positions 0 & 1
proj_buf[:bsz, 0:1, :] = last_talker_hidden
proj_buf[:bsz, 1:2, :] = layer0_embed
# Autoregressive loop: predict layers 1..G-1
for step in range(1, num_groups):
seq_len = step + 1
projected = proj_buf[:bsz, :seq_len, :]
# position_ids: [batch, seq_len] for HF-style RoPE
step_pos_ids = pos_ids[:, :seq_len].expand(bsz, -1)
hidden_out = model_fwd(projected, step_pos_ids)
# Inline sampling: top-k -> top-p -> softmax -> multinomial
logits = lm_heads[step - 1](hidden_out[:, -1, :]) # [bsz, vocab]
if self._top_k > 0:
topk_vals, _ = logits.topk(self._top_k, dim=-1)
logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf"))
if self._top_p < 1.0:
sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
cumulative_probs = F.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
# Remove tokens with cumulative probability above top_p
remove_mask = cumulative_probs - F.softmax(sorted_logits, dim=-1) >= self._top_p
sorted_logits[remove_mask] = float("-inf")
logits = sorted_logits.scatter(1, sorted_idx, sorted_logits)
probs = F.softmax(logits, dim=-1)
code = torch.multinomial(probs, num_samples=1) # [bsz, 1]
all_codes[:, step] = code
# Embed predicted code -> next buffer position
new_embed = codec_embeds[step - 1](code) # [batch, 1, hidden_size]
proj_buf[:bsz, step + 1 : step + 2, :] = new_embed
return all_codes, proj_buf[:bsz]
# ------------------------------------------------------------------
# Weight loading
# ------------------------------------------------------------------
def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights directly (no fused projection remapping needed).
Since we use separate nn.Linear for q/k/v/o and gate/up/down,
weight names match the HF checkpoint directly.
"""
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# Skip rotary embeddings
if "rotary_emb.inv_freq" in name:
continue
param = params_dict.get(name)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen3-TTS pipeline: Talker (text → RVQ codec) → Code2Wav (codec → audio).
Chunked vs end-to-end mode is dispatched from ``deploy.async_chunk``.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_tts"
QWEN3_TTS_PIPELINE = PipelineConfig(
model_type="qwen3_tts",
# Pipeline-level default; the code2wav stage overrides per-stage below.
model_arch="Qwen3TTSTalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="qwen3_tts",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
sampling_constraints={
"detokenize": False,
"stop_token_ids": [2150],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="Qwen3TTSCode2Wav",
sync_process_input_func=f"{_PROC}.talker2code2wav",
sampling_constraints={"detokenize": True},
extras={"tts_args": {"max_instructions_length": 500}},
),
),
)
@@ -1,93 +0,0 @@
model_type: qwen3_tts
async_chunk: true
stages:
- stage_id: 0
model_stage: qwen3_tts
stage_type: llm
is_comprehension: true
input_sources: []
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
runtime:
devices: "0"
engine_args:
max_num_seqs: 10
model_arch: Qwen3TTSTalkerForConditionalGeneration
hf_overrides:
architectures: [Qwen3TTSTalkerForConditionalGeneration]
enforce_eager: false
trust_remote_code: true
async_scheduling: true
enable_prefix_caching: false
engine_output_type: latent
gpu_memory_utilization: 0.08
distributed_executor_backend: "mp"
max_num_batched_tokens: 512
max_model_len: 4096
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
output_connectors:
to_stage_1: connector_of_shared_memory
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: false
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 1
model_stage: code2wav
stage_type: llm
input_sources: [0]
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
final_output: true
final_output_type: audio
runtime:
devices: "0"
engine_args:
max_num_seqs: 1
model_arch: Qwen3TTSCode2Wav
hf_overrides:
architectures: [Qwen3TTSCode2Wav]
enforce_eager: true
trust_remote_code: true
async_scheduling: true
enable_prefix_caching: false
engine_output_type: audio
gpu_memory_utilization: 0.08
distributed_executor_backend: "mp"
max_num_batched_tokens: 65536
max_model_len: 65536
input_connectors:
from_stage_0: connector_of_shared_memory
tts_args:
max_instructions_length: 500
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
detokenize: true
repetition_penalty: 1.0
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
shm_threshold_bytes: 65536
codec_streaming: true
connector_get_sleep_s: 0.01
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
codec_chunk_frames: 25
# Match the decoder sliding attention window to avoid chunk-boundary noise.
codec_left_context_frames: 72
edges:
- from: 0
to: 1
window_size: -1
@@ -1,318 +1,27 @@
"""Qwen3-TTS Code Predictor -- thin wrapper over CodePredictorWrapper."""
from __future__ import annotations
from collections.abc import Iterable
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
)
from vllm_omni.platforms import current_omni_platform
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
CodePredictorBaseModel,
CodePredictorWrapper,
CodePredictorWrapperConfig,
)
from .configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig, Qwen3TTSTalkerConfig
logger = init_logger(__name__)
# Backward-compat alias used by tests
Qwen3TTSTalkerCodePredictorModelVLLM = CodePredictorBaseModel
# ===================================================================
# HF-numerics-compatible layers for code predictor
# ===================================================================
#
# These use plain PyTorch ops (nn.Linear, manual RMSNorm in float32,
# rotate_half RoPE) to produce outputs numerically identical to the
# HuggingFace reference. vLLM's fused kernels (RMSNorm, QKVParallel,
# get_rope) introduce small precision differences that compound across
# the 15 autoregressive steps of the code predictor, causing severe
# audio quality degradation (UTMOS ~4.26 → ~2.66).
#
# See: https://github.com/vllm-project/vllm-omni/issues/2274
class _RMSNorm(nn.Module):
"""RMSNorm matching HuggingFace's Qwen3TTSRMSNorm exactly.
Computes variance in float32 to avoid bfloat16 precision loss.
"""
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
class _RotaryEmbedding(nn.Module):
"""RoPE matching HuggingFace's Qwen3TTSRotaryEmbedding exactly.
Forces float32 computation for cos/sin, matching HF's torch.autocast(enabled=False).
"""
def __init__(self, config: Qwen3TTSTalkerCodePredictorConfig) -> None:
super().__init__()
head_dim = getattr(
config,
"head_dim",
config.hidden_size // config.num_attention_heads,
)
# Standard default RoPE
rope_theta = getattr(config, "rope_theta", 10000.0)
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# position_ids: [batch, seq_len]
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
position_ids_expanded = position_ids[:, None, :].float()
# Force float32 (matching HF)
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos()
sin = emb.sin()
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
class _CodePredictorAttention(nn.Module):
"""Standalone multi-head attention for code predictor.
Uses F.scaled_dot_product_attention with HF-compatible RoPE and RMSNorm.
Input: [B, seq_len, hidden_size], output: [B, seq_len, hidden_size].
"""
def __init__(
self,
config: Qwen3TTSTalkerCodePredictorConfig,
*,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.head_dim = getattr(
config,
"head_dim",
config.hidden_size // config.num_attention_heads,
)
self.scaling = self.head_dim**-0.5
self._use_gqa = self.num_kv_heads != self.num_heads
# Separate q/k/v projections matching HF (no fused packing)
self.q_proj = nn.Linear(
self.hidden_size,
self.num_heads * self.head_dim,
bias=getattr(config, "attention_bias", False),
)
self.k_proj = nn.Linear(
self.hidden_size,
self.num_kv_heads * self.head_dim,
bias=getattr(config, "attention_bias", False),
)
self.v_proj = nn.Linear(
self.hidden_size,
self.num_kv_heads * self.head_dim,
bias=getattr(config, "attention_bias", False),
)
self.o_proj = nn.Linear(
self.num_heads * self.head_dim,
self.hidden_size,
bias=False,
)
self.q_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim)
hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim)
q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)
k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2)
v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2)
cos, sin = position_embeddings
# cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads
cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]
sin = sin.unsqueeze(1)
q = (q * cos) + (_rotate_half(q) * sin)
k = (k * cos) + (_rotate_half(k) * sin)
attn_out = F.scaled_dot_product_attention(
q,
k,
v,
scale=self.scaling,
is_causal=True,
enable_gqa=self._use_gqa,
)
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
output = self.o_proj(attn_out)
return output
class _CodePredictorMLP(nn.Module):
"""SiLU-gated MLP for code predictor, matching HF's Qwen3TTSTalkerTextMLP."""
def __init__(
self,
config: Qwen3TTSTalkerCodePredictorConfig,
*,
prefix: str = "",
) -> None:
super().__init__()
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
class _CodePredictorDecoderLayer(nn.Module):
"""Transformer decoder layer for code predictor (SDPA, no KV cache)."""
def __init__(
self,
config: Qwen3TTSTalkerCodePredictorConfig,
*,
prefix: str = "",
) -> None:
super().__init__()
self.self_attn = _CodePredictorAttention(config, prefix=f"{prefix}.self_attn")
self.mlp = _CodePredictorMLP(config, prefix=f"{prefix}.mlp")
self.input_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(hidden_states, position_embeddings)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
# ===================================================================
# Code Predictor Transformer Model
# ===================================================================
class Qwen3TTSTalkerCodePredictorModelVLLM(nn.Module):
"""Transformer model for the code predictor (re-prefill, no KV cache)."""
def __init__(
self,
config: Qwen3TTSTalkerCodePredictorConfig,
*,
talker_hidden_size: int | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.layers = nn.ModuleList(
[_CodePredictorDecoderLayer(config, prefix=f"{prefix}.layers.{i}") for i in range(config.num_hidden_layers)]
)
self.norm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = _RotaryEmbedding(config)
# Codec embeddings: one per residual group. Stored in talker hidden dim
# (some checkpoints use talker_hidden_size != code_predictor hidden_size).
emb_dim = int(talker_hidden_size) if talker_hidden_size is not None else int(config.hidden_size)
self.codec_embedding = nn.ModuleList(
[nn.Embedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
)
def get_input_embeddings(self) -> nn.ModuleList:
return self.codec_embedding
def forward(
self,
inputs_embeds: torch.Tensor,
position_ids: torch.Tensor,
) -> torch.Tensor:
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids)
for layer in self.layers:
hidden_states = layer(hidden_states, position_embeddings)
hidden_states = self.norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
param = params_dict.get(name)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
# ===================================================================
# Code Predictor Wrapper (optimized re-prefill + torch.compile)
# ===================================================================
class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
"""vLLM-native code_predictor for the AR talker (residual codebooks).
Re-prefill approach: each AR step forwards the full growing sequence
through the 5-layer transformer. No KV cache needed. This trades
~O(T^2) extra attention FLOPs (negligible for T=16, 5 layers) for
zero KV cache management overhead and a simpler execution model.
Uses HF-compatible layers (plain nn.Linear, float32 RMSNorm, rotate_half
RoPE) to ensure numerical fidelity with the reference implementation.
Precision matters here because small errors compound across 15 AR steps.
Optimizations preserved:
1. torch.compile on model forward -- fuses small kernel launches.
2. Pre-allocated embedding buffer [B, max_seq, H] -- no torch.cat per step.
3. Projection caching -- each token projected once and cached.
4. Pre-allocated position_ids -- no torch.arange per step.
5. Inline sampling -- no custom op / forward_context overhead.
6. Cached module references -- bypass nn.Module.__call__ overhead.
7. CUDA graphs per batch-size bucket.
"""
class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(CodePredictorWrapper):
"""Qwen3-TTS code predictor (CUDA graphs, per-call sampling, projection)."""
def __init__(
self,
@@ -322,250 +31,24 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
talker_config: Qwen3TTSTalkerConfig,
prefix: str = "code_predictor",
) -> None:
super().__init__()
self._vllm_config = vllm_config
self.config = config
self.talker_config = talker_config
self.model = Qwen3TTSTalkerCodePredictorModelVLLM(
config,
super().__init__(
vllm_config=vllm_config,
cp_config=config,
wrapper_config=CodePredictorWrapperConfig(
use_cuda_graphs=True,
use_parallel_embedding=False,
use_projection=(config.hidden_size != talker_config.hidden_size),
return_proj_buf=False,
sampling_mode="per_call",
),
talker_hidden_size=int(talker_config.hidden_size),
prefix=f"{prefix}.model",
prefix=prefix,
)
self.lm_head = nn.ModuleList(
[nn.Linear(config.hidden_size, config.vocab_size, bias=False) for _ in range(config.num_code_groups - 1)]
)
if config.hidden_size != talker_config.hidden_size:
self.small_to_mtp_projection = nn.Linear(talker_config.hidden_size, config.hidden_size, bias=True)
else:
self.small_to_mtp_projection = nn.Identity()
self._num_groups = int(config.num_code_groups)
self._talker_hidden = int(talker_config.hidden_size)
self._cp_hidden = int(config.hidden_size)
# Pre-allocated buffers (lazily initialized on first forward).
self._proj_buf: torch.Tensor | None = None
self._model_dtype: torch.dtype | None = None
# torch.compile + warmup state (lazily initialized in _setup_compile).
self._compiled_model_fwd = None
self._bucket_sizes: list[int] = []
self._bucket_pos_ids: dict[int, torch.Tensor] = {}
self._lm_heads_list: list[nn.Module] | None = None
self._codec_embeds_list: list[nn.Module] | None = None
self._cuda_graphs: dict[int, tuple[torch.cuda.CUDAGraph, torch.Tensor]] = {}
def get_input_embeddings(self) -> nn.ModuleList:
return self.model.get_input_embeddings()
# Store talker_config for backward compat (accessed by some callers)
self.talker_config = talker_config
self._vllm_config = vllm_config
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with vllm config context (required for VocabParallelEmbedding)."""
with set_current_vllm_config(self._vllm_config):
loaded: set[str] = set()
model_weights: list[tuple[str, torch.Tensor]] = []
other_weights: list[tuple[str, torch.Tensor]] = []
for name, w in weights:
if name.startswith("model."):
model_weights.append((name[len("model.") :], w))
else:
other_weights.append((name, w))
loaded_model = self.model.load_weights(model_weights)
loaded |= {f"model.{n}" for n in loaded_model}
params = dict(self.named_parameters(remove_duplicate=False))
for name, w in other_weights:
if name not in params:
continue
default_weight_loader(params[name], w)
loaded.add(name)
return loaded
# ------------------------------------------------------------------
# Pre-allocated buffer management
# ------------------------------------------------------------------
def _ensure_buffers(self, device: torch.device, dtype: torch.dtype) -> None:
max_seq = self._num_groups + 1
if self._proj_buf is not None and self._proj_buf.device == device and self._proj_buf.dtype == dtype:
return
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
self._proj_buf = torch.zeros(
max_bsz,
max_seq,
self._cp_hidden,
dtype=dtype,
device=device,
)
def _setup_compile(self) -> None:
"""Lazily set up torch.compile with manual CUDA graph capture."""
if self._compiled_model_fwd is not None:
return
# Cache model parameter dtype so forward() doesn't need to query it
# on every call. Also ensures warmup buffers match model precision
# even when upstream modules produce a different dtype (#2385).
self._model_dtype = next(self.model.parameters()).dtype
self._lm_heads_list = list(self.lm_head)
self._codec_embeds_list = list(self.model.codec_embedding)
if not current_omni_platform.supports_torch_inductor():
logger.warning_once("code_predictor: torch.compile disabled")
self._compiled_model_fwd = self.model.forward
return
# torch.compile fuses RMSNorm/RoPE in ways that lose float32
# precision, compounding across 15 AR steps. Use torch.compile
# with options that disable the problematic fusions while still
# getting kernel fusion benefits for the linear layers and SDPA.
self._compiled_model_fwd = torch.compile(
self.model.forward,
dynamic=False,
options={
"epilogue_fusion": False,
},
)
self._warmup_buckets()
self._capture_cuda_graphs()
logger.info("code_predictor: torch.compile (no epilogue fusion) + CUDA graphs")
def _padded_bsz(self, bsz: int) -> int:
for bucket in self._bucket_sizes:
if bsz <= bucket:
return bucket
return bsz
def _warmup_buckets(self) -> None:
"""Warmup power-of-2 batch-size buckets to front-load Inductor compilation."""
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
bucket_sizes = [1 << i for i in range(max_bsz.bit_length()) if (1 << i) <= max_bsz]
if max_bsz not in bucket_sizes:
bucket_sizes.append(max_bsz)
self._bucket_sizes = sorted(bucket_sizes)
max_seq = self._num_groups + 1
device = next(self.model.parameters()).device
# Ensure proj_buf matches model parameter dtype to avoid dtype
# mismatch during warmup compilation (see #2385).
self._ensure_buffers(device, self._model_dtype)
proj_buf = self._proj_buf
for bsz in self._bucket_sizes:
# position_ids: [batch, seq_len] for HF-style RoPE
pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1)
self._bucket_pos_ids[bsz] = pos_ids
for _ in range(3):
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
def _capture_cuda_graphs(self) -> None:
"""Capture a CUDA graph per bucket using vLLM's global graph pool."""
from vllm.platforms import current_platform
pool = current_platform.get_global_graph_pool()
max_seq = self._num_groups + 1
proj_buf = self._proj_buf
for bsz in self._bucket_sizes:
static_input = proj_buf[:bsz, :max_seq, :]
pos_ids = self._bucket_pos_ids[bsz]
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g, pool=pool):
static_output = self._compiled_model_fwd(static_input, pos_ids)
self._cuda_graphs[bsz] = (g, static_output)
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
# ------------------------------------------------------------------
# Optimized forward: re-prefill + torch.compile + projection cache
# ------------------------------------------------------------------
@torch.inference_mode()
def forward(
self,
layer0_code: torch.Tensor,
layer0_embed: torch.Tensor,
last_talker_hidden: torch.Tensor,
do_sample: bool = True,
temperature: float = 0.9,
top_k: int = 50,
top_p: float = 1.0,
) -> torch.Tensor:
"""Predict residual codebooks 1..Q-1 autoregressively via re-prefill.
torch.compile fuses the ~60 small kernel launches per step into fewer
fused kernels, reducing kernel launch overhead by ~75%.
Projection caching: each token is projected once via small_to_mtp_projection
and cached in _proj_buf, avoiding redundant re-projection of past tokens.
"""
bsz = int(layer0_code.shape[0])
num_groups = self._num_groups
device = layer0_code.device
all_codes = torch.empty(bsz, num_groups, dtype=torch.long, device=device)
all_codes[:, 0] = layer0_code.reshape(bsz)
# _setup_compile caches _model_dtype on first call; use it for buffers
# so they always match model weight precision (#2385).
self._setup_compile()
dtype = self._model_dtype
self._ensure_buffers(device, dtype)
proj_buf = self._proj_buf
max_seq = self._num_groups + 1
projection = self.small_to_mtp_projection
model_fwd = self._compiled_model_fwd
lm_heads = self._lm_heads_list
codec_embeds = self._codec_embeds_list
use_sampling = do_sample and temperature > 0
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
if use_sampling and top_p != 1.0:
raise NotImplementedError(
"top_p sampling is not implemented for the vLLM-native code predictor; please set top_p=1.0."
)
padded_bsz = self._padded_bsz(bsz)
proj_buf[:padded_bsz].zero_()
proj_buf[:bsz, 0, :] = projection(last_talker_hidden.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
proj_buf[:bsz, 1, :] = projection(layer0_embed.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
full_pos_ids = self._bucket_pos_ids.get(padded_bsz)
if full_pos_ids is None:
full_pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1)
# Use captured CUDA graph if available, otherwise call compiled fn.
cuda_graph_entry = self._cuda_graphs.get(padded_bsz)
for step in range(1, num_groups):
if cuda_graph_entry is not None:
cuda_graph_entry[0].replay()
hidden_out = cuda_graph_entry[1]
else:
hidden_out = model_fwd(proj_buf[:padded_bsz, :max_seq, :], full_pos_ids)
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
if use_sampling:
scaled = logits * inv_temperature
if top_k > 0:
topk_vals, _ = scaled.topk(top_k, dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
probs = F.softmax(scaled, dim=-1)
next_ids = torch.multinomial(probs, num_samples=1)
else:
next_ids = logits.argmax(dim=-1, keepdim=True)
all_codes[:, step] = next_ids.reshape(bsz)
if step < num_groups - 1:
new_embed = codec_embeds[step - 1](next_ids)
proj_buf[:bsz, step + 1, :] = projection(new_embed.reshape(bsz, 1, -1)).reshape(bsz, -1)
return all_codes
return super().load_weights(weights)
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.qwen3 import Qwen3Model
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper, maybe_prefix
from vllm.multimodal.audio import AudioResampler
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.output_templates import OmniOutput
@@ -414,6 +415,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
# In-memory LRU cache for voice extraction artifacts (Base voice clone).
self._voice_cache = VoiceEmbeddingCache()
raw_subtalker_sampling = getattr(vllm_config.model_config, "subtalker_sampling_params", None)
self._subtalker_sampling_params: dict[str, Any] = (
dict(raw_subtalker_sampling) if isinstance(raw_subtalker_sampling, Mapping) else {}
)
# -------------------- vLLM required hooks --------------------
@@ -1094,9 +1099,8 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
# Resample to 24kHz for speaker encoder.
target_sr = int(getattr(self.config.speaker_encoder_config, "sample_rate", 24000))
if sr != target_sr:
from vllm.multimodal.audio import resample_audio_resampy
wav = resample_audio_resampy(wav.astype(np.float32), orig_sr=int(sr), target_sr=target_sr)
resampler = AudioResampler(target_sr=target_sr)
wav = resampler.resample(wav.astype(np.float32), orig_sr=int(sr))
sr = target_sr
# Follow official implementation: mel_spectrogram expects 24kHz.
@@ -1638,6 +1642,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
input_embeds: torch.Tensor,
last_talker_hidden: torch.Tensor,
text_step: torch.Tensor,
do_sample: bool | None = None,
temperature: float | None = None,
top_k: int | None = None,
top_p: float | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""GPU fast-path used by OmniGPUModelRunner to predict residual codebooks (1..Q-1).
Returns (inputs_embeds, audio_codes) for the current step."""
@@ -1656,15 +1664,24 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
audio_codes = input_ids.reshape(bsz, 1)
return (last_id_hidden + text_step).reshape(bsz, -1), audio_codes
# Predict residual codes (1..Q-1) with HF reference sampling params.
subtalker_params = self._subtalker_sampling_params
if do_sample is None:
do_sample = bool(subtalker_params.get("do_sample", True))
if temperature is None:
temperature = float(subtalker_params.get("temperature", 0.9))
if top_k is None:
top_k = int(subtalker_params.get("top_k", 50))
if top_p is None:
top_p = float(subtalker_params.get("top_p", 1.0))
audio_codes = self.code_predictor(
layer0_code=input_ids.reshape(bsz, 1),
layer0_embed=last_id_hidden,
last_talker_hidden=past_hidden,
do_sample=True,
temperature=0.9,
top_k=50,
top_p=1.0,
do_sample=do_sample,
temperature=temperature,
top_k=top_k,
top_p=top_p,
) # [B, Q]
# Map invalid layer-0 ids (e.g. EOS) to PAD=0 so SpeechTokenizer sees only real codes.
@@ -22,7 +22,7 @@ import soundfile as sf
import torch
from torch.nn.utils.rnn import pad_sequence
from transformers import AutoConfig, AutoFeatureExtractor, AutoModel
from vllm.multimodal.audio import resample_audio_resampy
from vllm.multimodal.audio import AudioResampler
from vllm.multimodal.media.audio import load_audio as _load_audio_file
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
@@ -161,7 +161,8 @@ class Qwen3TTSTokenizer:
audio = np.mean(audio, axis=-1)
if sr != target_sr:
audio = resample_audio_resampy(audio, orig_sr=sr, target_sr=target_sr)
resampler = AudioResampler(target_sr=target_sr)
audio = resampler.resample(audio, orig_sr=sr)
return audio.astype(np.float32)
@@ -209,7 +210,8 @@ class Qwen3TTSTokenizer:
if a.ndim > 1:
a = np.mean(a, axis=-1)
if int(sr) != target_sr:
a = resample_audio_resampy(a.astype(np.float32), orig_sr=int(sr), target_sr=target_sr)
resampler = AudioResampler(target_sr=target_sr)
a = resampler.resample(a.astype(np.float32), orig_sr=int(sr))
out.append(a.astype(np.float32))
return out
@@ -17,17 +17,17 @@ import operator
from itertools import accumulate
import onnxruntime
import sox
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchaudio.compliance.kaldi as kaldi
from torch import Tensor
from vllm_omni.utils.audio import mel_filter_bank
from vllm_omni.model_executor.models.whisper_utils import Conv1d, ConvTranspose1d
from vllm_omni.utils.audio import mel_filter_bank, peak_normalize
from .core_vq import DistributedGroupResidualVectorQuantization
from .whisper_encoder import Conv1d, ConvTranspose1d, WhisperEncoder
from .whisper_encoder import WhisperEncoder
def dynamic_range_compression_torch(x, c=1, clip_val=1e-5):
@@ -153,9 +153,6 @@ class XVectorExtractor(nn.Module):
audio_codec_with_xvector, sess_options=option, providers=providers
)
self.tfm = sox.Transformer()
self.tfm.norm(db_level=-6)
self.mel_ext = MelSpectrogramFeatures(
filter_length=1024,
hop_length=160,
@@ -183,8 +180,7 @@ class XVectorExtractor(nn.Module):
return norm_embedding.numpy(), ref_mel.permute(0, 2, 1).squeeze(0).numpy()
def sox_norm(self, audio):
wav_norm = self.tfm.build_array(input_array=audio, sample_rate_in=16000)
return wav_norm
return peak_normalize(audio, db_level=-6)
class WhisperEncoderVQ(WhisperEncoder):
@@ -23,6 +23,7 @@ import torch.nn.functional as F
from torch import Tensor, nn
from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func
from vllm_omni.model_executor.models.whisper_utils import Conv1d, Linear, sinusoids
from vllm_omni.utils.audio import mel_filter_bank
N_FFT = 400
@@ -102,30 +103,6 @@ def get_mel_audio(audio, padding=False, audio_vq_ds_rate=1, n_mels=128):
return mel
def sinusoids(length, channels, max_timescale=10000):
"""Returns sinusoids for positional embedding"""
assert channels % 2 == 0
log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
class Conv1d(nn.Conv1d):
def _conv_forward(self, x: Tensor, weight: Tensor, bias: Tensor | None) -> Tensor:
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
class ConvTranspose1d(nn.ConvTranspose1d):
def _conv_forward(self, x: Tensor, weight: Tensor, bias: Tensor | None) -> Tensor:
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
class Linear(nn.Linear):
def forward(self, x: Tensor) -> Tensor:
return F.linear(x, self.weight.to(x.dtype), None if self.bias is None else self.bias.to(x.dtype))
class MultiHeadAttention(nn.Module):
def __init__(self, n_state: int, n_head: int, use_flash_attention: bool = True):
super().__init__()
@@ -145,6 +145,12 @@ _OMNI_MODELS = {
"fish_speech_dac_decoder",
"FishSpeechDACDecoder",
),
## VoxCPM
"VoxCPMForConditionalGeneration": (
"voxcpm",
"voxcpm",
"VoxCPMForConditionalGeneration",
),
## VoxCPM2
"VoxCPM2TalkerForConditionalGeneration": (
"voxcpm2",
@@ -174,6 +180,23 @@ _OMNI_MODELS = {
"dynin_omni",
"DyninOmniForConditionalGeneration",
),
## Ming-flash-omni-2.0
"MingFlashOmniForConditionalGeneration": (
"ming_flash_omni",
"ming_flash_omni",
"MingFlashOmniForConditionalGeneration",
),
"MingFlashOmniThinkerForConditionalGeneration": (
"ming_flash_omni",
"ming_flash_omni_thinker",
"MingFlashOmniThinkerForConditionalGeneration",
),
# Alias: HF repo currently ships this architecture name in config.json
"BailingMM2NativeForConditionalGeneration": (
"ming_flash_omni",
"ming_flash_omni",
"MingFlashOmniForConditionalGeneration",
),
}
@@ -0,0 +1,7 @@
from .configuration_voxcpm import VoxCPMConfig
from .voxcpm import VoxCPMForConditionalGeneration
__all__ = [
"VoxCPMConfig",
"VoxCPMForConditionalGeneration",
]
@@ -0,0 +1,3 @@
from vllm_omni.transformers_utils.configs.voxcpm import VoxCPMConfig
__all__ = ["VoxCPMConfig"]
@@ -0,0 +1,886 @@
from __future__ import annotations
import json
import os
import sys
import tempfile
import warnings
import wave
from collections.abc import Callable, Generator, Iterable
from pathlib import Path
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
from tqdm import tqdm
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.output_templates import OmniOutput
from .voxcpm_loader import (
_build_prompt_cache_with_soundfile,
_device_to_string,
_force_cuda_available_for_npu,
_import_voxcpm_audio_vae_classes,
_import_voxcpm_base_model_class,
_is_torchcodec_load_error,
_normalize_dtype_name,
_prepare_runtime_model_dir,
_resolve_runtime_device,
)
from .voxcpm_runtime_utils import resolve_voxcpm_model_dir
from .voxcpm_stage_wrappers import _DirectVoxCPMAudioVAE, _DirectVoxCPMLatentGenerator
logger = init_logger(__name__)
_VOXCPM_LATENT_MAGIC = 131071
def _make_voxcpm_model_for_omni(base: type[Any]) -> type[Any]:
"""Subclass upstream VoxCPMModel: local ``_inference`` + ``latents_only`` prompt-cache generation."""
from voxcpm.model.utils import get_dtype
class VoxCPMModelForOmni(base):
@torch.inference_mode()
def build_prompt_cache(self, *args: Any, **kwargs: Any):
try:
return super().build_prompt_cache(*args, **kwargs)
except (ImportError, ModuleNotFoundError, RuntimeError) as exc:
if not _is_torchcodec_load_error(exc):
raise
return _build_prompt_cache_with_soundfile(self, *args, **kwargs)
@torch.inference_mode()
def _inference(
self,
text: torch.Tensor,
text_mask: torch.Tensor,
feat: torch.Tensor,
feat_mask: torch.Tensor,
min_len: int = 2,
max_len: int = 2000,
inference_timesteps: int = 10,
cfg_value: float = 2.0,
streaming: bool = False,
streaming_prefix_len: int = 3,
) -> Generator[tuple[torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]:
B, _, _, _ = feat.shape
feat_embed = self.feat_encoder(feat)
feat_embed = self.enc_to_lm_proj(feat_embed)
scale_emb = self.config.lm_config.scale_emb if self.config.lm_config.use_mup else 1.0
text_embed = self.base_lm.embed_tokens(text) * scale_emb
combined_embed = text_mask.unsqueeze(-1) * text_embed + feat_mask.unsqueeze(-1) * feat_embed
prefix_feat_cond = feat[:, -1, ...]
pred_feat_seq: list[torch.Tensor] = []
audio_patch_count = int(feat_mask.sum().item())
if audio_patch_count > 0:
context_len = min(streaming_prefix_len - 1, audio_patch_count)
prompt_context_patches = list(feat[:, -context_len:, :, :].split(1, dim=1))
pred_feat_seq = prompt_context_patches + pred_feat_seq
enc_outputs, kv_cache_tuple = self.base_lm(
inputs_embeds=combined_embed,
is_causal=True,
)
self.base_lm.kv_cache.fill_caches(kv_cache_tuple)
enc_outputs = self.fsq_layer(enc_outputs) * feat_mask.unsqueeze(-1) + enc_outputs * text_mask.unsqueeze(-1)
lm_hidden = enc_outputs[:, -1, :]
residual_enc_outputs, residual_kv_cache_tuple = self.residual_lm(
inputs_embeds=enc_outputs + feat_mask.unsqueeze(-1) * feat_embed,
is_causal=True,
)
self.residual_lm.kv_cache.fill_caches(residual_kv_cache_tuple)
residual_hidden = residual_enc_outputs[:, -1, :]
for step_idx in tqdm(range(max_len)):
dit_hidden = self.lm_to_dit_proj(lm_hidden) + self.res_to_dit_proj(residual_hidden)
pred_feat = self.feat_decoder(
mu=dit_hidden,
patch_size=self.patch_size,
cond=prefix_feat_cond.transpose(1, 2).contiguous(),
n_timesteps=inference_timesteps,
cfg_value=cfg_value,
).transpose(1, 2)
curr_embed = self.enc_to_lm_proj(self.feat_encoder(pred_feat.unsqueeze(1)))
pred_feat_seq.append(pred_feat.unsqueeze(1))
prefix_feat_cond = pred_feat
if streaming:
pred_feat_chunk = torch.cat(pred_feat_seq[-streaming_prefix_len:], dim=1)
feat_pred = rearrange(pred_feat_chunk, "b t p d -> b d (t p)", b=B, p=self.patch_size)
yield feat_pred, pred_feat_seq
stop_flag = self.stop_head(self.stop_actn(self.stop_proj(lm_hidden))).argmax(dim=-1)[0].cpu().item()
if step_idx > min_len and stop_flag == 1:
break
lm_hidden = self.base_lm.forward_step(
curr_embed[:, 0, :],
torch.tensor([self.base_lm.kv_cache.step()], device=curr_embed.device),
).clone()
lm_hidden = self.fsq_layer(lm_hidden)
residual_hidden = self.residual_lm.forward_step(
lm_hidden + curr_embed[:, 0, :],
torch.tensor([self.residual_lm.kv_cache.step()], device=curr_embed.device),
).clone()
if not streaming:
pred_feat_seq_cat = torch.cat(pred_feat_seq, dim=1)
feat_pred = rearrange(pred_feat_seq_cat, "b t p d -> b d (t p)", b=B, p=self.patch_size)
yield feat_pred, pred_feat_seq_cat.squeeze(0).cpu()
@torch.inference_mode()
def generate_latents_with_prompt_cache(
self,
target_text: str,
prompt_cache: dict,
min_len: int = 2,
max_len: int = 2000,
inference_timesteps: int = 10,
cfg_value: float = 2.0,
retry_badcase: bool = False,
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0,
streaming_prefix_len: int = 3,
) -> tuple[None, torch.Tensor, torch.Tensor]:
return next(
self._generate_with_prompt_cache(
target_text=target_text,
prompt_cache=prompt_cache,
min_len=min_len,
max_len=max_len,
inference_timesteps=inference_timesteps,
cfg_value=cfg_value,
retry_badcase=retry_badcase,
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
streaming=False,
streaming_prefix_len=streaming_prefix_len,
latents_only=True,
)
)
@torch.inference_mode()
def generate_latents_with_prompt_cache_streaming(
self,
target_text: str,
prompt_cache: dict,
min_len: int = 2,
max_len: int = 2000,
inference_timesteps: int = 10,
cfg_value: float = 2.0,
retry_badcase: bool = False,
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0,
streaming_prefix_len: int = 3,
) -> Generator[tuple[None, torch.Tensor, torch.Tensor], None, None]:
return self._generate_with_prompt_cache(
target_text=target_text,
prompt_cache=prompt_cache,
min_len=min_len,
max_len=max_len,
inference_timesteps=inference_timesteps,
cfg_value=cfg_value,
retry_badcase=retry_badcase,
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
streaming=True,
streaming_prefix_len=streaming_prefix_len,
latents_only=True,
)
@torch.inference_mode()
def _generate_with_prompt_cache(
self,
target_text: str,
prompt_cache: dict,
min_len: int = 2,
max_len: int = 2000,
inference_timesteps: int = 10,
cfg_value: float = 2.0,
retry_badcase: bool = False,
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0,
streaming: bool = False,
streaming_prefix_len: int = 3,
latents_only: bool = False,
) -> Generator[tuple[torch.Tensor | None, torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]:
if retry_badcase and streaming:
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
retry_badcase = False
if prompt_cache is None:
prompt_audio_feat = torch.empty((0, self.patch_size, self.audio_vae.latent_dim), dtype=torch.float32)
text = target_text
else:
prompt_audio_feat = prompt_cache["audio_feat"]
prompt_text = prompt_cache["prompt_text"]
text = prompt_text + target_text
text_token = torch.LongTensor(self.text_tokenizer(text))
text_token = torch.cat(
[
text_token,
torch.tensor([self.audio_start_token], dtype=torch.int32, device=text_token.device),
],
dim=-1,
)
target_text_token = torch.LongTensor(self.text_tokenizer(target_text))
audio_length = prompt_audio_feat.size(0)
text_length = text_token.shape[0]
text_pad_token = torch.zeros(audio_length, dtype=torch.int32, device=text_token.device)
audio_pad_feat = torch.zeros(
(text_token.shape[0], self.patch_size, self.audio_vae.latent_dim),
dtype=torch.float32,
device=text_token.device,
)
text_token = torch.cat([text_token, text_pad_token])
audio_feat = torch.cat([audio_pad_feat, prompt_audio_feat], dim=0)
text_mask = (
torch.cat([torch.ones(text_length), torch.zeros(audio_length)]).type(torch.int32).to(text_token.device)
)
audio_mask = (
torch.cat([torch.zeros(text_length), torch.ones(audio_length)]).type(torch.int32).to(text_token.device)
)
text_token = text_token.unsqueeze(0).to(self.device)
text_mask = text_mask.unsqueeze(0).to(self.device)
audio_feat = audio_feat.unsqueeze(0).to(self.device).to(get_dtype(self.config.dtype))
audio_mask = audio_mask.unsqueeze(0).to(self.device)
target_text_length = len(self.text_tokenizer(target_text))
retry_badcase_times = 0
while retry_badcase_times < retry_badcase_max_times:
inference_result = self._inference(
text_token,
text_mask,
audio_feat,
audio_mask,
min_len=min_len,
max_len=min(int(target_text_length * retry_badcase_ratio_threshold + 10), max_len),
inference_timesteps=inference_timesteps,
cfg_value=cfg_value,
streaming=streaming,
streaming_prefix_len=streaming_prefix_len,
)
if streaming:
patch_len = self.patch_size * self.chunk_size
for latent_pred, pred_audio_feat in inference_result:
if latents_only:
decode_audio = None
yield (decode_audio, target_text_token, latent_pred)
else:
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
yield (decode_audio, target_text_token, pred_audio_feat)
break
latent_pred, pred_audio_feat = next(inference_result)
if retry_badcase and pred_audio_feat.shape[0] >= target_text_length * retry_badcase_ratio_threshold:
ratio = pred_audio_feat.shape[0] / target_text_length
print(f" Badcase detected, audio_text_ratio={ratio}, retrying...", file=sys.stderr)
retry_badcase_times += 1
continue
break
if not streaming:
if latents_only:
decode_audio = None
else:
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
patch_len = self.patch_size * self.chunk_size
if audio_mask.sum().item() > 0:
decode_audio = decode_audio[..., patch_len * (streaming_prefix_len - 1) :].squeeze(1).cpu()
else:
decode_audio = decode_audio[..., :].squeeze(1).cpu()
yield (decode_audio, target_text_token, pred_audio_feat)
VoxCPMModelForOmni.__name__ = "VoxCPMModelForOmni"
VoxCPMModelForOmni.__qualname__ = "VoxCPMModelForOmni"
return VoxCPMModelForOmni
def _import_voxcpm_model_class() -> type[Any]:
base = _import_voxcpm_base_model_class()
return _make_voxcpm_model_for_omni(base)
def _load_native_voxcpm_model(
model_path: str,
*,
device: torch.device,
dtype: str | None,
):
VoxCPMModel = _import_voxcpm_model_class()
model_dir = resolve_voxcpm_model_dir(model_path)
runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype=dtype)
if device.type == "npu" and hasattr(torch, "npu"):
torch.npu.set_device(device)
with _force_cuda_available_for_npu(device):
return VoxCPMModel.from_local(
runtime_model_path,
optimize=device.type == "cuda",
)
def _load_native_voxcpm_latent_generator(
model_path: str,
*,
device: torch.device,
dtype: str | None,
) -> _DirectVoxCPMLatentGenerator:
return _DirectVoxCPMLatentGenerator(_load_native_voxcpm_model(model_path, device=device, dtype=dtype))
def _load_native_voxcpm_audio_vae(
model_path: str,
*,
device: torch.device,
) -> _DirectVoxCPMAudioVAE:
AudioVAE, AudioVAEConfig = _import_voxcpm_audio_vae_classes()
model_dir = resolve_voxcpm_model_dir(model_path)
runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype="float32")
config_dict = json.loads((Path(runtime_model_path) / "config.json").read_text())
audio_vae_config = config_dict.get("audio_vae_config")
audio_vae = AudioVAE(config=AudioVAEConfig(**audio_vae_config)) if audio_vae_config is not None else AudioVAE()
state_dict = torch.load(
Path(runtime_model_path) / "audiovae.pth",
map_location="cpu",
weights_only=True,
)["state_dict"]
audio_vae.load_state_dict(state_dict, strict=True)
audio_vae = audio_vae.to(device=device, dtype=torch.float32).eval()
if device.type == "npu" and hasattr(torch, "npu"):
torch.npu.set_device(device)
patch_size = int(config_dict.get("patch_size", 2))
return _DirectVoxCPMAudioVAE(audio_vae, patch_size=patch_size)
class VoxCPMForConditionalGeneration(nn.Module):
input_modalities = "audio"
_LATENT_STAGES = {"latent_generator", "latent", "ar_dit"}
_VAE_STAGES = {"vae", "audio_vae"}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
del prefix
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
self.model_stage = getattr(vllm_config.model_config, "model_stage", "latent_generator")
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
self.inject_omni_request_id_into_runtime_info = True
self._pipeline = None
self._latent_stream_gens: dict[str, Any] = {}
self._latent_stream_terminal_pending: dict[str, int] = {}
self._latent_stream_completed: set[str] = set()
self._next_local_stream_key = 0
self._ar_emit_stop_token = True
def _runner_hidden_device_dtype(self) -> tuple[torch.device, torch.dtype]:
device = _resolve_runtime_device(self.vllm_config)
model_config = getattr(self.vllm_config, "model_config", None)
dtype = getattr(model_config, "dtype", torch.float32) if model_config is not None else torch.float32
return device, dtype
def _ensure_model_loaded(self):
if self._pipeline is not None:
return
target_device = _resolve_runtime_device(self.vllm_config)
model_dtype = getattr(self.vllm_config.model_config, "dtype", None)
normalized_dtype = _normalize_dtype_name(model_dtype)
if self.model_stage in self._LATENT_STAGES:
self._pipeline = _load_native_voxcpm_latent_generator(
self.model_path,
device=target_device,
dtype=normalized_dtype,
)
elif self.model_stage in self._VAE_STAGES:
self._pipeline = _load_native_voxcpm_audio_vae(
self.model_path,
device=target_device,
)
else:
raise ValueError(
f"Unsupported VoxCPM model_stage: {self.model_stage}. "
"pure_voxcpm only supports split-stage latent_generator/vae inference."
)
logger.info("Loaded VoxCPM stage '%s' on %s", self.model_stage, _device_to_string(target_device))
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
del weights
self._ensure_model_loaded()
return set()
@staticmethod
def _extract_val(info: dict[str, Any], key: str, default: Any) -> Any:
value = info.get(key, default)
if isinstance(value, list):
return value[0] if value else default
return value
def _resolve_stream_request_key(self, info: dict[str, Any]) -> str:
request_key = info.get("__voxcpm_stream_key")
if request_key is not None:
return str(request_key)
request_key = info.get("_omni_req_id")
if request_key is not None:
request_key = str(request_key)
info["__voxcpm_stream_key"] = request_key
return request_key
request_key = f"voxcpm-local-{self._next_local_stream_key}"
self._next_local_stream_key += 1
info["__voxcpm_stream_key"] = request_key
return str(request_key)
def _recover_latent_from_input_ids(self, input_ids: torch.Tensor | None) -> torch.Tensor | None:
if input_ids is None or input_ids.numel() == 0:
return None
flat_ids = input_ids.detach().reshape(-1).to("cpu")
if flat_ids.numel() < 4 or int(flat_ids[0].item()) != _VOXCPM_LATENT_MAGIC:
return None
latent_dim = int(flat_ids[1].item())
time_dim = int(flat_ids[2].item())
payload = flat_ids[3:]
expected = latent_dim * time_dim
if latent_dim <= 0 or time_dim <= 0:
raise ValueError(f"Invalid VoxCPM latent header: latent_dim={latent_dim}, time_dim={time_dim}")
if int(payload.numel()) != expected:
raise ValueError(
"Invalid VoxCPM latent payload size: "
f"expected={expected}, actual={int(payload.numel())}, "
f"latent_dim={latent_dim}, time_dim={time_dim}"
)
packed = payload.to(dtype=torch.int32).to(torch.uint16)
return packed.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim)
def _maybe_recover_vae_infos(
self,
infos: list[dict[str, Any]],
input_ids: torch.Tensor | None,
*,
async_chunk: bool,
) -> list[dict[str, Any]]:
if not async_chunk:
return infos
if any(self._extract_val(info, "latent_audio_feat", None) is not None for info in infos):
return infos
recovered = self._recover_latent_from_input_ids(input_ids)
if recovered is None:
return infos
return [{"latent_audio_feat": recovered}]
@staticmethod
def _normalize_audio_samples(samples: Any) -> np.ndarray:
if isinstance(samples, torch.Tensor):
return samples.detach().cpu().float().reshape(-1).numpy()
return np.asarray(samples, dtype=np.float32).reshape(-1)
@classmethod
def _normalize_ref_audio(cls, ref_audio: Any) -> tuple[np.ndarray, int]:
if isinstance(ref_audio, str):
raise TypeError("String ref_audio should be handled as a path before waveform normalization.")
if isinstance(ref_audio, dict):
sample_rate = ref_audio.get("sample_rate") or ref_audio.get("sampling_rate") or ref_audio.get("sr")
samples = None
for key in ("audio", "wav", "samples", "array", "waveform"):
if key in ref_audio and ref_audio[key] is not None:
samples = ref_audio[key]
break
if sample_rate is None or samples is None:
raise ValueError("ref_audio dict must contain waveform data and sample rate.")
return cls._normalize_audio_samples(samples), int(sample_rate)
if isinstance(ref_audio, (list, tuple)):
if len(ref_audio) == 1:
return cls._normalize_ref_audio(ref_audio[0])
if len(ref_audio) == 2 and np.isscalar(ref_audio[1]):
return cls._normalize_audio_samples(ref_audio[0]), int(ref_audio[1])
raise TypeError(f"Unsupported ref_audio format: {type(ref_audio)!r}")
@staticmethod
def _write_temp_prompt_wav(waveform: np.ndarray, sample_rate: int) -> str:
prompt_file = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
prompt_file.close()
wav = np.asarray(waveform, dtype=np.float32).reshape(-1)
wav = np.clip(wav, -1.0, 1.0)
pcm16 = (wav * 32767.0).astype(np.int16)
with wave.open(prompt_file.name, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(int(sample_rate))
wav_file.writeframes(pcm16.tobytes())
return prompt_file.name
@classmethod
def _resolve_prompt_inputs(cls, info: dict[str, Any]) -> tuple[str | None, str | None, str | None]:
prompt_text = cls._extract_val(info, "prompt_text", None)
prompt_wav_path = cls._extract_val(info, "prompt_wav_path", None)
if prompt_wav_path:
if prompt_text is None:
prompt_text = cls._extract_val(info, "ref_text", None)
return prompt_wav_path, prompt_text, None
ref_audio = cls._extract_val(info, "ref_audio", None)
ref_text = cls._extract_val(info, "ref_text", None)
if ref_audio is None or ref_text is None:
return None, None, None
if isinstance(ref_audio, str):
return ref_audio, ref_text, None
waveform, sample_rate = cls._normalize_ref_audio(ref_audio)
temp_prompt_wav = cls._write_temp_prompt_wav(waveform, sample_rate)
return temp_prompt_wav, ref_text, temp_prompt_wav
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def _get_vocab_size(self) -> int:
model_config = getattr(self.vllm_config, "model_config", None)
if model_config is not None:
getter = getattr(model_config, "get_vocab_size", None)
if callable(getter):
try:
return int(getter())
except Exception:
pass
hf_config = getattr(model_config, "hf_text_config", None)
if hf_config is not None and hasattr(hf_config, "vocab_size"):
return int(hf_config.vocab_size)
return 32000
def _make_empty_output(
self,
*,
output_key: str,
payload_factory: Callable[[], torch.Tensor],
infos: list[dict[str, Any]],
sample_rate: int,
out_device: torch.device,
out_dtype: torch.dtype,
hidden_rows: int | None = None,
) -> OmniOutput:
if hidden_rows is None:
hidden_rows = len(infos)
return OmniOutput(
text_hidden_states=torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype),
multimodal_outputs={
output_key: [payload_factory() for _ in infos],
"sr": [torch.tensor(sample_rate, dtype=torch.int32) for _ in infos],
},
)
def _finalize_stage_output(
self,
*,
output_key: str,
outputs: list[torch.Tensor],
sample_rates: list[torch.Tensor],
out_device: torch.device,
out_dtype: torch.dtype,
hidden_rows: int | None = None,
) -> OmniOutput:
multimodal_outputs: dict[str, Any] = {output_key: outputs, "sr": sample_rates}
if hidden_rows is not None:
text_hidden_states = torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype)
elif outputs:
outputs_tensor = torch.stack(outputs)
text_hidden_states = (
outputs_tensor.unsqueeze(-1)
if outputs_tensor.ndim == 1
else outputs_tensor.reshape(-1, outputs_tensor.shape[-1])
)
else:
text_hidden_states = torch.zeros((0, 1), device=out_device, dtype=out_dtype)
text_hidden_states = text_hidden_states.to(device=out_device, dtype=out_dtype)
return OmniOutput(
text_hidden_states=text_hidden_states,
multimodal_outputs=multimodal_outputs,
)
def _forward_vae_stage(
self,
infos: list[dict[str, Any]],
*,
sample_rate: int,
async_chunk: bool,
out_device: torch.device,
out_dtype: torch.dtype,
) -> OmniOutput:
if all(self._extract_val(info, "latent_audio_feat", None) is None for info in infos):
self._ar_emit_stop_token = True
return self._make_empty_output(
output_key="model_outputs",
payload_factory=lambda: torch.zeros((0,), dtype=torch.float32),
infos=infos,
sample_rate=sample_rate,
out_device=out_device,
out_dtype=out_dtype,
)
outputs: list[torch.Tensor] = []
sample_rates: list[torch.Tensor] = []
for info in infos:
latent_audio_feat = self._extract_val(info, "latent_audio_feat", None)
audio_tensor = self._pipeline.decode(latent_audio_feat, trim_streaming_patch=async_chunk)
outputs.append(audio_tensor.float().cpu())
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
self._ar_emit_stop_token = True
return self._finalize_stage_output(
output_key="model_outputs",
outputs=outputs,
sample_rates=sample_rates,
out_device=out_device,
out_dtype=out_dtype,
)
def _forward_latent_stage(
self,
infos: list[dict[str, Any]],
*,
sample_rate: int,
async_chunk: bool,
out_device: torch.device,
out_dtype: torch.dtype,
hidden_rows: int,
) -> OmniOutput:
texts = [self._extract_val(info, "text", "") for info in infos]
if all(not text for text in texts):
self._ar_emit_stop_token = True
return self._make_empty_output(
output_key="latent_audio_feat",
payload_factory=lambda: torch.zeros((0,), dtype=torch.float32),
infos=infos,
sample_rate=sample_rate,
out_device=out_device,
out_dtype=out_dtype,
hidden_rows=hidden_rows,
)
outputs: list[torch.Tensor] = []
sample_rates: list[torch.Tensor] = []
last_chunk_flags: list[bool] | None = [] if async_chunk else None
payload_finished_flags: list[bool] | None = [] if async_chunk else None
for info in infos:
text = self._extract_val(info, "text", "")
cfg_value = float(self._extract_val(info, "cfg_value", 2.0))
inference_timesteps = int(self._extract_val(info, "inference_timesteps", 10))
min_len = int(self._extract_val(info, "min_len", 2))
max_len = int(self._extract_val(info, "max_len", self._extract_val(info, "max_new_tokens", 4096)))
retry_badcase = bool(self._extract_val(info, "retry_badcase", True))
retry_badcase_max_times = int(self._extract_val(info, "retry_badcase_max_times", 3))
retry_badcase_ratio_threshold = float(self._extract_val(info, "retry_badcase_ratio_threshold", 6.0))
streaming_prefix_len = int(self._extract_val(info, "streaming_prefix_len", 3))
request_key = self._resolve_stream_request_key(info)
created_temp: str | None = None
if async_chunk:
terminal_pending = self._latent_stream_terminal_pending.get(request_key, 0)
if terminal_pending > 0:
outputs.append(torch.zeros((0,), dtype=torch.float32))
assert last_chunk_flags is not None
last_chunk_flags.append(True)
assert payload_finished_flags is not None
payload_finished_flags.append(terminal_pending == 1)
if terminal_pending == 1:
self._latent_stream_terminal_pending.pop(request_key, None)
else:
self._latent_stream_terminal_pending[request_key] = terminal_pending - 1
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
continue
if request_key in self._latent_stream_completed:
outputs.append(torch.zeros((0,), dtype=torch.float32))
assert last_chunk_flags is not None
last_chunk_flags.append(True)
assert payload_finished_flags is not None
payload_finished_flags.append(False)
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
continue
if request_key not in self._latent_stream_gens:
prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info)
created_temp = temp_prompt_wav
self._latent_stream_gens[request_key] = self._pipeline.iter_latent_chunks_streaming(
text=text,
prompt_wav_path=prompt_wav_path,
prompt_text=prompt_text,
cfg_value=cfg_value,
inference_timesteps=inference_timesteps,
min_len=min_len,
max_len=max_len,
streaming_prefix_len=streaming_prefix_len,
retry_badcase=False,
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
)
generator = self._latent_stream_gens[request_key]
try:
chunk_latent, is_last = next(generator)
except StopIteration:
self._latent_stream_gens.pop(request_key, None)
self._latent_stream_terminal_pending[request_key] = 1
self._latent_stream_completed.add(request_key)
outputs.append(torch.zeros((0,), dtype=torch.float32))
assert last_chunk_flags is not None
last_chunk_flags.append(True)
assert payload_finished_flags is not None
payload_finished_flags.append(True)
else:
if is_last:
self._latent_stream_gens.pop(request_key, None)
self._latent_stream_terminal_pending[request_key] = 1
self._latent_stream_completed.add(request_key)
outputs.append(chunk_latent.detach().float().cpu())
assert last_chunk_flags is not None
last_chunk_flags.append(bool(is_last))
assert payload_finished_flags is not None
payload_finished_flags.append(False)
finally:
if created_temp is not None and os.path.exists(created_temp):
os.unlink(created_temp)
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
continue
prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info)
try:
latent_audio_feat = self._pipeline.generate_latents(
text=text,
prompt_wav_path=prompt_wav_path,
prompt_text=prompt_text,
cfg_value=cfg_value,
inference_timesteps=inference_timesteps,
min_len=min_len,
max_len=max_len,
retry_badcase=retry_badcase,
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
)
outputs.append(latent_audio_feat.float().cpu())
finally:
if temp_prompt_wav is not None and os.path.exists(temp_prompt_wav):
os.unlink(temp_prompt_wav)
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
self._ar_emit_stop_token = all(last_chunk_flags) if async_chunk and last_chunk_flags else True
output = self._finalize_stage_output(
output_key="latent_audio_feat",
outputs=outputs,
sample_rates=sample_rates,
out_device=out_device,
out_dtype=out_dtype,
hidden_rows=hidden_rows,
)
if async_chunk and payload_finished_flags is not None:
output.multimodal_outputs["finished"] = [
torch.tensor(flag, dtype=torch.bool) for flag in payload_finished_flags
]
return output
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> torch.Tensor:
del sampling_metadata
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if hidden_states is None:
device, dtype = self._runner_hidden_device_dtype()
hidden_states = torch.zeros((0, 1), device=device, dtype=dtype)
if hidden_states.ndim == 1:
hidden_states = hidden_states.unsqueeze(-1)
elif hidden_states.ndim > 2:
hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1])
vocab_size = self._get_vocab_size()
num_rows = int(hidden_states.shape[0])
logits = torch.zeros((num_rows, vocab_size), dtype=torch.float32, device=hidden_states.device)
eos_id = 2 if vocab_size > 2 else 0
safe_id = 1 if vocab_size > 1 and 1 != eos_id else 0
emit_stop = getattr(self, "_ar_emit_stop_token", True)
if num_rows > 0:
if emit_stop:
logits[:, eos_id] = 1.0e6
else:
logits[:, eos_id] = -1.0e9
logits[:, safe_id] = 1.0e6
return logits
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
model_intermediate_buffer: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
del positions, intermediate_tensors, inputs_embeds, kwargs
self._ensure_model_loaded()
out_device, out_dtype = self._runner_hidden_device_dtype()
if input_ids is not None and input_ids.device.type == out_device.type:
out_device = input_ids.device
infos = model_intermediate_buffer or runtime_additional_information or [{}]
hidden_rows = len(infos)
if input_ids is not None and len(input_ids.shape) > 0:
hidden_rows = max(hidden_rows, int(input_ids.shape[0]))
sample_rate = int(getattr(self._pipeline, "sample_rate", 24000))
async_chunk = bool(getattr(self.vllm_config.model_config, "async_chunk", False))
if self.model_stage in self._VAE_STAGES:
infos = self._maybe_recover_vae_infos(infos, input_ids, async_chunk=async_chunk)
return self._forward_vae_stage(
infos,
sample_rate=sample_rate,
async_chunk=async_chunk,
out_device=out_device,
out_dtype=out_dtype,
)
if self.model_stage in self._LATENT_STAGES:
return self._forward_latent_stage(
infos,
sample_rate=sample_rate,
async_chunk=async_chunk,
out_device=out_device,
out_dtype=out_dtype,
hidden_rows=hidden_rows,
)
raise ValueError(f"Unsupported VoxCPM model_stage at runtime: {self.model_stage}")
def make_empty_intermediate_tensors(
self, batch_size: int, dtype: torch.dtype, device: torch.device
) -> IntermediateTensors:
del batch_size, dtype, device
return {}
__all__ = ["VoxCPMForConditionalGeneration"]
@@ -0,0 +1,247 @@
from __future__ import annotations
import importlib
import json
import os
import shutil
import sys
import tempfile
from contextlib import contextmanager
from hashlib import sha256
from pathlib import Path
from typing import Any
from unittest.mock import patch
import numpy as np
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
logger = init_logger(__name__)
def _iter_voxcpm_src_candidates() -> list[Path]:
candidates: list[Path] = []
env_path = os.environ.get("VLLM_OMNI_VOXCPM_CODE_PATH")
if env_path:
candidates.append(Path(env_path).expanduser())
repo_root = Path(__file__).resolve().parents[4]
candidates.append(repo_root.parent / "VoxCPM" / "src")
unique_candidates: list[Path] = []
seen: set[str] = set()
for candidate in candidates:
candidate_key = str(candidate)
if candidate_key in seen:
continue
seen.add(candidate_key)
unique_candidates.append(candidate)
return unique_candidates
def _prepend_voxcpm_src(candidate: Path) -> None:
candidate_str = str(candidate)
if candidate_str not in sys.path:
sys.path.insert(0, candidate_str)
def _import_voxcpm_attrs(module_name: str, *attr_names: str) -> tuple[Any, ...]:
last_exc: ImportError | None = None
for candidate in _iter_voxcpm_src_candidates():
if not candidate.exists():
continue
_prepend_voxcpm_src(candidate)
try:
module = importlib.import_module(module_name)
return tuple(getattr(module, attr_name) for attr_name in attr_names)
except ImportError as exc:
last_exc = exc
try:
module = importlib.import_module(module_name)
return tuple(getattr(module, attr_name) for attr_name in attr_names)
except ImportError as exc:
last_exc = exc
raise ImportError(f"Failed to import {module_name}.") from last_exc
def _import_voxcpm_base_model_class():
"""Import upstream ``VoxCPMModel`` from ``VoxCPM/src/voxcpm`` (env, sibling tree, or pip)."""
try:
(VoxCPMModel,) = _import_voxcpm_attrs("voxcpm.model.voxcpm", "VoxCPMModel")
return VoxCPMModel
except ImportError as exc:
raise ImportError(
"Failed to import VoxCPMModel. Install the `voxcpm` package or set "
"`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory "
"(the parent of the `voxcpm` package that contains `model/` and `modules/`)."
) from exc
def _import_voxcpm_audio_vae_classes():
try:
return _import_voxcpm_attrs("voxcpm.modules.audiovae", "AudioVAE", "AudioVAEConfig")
except ImportError as exc:
raise ImportError(
"Failed to import VoxCPM AudioVAE. Install the `voxcpm` package or set "
"`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory."
) from exc
def _device_to_string(device: torch.device) -> str:
if device.index is None:
return device.type
return f"{device.type}:{device.index}"
def _normalize_dtype_name(dtype: Any) -> str | None:
if dtype is None:
return None
if isinstance(dtype, torch.dtype):
mapping = {
torch.bfloat16: "bfloat16",
torch.float16: "float16",
torch.float32: "float32",
}
return mapping.get(dtype, str(dtype).removeprefix("torch."))
dtype_str = str(dtype)
return dtype_str.removeprefix("torch.")
def _resolve_runtime_device(vllm_config: VllmConfig) -> torch.device:
try:
from vllm_omni.platforms import current_omni_platform
return current_omni_platform.get_torch_device()
except Exception:
pass
device = getattr(getattr(vllm_config, "device_config", None), "device", None)
if isinstance(device, torch.device):
return device
if device:
return torch.device(device)
return torch.device("cpu")
def _prepare_runtime_model_dir(
model_path: str | Path,
*,
target_device: torch.device,
target_dtype: str | None,
) -> str:
source_dir = Path(model_path)
config_path = source_dir / "config.json"
if not config_path.exists():
return str(source_dir)
config_text = config_path.read_text()
config_dict = json.loads(config_text)
desired_device = target_device.type
desired_dtype = target_dtype or config_dict.get("dtype")
if config_dict.get("device") == desired_device and config_dict.get("dtype") == desired_dtype:
return str(source_dir)
digest = sha256(f"{source_dir.resolve()}:{config_text}:{desired_device}:{desired_dtype}".encode()).hexdigest()[:16]
runtime_dir = Path(tempfile.gettempdir()) / "vllm_omni_voxcpm_runtime" / digest
runtime_dir.mkdir(parents=True, exist_ok=True)
for entry in source_dir.iterdir():
target = runtime_dir / entry.name
if entry.name == "config.json" or target.exists():
continue
try:
target.symlink_to(entry, target_is_directory=entry.is_dir())
except OSError as exc:
logger.warning(
"Falling back to copying VoxCPM runtime artifact %s into %s because symlink creation failed: %s",
entry,
runtime_dir,
exc,
)
if entry.is_dir():
shutil.copytree(entry, target, dirs_exist_ok=True)
else:
shutil.copy2(entry, target)
patched_config = dict(config_dict)
patched_config["device"] = desired_device
if desired_dtype is not None:
patched_config["dtype"] = desired_dtype
(runtime_dir / "config.json").write_text(json.dumps(patched_config, indent=2, sort_keys=True))
return str(runtime_dir)
@contextmanager
def _force_cuda_available_for_npu(device: torch.device):
if device.type != "npu":
yield
return
with patch("torch.cuda.is_available", return_value=True):
yield
def _is_torchcodec_load_error(exc: BaseException) -> bool:
message = str(exc).lower()
return "torchcodec" in message or "load_with_torchcodec" in message
def _load_audio_with_soundfile(
prompt_wav_path: str,
*,
sample_rate: int,
) -> torch.Tensor:
try:
import soundfile as sf
except ImportError:
raise
audio_np, source_sr = sf.read(prompt_wav_path, dtype="float32", always_2d=True)
audio = torch.from_numpy(np.ascontiguousarray(audio_np.T))
if audio.size(0) > 1:
audio = audio.mean(dim=0, keepdim=True)
if int(source_sr) != int(sample_rate):
try:
import torchaudio
except ImportError as exc:
raise ImportError("torchaudio is required for resampling prompt audio.") from exc
audio = torchaudio.functional.resample(audio, int(source_sr), int(sample_rate))
return audio
def _build_prompt_cache_with_soundfile(model: Any, *args: Any, **kwargs: Any) -> dict[str, Any]:
if args:
prompt_text = args[0]
prompt_wav_path = args[1] if len(args) > 1 else kwargs.get("prompt_wav_path")
else:
prompt_text = kwargs.get("prompt_text")
prompt_wav_path = kwargs.get("prompt_wav_path")
if not prompt_text or not prompt_wav_path:
raise ValueError("prompt_text and prompt_wav_path are required")
audio = _load_audio_with_soundfile(prompt_wav_path, sample_rate=int(model.sample_rate))
patch_len = model.patch_size * model.chunk_size
if audio.size(1) % patch_len != 0:
padding_size = patch_len - audio.size(1) % patch_len
audio = torch.nn.functional.pad(audio, (padding_size, 0))
audio_feat = model.audio_vae.encode(audio.to(model.device), model.sample_rate).cpu()
audio_feat = audio_feat.view(
model.audio_vae.latent_dim,
-1,
model.patch_size,
).permute(1, 2, 0)
return {
"prompt_text": prompt_text,
"audio_feat": audio_feat,
}
@@ -0,0 +1,44 @@
from __future__ import annotations
import json
import shutil
from pathlib import Path
def resolve_voxcpm_model_dir(model: str) -> Path:
model_path = Path(model).expanduser()
if model_path.exists():
return model_path
from huggingface_hub import snapshot_download
return Path(snapshot_download(repo_id=model))
def prepare_voxcpm_hf_config_dir(model_dir: str | Path, hf_config_dir: str | Path) -> Path:
model_dir = Path(model_dir).expanduser()
hf_config_dir = Path(hf_config_dir).expanduser()
hf_config_dir.mkdir(parents=True, exist_ok=True)
source_config_path = model_dir / "config.json"
if not source_config_path.exists():
raise FileNotFoundError(f"VoxCPM config.json not found under {model_dir}")
config_path = hf_config_dir / "config.json"
shutil.copy2(source_config_path, config_path)
source_generation_config_path = model_dir / "generation_config.json"
if source_generation_config_path.exists():
shutil.copy2(source_generation_config_path, hf_config_dir / "generation_config.json")
config_dict = json.loads(config_path.read_text(encoding="utf-8"))
config_dict["model_type"] = "voxcpm"
config_dict.setdefault("architectures", ["VoxCPMForConditionalGeneration"])
config_path.write_text(json.dumps(config_dict, indent=2, ensure_ascii=False), encoding="utf-8")
return hf_config_dir
__all__ = [
"prepare_voxcpm_hf_config_dir",
"resolve_voxcpm_model_dir",
]
@@ -0,0 +1,185 @@
from __future__ import annotations
import os
from collections.abc import Generator
from typing import Any
import torch
import torch.nn as nn
from einops import rearrange
class _DirectVoxCPMLatentGenerator:
def __init__(self, tts_model: Any):
self.tts_model = tts_model
self.sample_rate = int(getattr(tts_model, "sample_rate", 24000))
def generate_latents(
self,
*,
text: str,
prompt_wav_path: str | None = None,
prompt_text: str | None = None,
cfg_value: float = 2.0,
inference_timesteps: int = 10,
min_len: int = 2,
max_len: int = 4096,
retry_badcase: bool = True,
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0,
) -> torch.Tensor:
if not isinstance(text, str) or not text.strip():
raise ValueError("target text must be a non-empty string")
if (prompt_wav_path is None) != (prompt_text is None):
raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None")
if prompt_wav_path is not None and not os.path.exists(prompt_wav_path):
raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}")
prompt_cache = None
if prompt_wav_path is not None and prompt_text is not None:
prompt_cache = self.tts_model.build_prompt_cache(
prompt_text=prompt_text,
prompt_wav_path=prompt_wav_path,
)
gen_kw = dict(
target_text=" ".join(text.split()),
prompt_cache=prompt_cache,
min_len=min_len,
max_len=max_len,
inference_timesteps=inference_timesteps,
cfg_value=cfg_value,
retry_badcase=retry_badcase,
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
)
latent_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache", None)
if latent_entry is not None:
_, _, pred_audio_feat = latent_entry(**gen_kw)
else:
try:
_, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(
**gen_kw,
latents_only=True,
)
except TypeError:
_, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(**gen_kw)
return pred_audio_feat.detach().cpu().to(torch.float32)
def iter_latent_chunks_streaming(
self,
*,
text: str,
prompt_wav_path: str | None = None,
prompt_text: str | None = None,
cfg_value: float = 2.0,
inference_timesteps: int = 10,
min_len: int = 2,
max_len: int = 4096,
streaming_prefix_len: int = 3,
retry_badcase: bool = False,
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0,
) -> Generator[tuple[torch.Tensor, bool], None, None]:
"""Yield ``(latent_window, is_last_chunk)`` for Omni async_chunk latent to VAE."""
if not isinstance(text, str) or not text.strip():
raise ValueError("target text must be a non-empty string")
if (prompt_wav_path is None) != (prompt_text is None):
raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None")
if prompt_wav_path is not None and not os.path.exists(prompt_wav_path):
raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}")
prompt_cache = None
if prompt_wav_path is not None and prompt_text is not None:
prompt_cache = self.tts_model.build_prompt_cache(
prompt_text=prompt_text,
prompt_wav_path=prompt_wav_path,
)
gen_kw = dict(
target_text=" ".join(text.split()),
prompt_cache=prompt_cache,
min_len=min_len,
max_len=max_len,
inference_timesteps=inference_timesteps,
cfg_value=cfg_value,
retry_badcase=retry_badcase,
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
streaming_prefix_len=streaming_prefix_len,
)
stream_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache_streaming", None)
if stream_entry is not None:
gen = stream_entry(**gen_kw)
else:
fallback_stream_entry = getattr(self.tts_model, "generate_with_prompt_cache_streaming", None)
if fallback_stream_entry is not None:
gen = fallback_stream_entry(**gen_kw, latents_only=True)
else:
gen = self.tts_model._generate_with_prompt_cache(streaming=True, latents_only=True, **gen_kw)
iterator = iter(gen)
previous = next(iterator, None)
while previous is not None:
current = next(iterator, None)
_, _target_tok, chunk_latent = previous
if not isinstance(chunk_latent, torch.Tensor):
chunk_latent = torch.as_tensor(chunk_latent)
yield chunk_latent, current is None
previous = current
class _DirectVoxCPMAudioVAE:
def __init__(self, audio_vae: nn.Module, *, patch_size: int = 2):
self.audio_vae = audio_vae
self.sample_rate = int(getattr(audio_vae, "sample_rate", 24000))
self.latent_dim = int(getattr(audio_vae, "latent_dim", 64))
self.patch_size = int(patch_size)
self._chunk_size = int(getattr(audio_vae, "chunk_size", 1))
self._stream_audio_patch_samples = max(1, self.patch_size * self._chunk_size)
def _prepare_latents_for_decode(self, latent_audio_feat: Any) -> torch.Tensor:
latents = latent_audio_feat
if not isinstance(latents, torch.Tensor):
latents = torch.tensor(latents, dtype=torch.float32)
latents = latents.detach().to(torch.float32)
if latents.ndim == 3:
if latents.shape[-1] == self.latent_dim:
latents = rearrange(latents, "t p d -> 1 d (t p)")
elif latents.shape[1] == self.latent_dim:
latents = latents.contiguous()
else:
raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}")
elif latents.ndim == 2:
if latents.shape[0] == self.latent_dim:
latents = latents.unsqueeze(0)
elif latents.shape[1] == self.latent_dim:
latents = rearrange(latents, "t d -> 1 d t")
else:
raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}")
else:
raise ValueError(f"Unsupported latent_audio_feat ndim: {latents.ndim}")
return latents
@torch.no_grad()
def decode(self, latent_audio_feat: Any, *, trim_streaming_patch: bool = False) -> torch.Tensor:
latents = self._prepare_latents_for_decode(latent_audio_feat)
device = next(self.audio_vae.parameters()).device
raw = self.audio_vae.decode(latents.to(device=device, dtype=torch.float32))
if isinstance(raw, dict):
audio = raw.get("audio")
if audio is None:
audio = next(v for v in raw.values() if isinstance(v, torch.Tensor))
else:
audio = raw
if audio.dim() == 3:
stream = audio.squeeze(1)
elif audio.dim() == 2:
stream = audio
else:
stream = audio.reshape(audio.shape[0], -1)
if trim_streaming_patch:
stream = stream[..., -self._stream_audio_patch_samples :]
return stream.reshape(-1).detach().cpu().to(torch.float32)
@@ -307,32 +307,39 @@ class MiniCPM4PagedForVoxCPM2(nn.Module):
hidden_states = self.norm(hidden_states)
return hidden_states
def precompute_fused_qkv(self) -> None:
"""Materialize fused QKV weights before CUDA Graph capture."""
for layer in self.layers:
attn = layer.self_attn
if attn._fused_qkv_weight is None:
attn._fused_qkv_weight = torch.cat(
[attn.q_proj.weight, attn.k_proj.weight, attn.v_proj.weight],
dim=0,
).detach()
def compile_selective(self) -> list[str]:
"""Compile MLP + o_proj; keep RMSNorm/RoPE eager for precision."""
compiled: list[str] = []
for i, layer in enumerate(self.layers):
if i in self._compiled_layers:
continue
try:
layer.mlp = torch.compile(
layer.mlp,
mode="default",
fullgraph=True,
)
layer.self_attn.o_proj = torch.compile(
layer.self_attn.o_proj,
mode="default",
fullgraph=True,
)
layer.self_attn._fused_qkv_weight = None
self._compiled_layers.add(i)
if i == 0:
compiled.append(f"layers.*.mlp (×{len(self.layers)})")
compiled.append(f"layers.*.self_attn.o_proj (×{len(self.layers)})")
except Exception as e:
logger.warning("compile_selective: layer %d failed: %s", i, e)
break
return compiled
"""Compile the full model forward as one graph.
Earlier versions compiled ``layer.mlp`` + ``layer.self_attn.o_proj``
(PR #2690) and then the whole ``layer`` (perf/voxcpm2-streaming-vae).
Both still paid one Dynamo dispatch per layer per decode step.
V3 profiling showed 1,332 per-layer dispatches (~28 layers × ~47
decode steps) costing ~726 ms of CPU self-time for a long prompt.
Compiling ``forward`` at the model level lets Dynamo unroll the
28-layer Python loop inside the graph. Graph breaks at
PagedAttention produce sub-graphs but Dynamo memoises the whole
trace once, so the per-step dispatch drops from 28 to just a few.
"""
if self._compiled_layers:
return []
# Null the fused-qkv caches so the compile sees the real weight layout.
for layer in self.layers:
layer.self_attn._fused_qkv_weight = None
self.forward = torch.compile(self.forward, mode="default", fullgraph=False)
# Mark every layer as compiled so idempotent callers don't double-wrap.
self._compiled_layers.update(range(len(self.layers)))
return ["forward (whole model)"]
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights from native checkpoint (base_lm. prefix pre-stripped)."""
@@ -414,23 +421,25 @@ class MiniCPM4PagedResidualLM(nn.Module):
hidden_states = self.norm(hidden_states)
return hidden_states
def precompute_fused_qkv(self) -> None:
"""Materialize fused QKV weights before CUDA Graph capture."""
for layer in self.layers:
attn = layer.self_attn
if attn._fused_qkv_weight is None:
attn._fused_qkv_weight = torch.cat(
[attn.q_proj.weight, attn.k_proj.weight, attn.v_proj.weight],
dim=0,
).detach()
def compile_selective(self) -> list[str]:
"""Compile MLP + o_proj (same as base_lm)."""
compiled: list[str] = []
for i, layer in enumerate(self.layers):
if i in self._compiled_layers:
continue
try:
layer.mlp = torch.compile(layer.mlp, mode="default", fullgraph=True)
layer.self_attn.o_proj = torch.compile(layer.self_attn.o_proj, mode="default", fullgraph=True)
layer.self_attn._fused_qkv_weight = None
self._compiled_layers.add(i)
if i == 0:
compiled.append(f"layers.*.mlp (×{len(self.layers)})")
compiled.append(f"layers.*.self_attn.o_proj (×{len(self.layers)})")
except Exception as e:
logger.warning("compile_selective: residual layer %d failed: %s", i, e)
return compiled
"""Compile the full residual model forward as one graph (same strategy as base_lm)."""
if self._compiled_layers:
return []
for layer in self.layers:
layer.self_attn._fused_qkv_weight = None
self.forward = torch.compile(self.forward, mode="default", fullgraph=False)
self._compiled_layers.update(range(len(self.layers)))
return ["forward (whole residual)"]
def load_weights_from_native(self, native_residual_lm: nn.Module) -> int:
"""Load weights from native residual_lm. Returns param count."""
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""VoxCPM2 pipeline topology (frozen).
Single-stage AR TTS: text → speech waveform in one pass.
Uses the native MiniCPM4 base_lm with a per-request StaticKVCache that the
talker restores into the paged attention layer at step boundaries.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
VOXCPM2_PIPELINE = PipelineConfig(
model_type="voxcpm2",
model_arch="VoxCPM2TalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="latent_generator",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="audio",
owns_tokenizer=True,
engine_output_type="audio",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [1],
},
),
),
)
@@ -10,23 +10,26 @@ Architecture:
from __future__ import annotations
import copy
import dataclasses
import logging
import math
import os
import time
from collections.abc import Iterable
from typing import Any
import librosa
import torch
import torch.nn as nn
from einops import rearrange
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context, override_forward_context
from vllm.logger import init_logger
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
)
from vllm.multimodal.audio import AudioResampler
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.output_templates import OmniOutput
@@ -38,6 +41,88 @@ logger = init_logger(__name__)
_ENABLE_PROFILING = os.environ.get("VOXCPM2_PROFILE", "0") == "1"
# Lower bound for the _active_states leak-warn threshold. The effective
# threshold is max(_ACTIVE_STATE_LEAK_WARN_MIN, 4 * max_batch_size) so small
# deployments still get a usable floor instead of a tiny noisy one.
_ACTIVE_STATE_LEAK_WARN_MIN = 512
def is_cjk_char(c: str) -> bool:
"""Check if a character is a CJK ideograph."""
cp = ord(c)
return (
0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= cp <= 0x4DBF # Extension A
or 0xF900 <= cp <= 0xFAFF # Compatibility Ideographs
or 0x20000 <= cp <= 0x2A6DF # Extension B
or 0x2A700 <= cp <= 0x2B73F # Extension C
or 0x2B740 <= cp <= 0x2B81F # Extension D
or 0x2F800 <= cp <= 0x2FA1F # Compatibility Supplement
)
def build_cjk_split_map(tokenizer: Any) -> dict[int, list[int]]:
"""Build {multichar_cjk_token_id: [single_char_ids]} from tokenizer vocab."""
vocab = tokenizer.get_vocab()
split_map: dict[int, list[int]] = {}
for token, token_id in vocab.items():
clean = token.replace("\u2581", "")
if len(clean) >= 2 and all(is_cjk_char(c) for c in clean):
char_ids = tokenizer.convert_tokens_to_ids(list(clean))
if all(cid != tokenizer.unk_token_id for cid in char_ids):
split_map[token_id] = char_ids
return split_map
def split_multichar_chinese(token_ids: list[int], split_map: dict[int, list[int]]) -> list[int]:
"""Replace multichar Chinese token IDs with single-char IDs (idempotent)."""
result: list[int] = []
for tid in token_ids:
expansion = split_map.get(tid)
if expansion is not None:
result.extend(expansion)
else:
result.append(tid)
return result
def build_voxcpm2_prompt(
hf_config: Any,
tokenizer: Any,
split_map: dict[int, list[int]],
text: str,
ref_audio: Any | None = None,
ref_sr: int | None = None,
ref_text: str | None = None,
) -> dict[str, Any]:
"""Build a VoxCPM2 prefill prompt whose ``prompt_token_ids`` length matches
the talker-side prefill length.
Used by both online serving (``serving_speech._build_voxcpm2_prompt``) and
the offline example, so the talker-side length assertion never fires.
"""
ids = split_multichar_chinese(tokenizer.encode(text, add_special_tokens=True), split_map)
bos = tokenizer.bos_token_id
if ids and ids[0] == bos:
ids = ids[1:]
prefill_len = len(ids) + 1 # + audio_start
additional: dict[str, Any] = {"text_token_ids": [ids]}
if ref_audio is not None:
vae = hf_config.audio_vae_config
patch_samples = hf_config.patch_size * math.prod(vae["encoder_rates"])
ref_len = math.ceil(math.ceil(len(ref_audio) * vae["sample_rate"] / ref_sr) / patch_samples)
if ref_text is not None:
additional["prompt_audio"] = [[ref_audio, ref_sr]]
additional["prompt_text"] = [ref_text]
ref_ids = split_multichar_chinese(tokenizer.encode(ref_text, add_special_tokens=True), split_map)
if ref_ids and ref_ids[0] == bos:
ref_ids = ref_ids[1:]
prefill_len += ref_len + len(ref_ids)
else:
additional["reference_audio"] = [[ref_audio, ref_sr]]
prefill_len += ref_len + 2 # ref_start / ref_end
return {"prompt_token_ids": [1] * prefill_len, "additional_information": additional}
def _encode_raw_audio(
tts: nn.Module,
@@ -60,7 +145,8 @@ def _encode_raw_audio(
encode_sr = tts._encode_sample_rate
if sr != encode_sr:
audio_np = audio.squeeze(0).numpy()
audio_np = librosa.resample(audio_np, orig_sr=sr, target_sr=encode_sr)
resampler = AudioResampler(target_sr=encode_sr)
audio_np = resampler.resample(audio_np, orig_sr=sr)
audio = torch.from_numpy(audio_np).unsqueeze(0)
patch_len = tts.patch_size * tts.chunk_size
@@ -86,7 +172,11 @@ class _RequestState:
curr_prefix_feat_cond: torch.Tensor | None = None
last_audio_patch_gpu: torch.Tensor | None = None
precomputed_stop_logits: torch.Tensor | None = None
accumulated_patches: list[torch.Tensor] = dataclasses.field(default_factory=list)
# Rolling tail of previously-decoded latents used as VAE receptive-field context.
# Shape (n_pad_frames, feat_dim) on GPU. None before first decode.
decode_pad: torch.Tensor | None = None
# Audio chunks already emitted (CPU float32), concatenated for cumulative output.
audio_chunks: list[torch.Tensor] = dataclasses.field(default_factory=list)
decode_step_count: int = 0
request_start_time: float = 0.0
prefill_completed: bool = False
@@ -97,6 +187,14 @@ class _RequestState:
last_decoded_audio: torch.Tensor | None = None
@dataclasses.dataclass
class _CapturedGraph:
graph: torch.cuda.CUDAGraph
input_embeds: torch.Tensor
positions: torch.Tensor
output: torch.Tensor
# ===================================================================
# Profiling timer
# ===================================================================
@@ -229,11 +327,11 @@ def _optimized_solve_euler(
buffers.x_in[b : 2 * b].copy_(x)
buffers.mu_in[:b].copy_(mu)
buffers.mu_in[b : 2 * b].zero_()
buffers.t_in[:b].fill_(t.item())
buffers.t_in[b : 2 * b].fill_(t.item())
# Broadcast the 0-dim GPU scalar directly instead of
# ``.fill_(t.item())`` — ``.item()`` forces a GPU->CPU sync.
buffers.t_in[: 2 * b].copy_(t)
if mean_mode:
buffers.dt_in[:b].fill_(dt.item())
buffers.dt_in[b : 2 * b].fill_(dt.item())
buffers.dt_in[: 2 * b].copy_(dt)
else:
buffers.dt_in.zero_()
buffers.cond_in[:b].copy_(cond[:b])
@@ -263,9 +361,10 @@ def _optimized_solve_euler(
else:
buffers.x_in[:b].copy_(x)
buffers.mu_in[:b].copy_(mu)
buffers.t_in[:b].fill_(t.item())
# Broadcast the 0-dim GPU scalar; ``.fill_(t.item())`` would sync.
buffers.t_in[:b].copy_(t)
if mean_mode:
buffers.dt_in[:b].fill_(dt.item())
buffers.dt_in[:b].copy_(dt)
else:
buffers.dt_in[:b].zero_()
buffers.cond_in[:b].copy_(cond[:b])
@@ -320,7 +419,10 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
self._inference_timesteps = 10
self._cfg_value = 2.0
self._cfg_cutoff_ratio = 1.0
self._vae_decode_interval = 5
# Number of trailing latent frames to keep as VAE receptive-field context
# for sliding-window streaming decode. 12 matches the nanovllm reference
# implementation and covers the longest VAE decoder receptive field.
self._n_decode_pad_frames = 12
self._enable_torch_compile = True
self._compile_vae = True
self._max_decode_steps = 2000
@@ -328,6 +430,15 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
self._perf = _PerfTimer(enabled=_ENABLE_PROFILING)
self._cfm_buffers: _CFMBufferManager | None = None
self._enable_cuda_graph = True
self._scaffold_graphs: dict[int, _CapturedGraph] = {}
self._residual_graphs: dict[int, _CapturedGraph] = {}
self._max_cached_graphs = self._max_batch_size
self._cuda_graph_pool: tuple | None = None
self._cuda_graph_warmup_steps = 0
self._cuda_graph_warmup_threshold = 3
self._multichar_zh_split: dict[int, list[int]] | None = None
self._active_states: dict[str, _RequestState] = {}
self._current_request_id: str | None = None
@@ -335,6 +446,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
self._results_queue: list[tuple[str, torch.Tensor | None]] = []
self._audio_queue: list[tuple[str, Any]] = []
self._deferred_cleanup_ids: set[str] = set()
self._active_state_warn_threshold = max(_ACTIVE_STATE_LEAK_WARN_MIN, 4 * self._max_batch_size)
# one-shot by design: fires at most once per process to avoid log spam.
self._active_state_warned = False
@property
def tts(self) -> nn.Module:
@@ -344,9 +458,20 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
# -------------------- request state management --------------------
def _get_or_create_state(self, request_id: str) -> _RequestState:
if request_id not in self._active_states:
self._active_states[request_id] = _RequestState(request_id=request_id)
return self._active_states[request_id]
state = self._active_states.get(request_id)
if state is None:
state = _RequestState(request_id=request_id)
self._active_states[request_id] = state
if len(self._active_states) > self._active_state_warn_threshold and not self._active_state_warned:
logger.warning(
"VoxCPM2: _active_states size=%d exceeds threshold %d "
"(max_batch_size=%d); possible cleanup path leak",
len(self._active_states),
self._active_state_warn_threshold,
self._max_batch_size,
)
self._active_state_warned = True
return state
def _switch_to_request(self, request_id: str) -> _RequestState:
if request_id != self._current_request_id:
@@ -475,19 +600,24 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
except Exception as e:
logger.warning("torch.compile AudioVAE failed: %s", e)
if not getattr(self.model, "_selective_compiled", False):
try:
targets.extend(f"scaffold.{t}" for t in self.model.compile_selective())
self.model._selective_compiled = True
except Exception as e:
logger.warning("scaffold compile failed: %s", e)
if not self._enable_cuda_graph:
if not getattr(self.model, "_selective_compiled", False):
try:
targets.extend(f"scaffold.{t}" for t in self.model.compile_selective())
self.model._selective_compiled = True
except Exception as e:
logger.warning("scaffold compile failed: %s", e)
if not getattr(self.residual_model, "_selective_compiled", False):
try:
targets.extend(f"residual.{t}" for t in self.residual_model.compile_selective())
self.residual_model._selective_compiled = True
except Exception as e:
logger.warning("residual compile failed: %s", e)
if not getattr(self.residual_model, "_selective_compiled", False):
try:
targets.extend(f"residual.{t}" for t in self.residual_model.compile_selective())
self.residual_model._selective_compiled = True
except Exception as e:
logger.warning("residual compile failed: %s", e)
else:
self.model.precompute_fused_qkv()
self.residual_model.precompute_fused_qkv()
targets.append("scaffold+residual (CUDA Graph, skipping compile)")
if not getattr(self, "_projections_compiled", False):
try:
@@ -510,6 +640,90 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
tts = self.tts
return tts.stop_head(tts.stop_actn(tts.stop_proj(lm_h)))
def _get_cuda_graph_pool(self) -> tuple:
if self._cuda_graph_pool is None:
self._cuda_graph_pool = torch.cuda.graph_pool_handle()
return self._cuda_graph_pool
@staticmethod
def _nullify_volatile_metadata(ctx: Any) -> Any:
"""Set ``scheduler_metadata`` to None on all attention layers.
This is the only tensor FA3 reallocates each step (variable shape).
All other metadata tensors are persistent model-runner buffers.
Setting it to None makes FA3 use default scheduling (~0.1ms cost).
"""
if not isinstance(ctx.attn_metadata, dict):
return ctx
ctx = copy.copy(ctx)
new_meta: dict[str, Any] = {}
for layer_name, meta in ctx.attn_metadata.items():
if getattr(meta, "scheduler_metadata", None) is not None:
meta = copy.copy(meta)
meta.scheduler_metadata = None
new_meta[layer_name] = meta
ctx.attn_metadata = new_meta
return ctx
def _capture_graph(
self,
model: nn.Module,
batch_size: int,
label: str,
is_residual: bool = False,
) -> _CapturedGraph:
"""Capture a CUDA Graph for *model* at *batch_size*."""
hidden_size = self.config.hidden_size
dtype = self._side_dtype
dev = torch.device(self._device)
pool = self._get_cuda_graph_pool()
model.precompute_fused_qkv()
g = _CapturedGraph(
graph=torch.cuda.CUDAGraph(),
input_embeds=torch.zeros(batch_size, hidden_size, device=dev, dtype=dtype),
positions=torch.zeros(batch_size, device=dev, dtype=torch.long),
output=torch.zeros(batch_size, hidden_size, device=dev, dtype=dtype),
)
if is_residual:
call_kwargs = dict(positions=g.positions, inputs_embeds=g.input_embeds)
else:
call_kwargs = dict(input_ids=None, positions=g.positions, inputs_embeds=g.input_embeds)
ctx = get_forward_context()
patched_ctx = self._nullify_volatile_metadata(ctx)
with override_forward_context(patched_ctx):
for _ in range(3):
_ = model(**call_kwargs)
with torch.cuda.graph(g.graph, pool=pool):
g.output = model(**call_kwargs)
logger.info("CUDA Graph captured for %s (batch_size=%d)", label, batch_size)
return g
def _replay_graph(
self,
g: _CapturedGraph,
inputs_embeds: torch.Tensor,
positions: torch.Tensor,
batch_size: int,
) -> torch.Tensor:
"""Copy fresh inputs into static buffers, then replay.
No metadata copy needed: persistent buffers (seq_lens, slot_mapping,
etc.) are updated in-place by the model runner. scheduler_metadata
was nullified at capture time so no kernel references it.
"""
g.input_embeds[:batch_size].copy_(inputs_embeds[:batch_size])
g.positions[:batch_size].copy_(positions[:batch_size])
g.graph.replay()
return g.output[:batch_size].clone()
# -------------------- vllm hooks --------------------
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
@@ -526,12 +740,35 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
self._perf.start("forward_total")
dev = input_ids.device
model_output = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
if isinstance(model_output, IntermediateTensors):
return model_output
scaffold_hidden = model_output
if isinstance(scaffold_hidden, tuple):
scaffold_hidden = scaffold_hidden[0]
num_reqs = len(self._pending_requests)
num_decode = sum(1 for _, is_p, _, n in self._pending_requests if not is_p and n == 1)
is_all_decode = num_decode == num_reqs and num_reqs > 0
tts_compiled = getattr(self.tts.feat_decoder.estimator, "_compiled", False) if self._tts is not None else False
graph_ready = tts_compiled and self._cuda_graph_warmup_steps >= self._cuda_graph_warmup_threshold
if num_decode > 0:
self._cuda_graph_warmup_steps += 1
can_use_graph = (
self._enable_cuda_graph and graph_ready and intermediate_tensors is None and inputs_embeds is not None
)
if can_use_graph and is_all_decode and num_reqs <= self._max_cached_graphs:
self._perf.start("scaffold_fwd")
if num_reqs not in self._scaffold_graphs:
self._scaffold_graphs[num_reqs] = self._capture_graph(self.model, num_reqs, "scaffold")
scaffold_hidden = self._replay_graph(self._scaffold_graphs[num_reqs], inputs_embeds, positions, num_reqs)
self._perf.stop("scaffold_fwd")
else:
self._perf.start("scaffold_fwd")
model_output = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
self._perf.stop("scaffold_fwd")
if isinstance(model_output, IntermediateTensors):
return model_output
scaffold_hidden = model_output
if isinstance(scaffold_hidden, tuple):
scaffold_hidden = scaffold_hidden[0]
# Phase 1: per-request FSQ + residual input
token_offset = 0
@@ -563,7 +800,28 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
if residual_inputs:
batch_in = torch.cat(residual_inputs, dim=0)
batch_pos = torch.cat(residual_positions, dim=0)
batch_out = self.residual_model(batch_pos, batch_in)
residual_batch_size = batch_in.shape[0]
use_residual_graph = (
self._enable_cuda_graph
and is_all_decode
and graph_ready
and residual_batch_size == num_reqs # 1 token per request
and residual_batch_size <= self._max_cached_graphs
)
self._perf.start("residual_fwd")
if use_residual_graph:
if residual_batch_size not in self._residual_graphs:
self._residual_graphs[residual_batch_size] = self._capture_graph(
self.residual_model, residual_batch_size, "residual", is_residual=True
)
batch_out = self._replay_graph(
self._residual_graphs[residual_batch_size], batch_in, batch_pos, residual_batch_size
)
else:
batch_out = self.residual_model(batch_pos, batch_in)
self._perf.stop("residual_fwd")
# Phase 3: per-request LocDiT + update
offset = 0
@@ -594,19 +852,12 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
tts_len = text_mask.shape[1]
scaffold_len = base_lm_out.shape[0]
if scaffold_len < tts_len:
# Voice clone / continuation: scaffold only processed vllm tokens.
# Pad to match TTS sequence length (extra positions are masked out).
pad = torch.zeros(
tts_len - scaffold_len,
base_lm_out.shape[-1],
device=base_lm_out.device,
dtype=base_lm_out.dtype,
)
enc_out = torch.cat([base_lm_out, pad], dim=0).unsqueeze(0)
else:
enc_out = base_lm_out.unsqueeze(0)
assert scaffold_len == tts_len, (
f"voxcpm2 prefill length mismatch: scaffold_len={scaffold_len} tts_len={tts_len}; "
"caller must pad prompt_token_ids to the full prefill length "
"(see serving_speech._build_voxcpm2_prompt or the offline example)."
)
enc_out = base_lm_out.unsqueeze(0)
prefix_feat_cond = (
feat[:, -1, ...]
@@ -686,7 +937,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
state.request_start_time = time.perf_counter()
state.prefill_completed = True
logger.info("PREFILL[%s]: patch norm=%.4f", state.request_id, pred_feat.norm().item())
if logger.isEnabledFor(logging.DEBUG):
# Only compute the norm (which forces a GPU->CPU sync) if we will log it.
logger.debug("PREFILL[%s]: patch norm=%.4f", state.request_id, pred_feat.norm().item())
self._perf.reset()
def _finish_decode(self, state: _RequestState, meta: dict, res_out: torch.Tensor, dev: Any):
@@ -720,26 +973,54 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
# -------------------- audio collection --------------------
def _collect_audio(self, state: _RequestState) -> torch.Tensor | None:
"""Per-step sliding-window VAE decode (nanovllm pattern).
Each decode step feeds ``[decode_pad, new_patch]`` through the VAE
and slices out only the audio region corresponding to the new patch.
The pad buffer (last ``_n_decode_pad_frames`` latent frames) provides
the receptive-field context needed by the VAE's transposed convolutions,
eliminating boundary artifacts between chunks.
Returns the delta audio chunk (not cumulative) so the output processor
can stream each chunk to the client independently.
"""
patch = state.last_audio_patch_gpu
if patch is not None:
state.last_audio_patch_gpu = None
state.accumulated_patches.append(patch.reshape(1, -1).float())
if not state.accumulated_patches:
if patch is None:
return None
state.last_audio_patch_gpu = None
n = len(state.accumulated_patches)
if n <= 1 or n % self._vae_decode_interval == 0 or state.is_stopping:
self._perf.start("vae_decode")
all_p = torch.cat(state.accumulated_patches, dim=0)
state.accumulated_patches = [all_p]
feat = rearrange(all_p.reshape(1, -1, self._feat_dim), "b t d -> b d t")
with torch.no_grad():
audio = self.tts.audio_vae.decode(feat.to(self._device)).reshape(-1).cpu().float()
self._perf.stop("vae_decode")
state.last_decoded_audio = audio
return audio
return state.last_decoded_audio
# patch shape: (patch_size, feat_dim) or (1, patch_size, feat_dim)
new_latent = patch.reshape(-1, self._feat_dim).to(torch.float32)
n_new = new_latent.shape[0] # = patch_size (typically 4)
self._perf.start("vae_decode")
# Build VAE input: [pad_frames | new_latent]
if state.decode_pad is not None:
vae_input = torch.cat([state.decode_pad, new_latent], dim=0)
pad_frames = state.decode_pad.shape[0]
else:
vae_input = new_latent
pad_frames = 0
# VAE decode: (1, feat_dim, T_frames) -> (1, 1, T_samples)
feat = vae_input.unsqueeze(0).transpose(1, 2).contiguous()
with torch.no_grad():
audio = self.tts.audio_vae.decode(feat.to(self._device)).reshape(-1)
# Slice out only the new audio (after the pad region).
# Each latent frame maps to decoder_chunk_size audio samples.
dcs = int(getattr(self.tts.audio_vae, "decode_chunk_size", audio.numel() // vae_input.shape[0]))
new_audio = audio[pad_frames * dcs : (pad_frames + n_new) * dcs].detach().cpu().float()
# Roll the pad buffer: keep last N latent frames as context for next step.
all_latents = vae_input # [pad + new]
state.decode_pad = all_latents[-self._n_decode_pad_frames :].detach()
state.audio_chunks.append(new_audio)
state.last_decoded_audio = new_audio
self._perf.stop("vae_decode")
return new_audio
# -------------------- compute_logits --------------------
@@ -797,6 +1078,17 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=mm)
# -------------------- Chinese token splitting --------------------
def _get_multichar_zh_split(self) -> dict[int, list[int]]:
"""Lazy-build {multichar_chinese_token_id: [char_id, ...]} map."""
if self._multichar_zh_split is not None:
return self._multichar_zh_split
base_tokenizer = self.tts.text_tokenizer.tokenizer
self._multichar_zh_split = build_cjk_split_map(base_tokenizer)
logger.info("VoxCPM2: built multichar Chinese split map (%d entries)", len(self._multichar_zh_split))
return self._multichar_zh_split
# -------------------- preprocess / postprocess --------------------
def preprocess(
@@ -815,22 +1107,29 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
is_prefill = span_len > 1
if is_prefill:
# Evict stale states
pending_ids = {rid for rid, *_ in self._pending_requests}
pending_ids.add(req_id)
if self._current_request_id:
pending_ids.add(self._current_request_id)
for rid in [r for r, s in self._active_states.items() if r not in pending_ids and s.prefill_completed]:
self._cleanup_request(rid)
# VoxCPM2Tokenizer does char-level Chinese splitting, so use input_ids directly
token_ids = input_ids.tolist()
# Do not evict state here: _pending_requests is a per-step prefix,
# not the full batch. Cleanup is driven by on_requests_finished ->
# _flush_deferred_cleanup (fed by vLLM scheduler._free_request via
# gpu_ar_model_runner.py).
real = info_dict.get("text_token_ids")
token_ids = input_ids.tolist() if real is None else real[0]
# Fail-fast: unsplit multichar Chinese IDs in input_ids means the
# serving layer didn't pre-split. Silent fixup here would cause
# input_ids/embeds length mismatch (scheduler slot count is fixed).
split_map = self._get_multichar_zh_split()
if split_map and any(tid in split_map for tid in token_ids):
raise ValueError(
"VoxCPM2 preprocess received unsplit multichar Chinese "
"token IDs. The serving layer must send prompt_token_ids "
"with single-char CJK IDs (see _voxcpm2_encode)."
)
if token_ids and token_ids[0] == self.config.bos_token_id:
token_ids = token_ids[1:]
state = self._get_or_create_state(req_id)
state.prefill_text = ""
state.accumulated_patches = []
state.decode_pad = None
state.audio_chunks = []
state.prefill_completed = False
state.decode_step_count = 0
state.precomputed_stop_logits = None
Whitespace-only changes.
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Voxtral TTS pipeline topology (frozen).
Stage 0: audio_generation — text → acoustic latents (LLM_AR, tokenizer owner).
Stage 1: audio_tokenizer — acoustic latents → waveform (LLM_GENERATION).
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts"
VOXTRAL_TTS_PIPELINE = PipelineConfig(
model_type="voxtral_tts",
model_arch="VoxtralTTSForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audio_generation",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=False,
final_output_type="text",
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.generator2tokenizer_async_chunk"),
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="audio_tokenizer",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sampling_constraints={"detokenize": True},
extras={"tts_args": {"max_instructions_length": 500}},
),
),
)
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) 2022 OpenAI
#
# Shared Whisper encoder primitives used by multiple model implementations.
# Originally from the OpenAI Whisper codebase.
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
def sinusoids(length, channels, max_timescale=10000):
"""Returns sinusoids for positional embedding."""
assert channels % 2 == 0
log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
class Conv1d(nn.Conv1d):
"""Conv1d with automatic dtype casting for mixed precision inference."""
def _conv_forward(self, x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
class ConvTranspose1d(nn.ConvTranspose1d):
def _conv_forward(self, x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
class Linear(nn.Linear):
"""Linear layer with automatic dtype casting for mixed precision inference."""
def forward(self, x: torch.Tensor) -> torch.Tensor:
return F.linear(x, self.weight.to(x.dtype), None if self.bias is None else self.bias.to(x.dtype))