Merge branch 'main' into feat/moss-tts-nano
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
611 files changed
+48775
-14783
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,654 @@
|
||||
"""Qwen3 Code Predictor -- optimized re-prefill, no KV cache.
|
||||
|
||||
Shared by Qwen3-Omni and Qwen3-TTS talker models.
|
||||
|
||||
* SDPA attention (F.scaled_dot_product_attention) with native GQA support
|
||||
* HF-compatible numerics (float32 RMSNorm, float32 RoPE, separate linear layers)
|
||||
* Per-call embedding buffer to avoid cross-request aliasing
|
||||
* Pre-allocated position_ids (read-only, safe to persist)
|
||||
* torch.compile (epilogue_fusion=False) on inner transformer by default
|
||||
* Optional manual CUDA graph capture per batch-size bucket
|
||||
* Inline sampling (top-k + top-p) -- no custom op overhead
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# HF-numerics-compatible layers for code predictor
|
||||
# ===================================================================
|
||||
#
|
||||
# These use plain PyTorch ops (nn.Linear, manual RMSNorm in float32,
|
||||
# rotate_half RoPE) to produce outputs numerically identical to the
|
||||
# HuggingFace reference. vLLM's fused kernels (RMSNorm, QKVParallel,
|
||||
# get_rope) introduce small precision differences that compound across
|
||||
# the autoregressive steps of the code predictor, causing severe
|
||||
# audio quality degradation.
|
||||
#
|
||||
# See: https://github.com/vllm-project/vllm-omni/issues/2274
|
||||
|
||||
|
||||
class _RMSNorm(nn.Module):
|
||||
"""RMSNorm matching HuggingFace's implementation exactly.
|
||||
|
||||
Computes variance in float32 to avoid bfloat16 precision loss.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
return self.weight * hidden_states.to(input_dtype)
|
||||
|
||||
|
||||
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Rotates half the hidden dims of the input."""
|
||||
x1 = x[..., : x.shape[-1] // 2]
|
||||
x2 = x[..., x.shape[-1] // 2 :]
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
class _RotaryEmbedding(nn.Module):
|
||||
"""RoPE matching HuggingFace's implementation exactly.
|
||||
|
||||
Forces float32 computation for cos/sin, matching HF's torch.autocast(enabled=False).
|
||||
"""
|
||||
|
||||
def __init__(self, config) -> None:
|
||||
super().__init__()
|
||||
head_dim = getattr(
|
||||
config,
|
||||
"head_dim",
|
||||
config.hidden_size // config.num_attention_heads,
|
||||
)
|
||||
rope_theta = getattr(config, "rope_theta", 10000.0)
|
||||
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# position_ids: [batch, seq_len]
|
||||
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
||||
position_ids_expanded = position_ids[:, None, :].float()
|
||||
|
||||
# Force float32 (matching HF)
|
||||
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
|
||||
with torch.autocast(device_type=device_type, enabled=False):
|
||||
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
||||
emb = torch.cat((freqs, freqs), dim=-1)
|
||||
cos = emb.cos()
|
||||
sin = emb.sin()
|
||||
|
||||
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Attention
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class CodePredictorAttention(nn.Module):
|
||||
"""Multi-head self-attention for code predictor.
|
||||
|
||||
Uses ``F.scaled_dot_product_attention`` with HF-compatible RoPE and RMSNorm.
|
||||
No KV cache -- the code predictor always re-prefills the full (short)
|
||||
sequence each AR step.
|
||||
|
||||
Input : [B, seq_len, hidden_size]
|
||||
Output: [B, seq_len, hidden_size]
|
||||
"""
|
||||
|
||||
def __init__(self, config, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_kv_heads = config.num_key_value_heads
|
||||
self.head_dim = getattr(
|
||||
config,
|
||||
"head_dim",
|
||||
config.hidden_size // config.num_attention_heads,
|
||||
)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self._use_gqa = self.num_kv_heads != self.num_heads
|
||||
|
||||
# Separate q/k/v projections matching HF (no fused packing)
|
||||
bias = getattr(config, "attention_bias", False)
|
||||
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=bias)
|
||||
self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=bias)
|
||||
self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=bias)
|
||||
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
|
||||
self.q_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim)
|
||||
hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim)
|
||||
|
||||
q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)
|
||||
k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2)
|
||||
v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2)
|
||||
|
||||
cos, sin = position_embeddings
|
||||
# cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads
|
||||
cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]
|
||||
sin = sin.unsqueeze(1)
|
||||
q = (q * cos) + (_rotate_half(q) * sin)
|
||||
k = (k * cos) + (_rotate_half(k) * sin)
|
||||
|
||||
attn_out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
scale=self.scaling,
|
||||
is_causal=True,
|
||||
enable_gqa=self._use_gqa,
|
||||
)
|
||||
|
||||
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
|
||||
return self.o_proj(attn_out)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# MLP
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class CodePredictorMLP(nn.Module):
|
||||
"""SiLU-gated MLP for code predictor, matching HF's implementation."""
|
||||
|
||||
def __init__(self, config, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
||||
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
||||
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Decoder Layer
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class CodePredictorDecoderLayer(nn.Module):
|
||||
"""Transformer decoder layer (SDPA, no KV cache)."""
|
||||
|
||||
def __init__(self, config, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.self_attn = CodePredictorAttention(config, prefix=f"{prefix}.self_attn")
|
||||
self.mlp = CodePredictorMLP(config, prefix=f"{prefix}.mlp")
|
||||
self.input_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(hidden_states, position_embeddings)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Base Transformer Model (re-prefill, no KV cache)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class CodePredictorBaseModel(nn.Module):
|
||||
"""Inner transformer for code predictor.
|
||||
|
||||
Signature: ``forward(inputs_embeds, position_ids) -> hidden_states``
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
*,
|
||||
embedding_dim: int | None = None,
|
||||
use_parallel_embedding: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
emb_dim = int(embedding_dim) if embedding_dim is not None else int(config.hidden_size)
|
||||
if use_parallel_embedding:
|
||||
self.codec_embedding = nn.ModuleList(
|
||||
[VocabParallelEmbedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
else:
|
||||
self.codec_embedding = nn.ModuleList(
|
||||
[nn.Embedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
CodePredictorDecoderLayer(config, prefix=f"{prefix}.layers.{idx}")
|
||||
for idx in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.norm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.rotary_emb = _RotaryEmbedding(config)
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.codec_embedding
|
||||
|
||||
def forward(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = inputs_embeds
|
||||
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(hidden_states, position_embeddings)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
param = params_dict.get(name)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Wrapper Configuration
|
||||
# ===================================================================
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CodePredictorWrapperConfig:
|
||||
"""Controls behavioral differences between model-specific code predictors."""
|
||||
|
||||
use_cuda_graphs: bool = False
|
||||
use_parallel_embedding: bool = False
|
||||
use_projection: bool = False
|
||||
return_proj_buf: bool = False
|
||||
sampling_mode: str = "stored"
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Code Predictor Wrapper (optimized re-prefill, persistent buffers)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class CodePredictorWrapper(nn.Module):
|
||||
"""Optimized code predictor -- re-prefill approach, no KV cache.
|
||||
|
||||
Each AR step forwards the full growing sequence (len 2 -> num_code_groups+1)
|
||||
through the transformer. The extra O(T^2) FLOPs are negligible for
|
||||
short sequences, and this avoids all KV-cache management overhead.
|
||||
|
||||
Optimizations:
|
||||
1. Per-call embedding buffer -- avoids cross-request aliasing.
|
||||
2. Pre-allocated position_ids -- no torch.arange per step.
|
||||
3. Cached module references -- bypass ModuleList indexing.
|
||||
4. torch.compile on inner transformer.
|
||||
5. Inline sampling (top-k + top-p) -- no custom op overhead.
|
||||
6. Optional manual CUDA graph capture per batch-size bucket.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
cp_config,
|
||||
wrapper_config: CodePredictorWrapperConfig,
|
||||
talker_hidden_size: int | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._vllm_config = vllm_config
|
||||
self.config = cp_config
|
||||
self._wrapper_config = wrapper_config
|
||||
self.prefix = prefix
|
||||
|
||||
self._num_groups = int(cp_config.num_code_groups)
|
||||
self._cp_hidden = int(cp_config.hidden_size)
|
||||
|
||||
# For Omni backward compat (accessed by the talker)
|
||||
self.num_code_groups = self._num_groups
|
||||
|
||||
# Determine embedding dimension
|
||||
_talker_hidden = int(talker_hidden_size) if talker_hidden_size is not None else self._cp_hidden
|
||||
|
||||
self.model = CodePredictorBaseModel(
|
||||
cp_config,
|
||||
embedding_dim=_talker_hidden,
|
||||
use_parallel_embedding=wrapper_config.use_parallel_embedding,
|
||||
prefix=f"{prefix}.model" if prefix else "model",
|
||||
)
|
||||
|
||||
self.lm_head = nn.ModuleList(
|
||||
[nn.Linear(cp_config.hidden_size, cp_config.vocab_size, bias=False) for _ in range(self._num_groups - 1)]
|
||||
)
|
||||
|
||||
# Projection: Identity when hidden sizes match or not needed
|
||||
if wrapper_config.use_projection and _talker_hidden != self._cp_hidden:
|
||||
self.small_to_mtp_projection = nn.Linear(_talker_hidden, self._cp_hidden, bias=True)
|
||||
else:
|
||||
self.small_to_mtp_projection = nn.Identity()
|
||||
|
||||
# Sampling defaults for "stored" mode
|
||||
self._top_k: int = 50
|
||||
self._top_p: float = 0.8
|
||||
|
||||
# Lazily initialised state
|
||||
self._proj_buf: torch.Tensor | None = None
|
||||
self._model_dtype: torch.dtype | None = None
|
||||
self._compiled_model_fwd = None
|
||||
self._bucket_sizes: list[int] = []
|
||||
self._bucket_pos_ids: dict[int, torch.Tensor] = {}
|
||||
self._lm_heads_list: list[nn.Module] | None = None
|
||||
self._codec_embeds_list: list[nn.Module] | None = None
|
||||
self._cuda_graphs: dict[int, tuple[torch.cuda.CUDAGraph, torch.Tensor]] = {}
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def set_sampling_params(self, top_k: int = 50, top_p: float = 0.8) -> None:
|
||||
"""Configure sampling parameters to maintain consistency with previous implementation."""
|
||||
self._top_k = top_k
|
||||
self._top_p = top_p
|
||||
logger.debug("Sampling parameters updated: top_k=%d, top_p=%.2f", top_k, top_p)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lazy-init helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_buffers(self, device: torch.device, dtype: torch.dtype, bsz: int) -> None:
|
||||
"""Ensure the projection buffer can hold at least *bsz* rows."""
|
||||
max_seq = self._num_groups + 1
|
||||
if (
|
||||
self._proj_buf is not None
|
||||
and self._proj_buf.device == device
|
||||
and self._proj_buf.dtype == dtype
|
||||
and self._proj_buf.shape[0] >= bsz
|
||||
):
|
||||
return
|
||||
self._proj_buf = torch.zeros(bsz, max_seq, self._cp_hidden, dtype=dtype, device=device)
|
||||
|
||||
def _setup_compile(self) -> None:
|
||||
"""Lazily set up torch.compile with optional CUDA graph capture."""
|
||||
if self._compiled_model_fwd is not None:
|
||||
return
|
||||
|
||||
# Cache model parameter dtype so forward() doesn't need to query it
|
||||
# on every call. Also ensures warmup buffers match model precision
|
||||
# even when upstream modules produce a different dtype (#2385).
|
||||
self._model_dtype = next(self.model.parameters()).dtype
|
||||
self._lm_heads_list = list(self.lm_head)
|
||||
self._codec_embeds_list = list(self.model.codec_embedding)
|
||||
|
||||
if not current_omni_platform.supports_torch_inductor():
|
||||
logger.warning_once("code_predictor: torch.compile disabled")
|
||||
self._compiled_model_fwd = self.model.forward
|
||||
return
|
||||
|
||||
# torch.compile fuses RMSNorm/RoPE in ways that lose float32
|
||||
# precision, compounding across AR steps. Use epilogue_fusion=False
|
||||
# to disable the problematic fusions while still getting kernel
|
||||
# fusion benefits for the linear layers and SDPA.
|
||||
self._compiled_model_fwd = torch.compile(
|
||||
self.model.forward,
|
||||
dynamic=False,
|
||||
options={"epilogue_fusion": False},
|
||||
)
|
||||
self._warmup_buckets()
|
||||
|
||||
if self._wrapper_config.use_cuda_graphs:
|
||||
self._capture_cuda_graphs()
|
||||
logger.info("code_predictor: torch.compile (no epilogue fusion) + CUDA graphs")
|
||||
else:
|
||||
logger.info("code_predictor: torch.compile (dynamic=False, no epilogue fusion)")
|
||||
|
||||
def _padded_bsz(self, bsz: int) -> int:
|
||||
"""Round batch size up to nearest power-of-2 bucket."""
|
||||
for bucket in self._bucket_sizes:
|
||||
if bsz <= bucket:
|
||||
return bucket
|
||||
return bsz
|
||||
|
||||
def _warmup_buckets(self) -> None:
|
||||
"""Warmup power-of-2 batch-size buckets to front-load Inductor compilation."""
|
||||
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
|
||||
bucket_sizes = [1 << i for i in range(max_bsz.bit_length()) if (1 << i) <= max_bsz]
|
||||
if max_bsz not in bucket_sizes:
|
||||
bucket_sizes.append(max_bsz)
|
||||
self._bucket_sizes = sorted(bucket_sizes)
|
||||
|
||||
max_seq = self._num_groups + 1
|
||||
device = next(self.model.parameters()).device
|
||||
|
||||
# Ensure proj_buf matches model parameter dtype to avoid dtype
|
||||
# mismatch during warmup compilation (see #2385).
|
||||
self._ensure_buffers(device, self._model_dtype, max(self._bucket_sizes))
|
||||
proj_buf = self._proj_buf
|
||||
|
||||
for bsz in self._bucket_sizes:
|
||||
pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
|
||||
self._bucket_pos_ids[bsz] = pos_ids
|
||||
for _ in range(3):
|
||||
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
|
||||
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
|
||||
|
||||
def _capture_cuda_graphs(self) -> None:
|
||||
"""Capture a CUDA graph per bucket using vLLM's global graph pool."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pool = current_platform.get_global_graph_pool()
|
||||
max_seq = self._num_groups + 1
|
||||
proj_buf = self._proj_buf
|
||||
|
||||
for bsz in self._bucket_sizes:
|
||||
static_input = proj_buf[:bsz, :max_seq, :]
|
||||
pos_ids = self._bucket_pos_ids[bsz]
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, pool=pool):
|
||||
static_output = self._compiled_model_fwd(static_input, pos_ids)
|
||||
|
||||
self._cuda_graphs[bsz] = (g, static_output)
|
||||
|
||||
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Forward -- re-prefill + inline sampling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
layer0_code: torch.Tensor,
|
||||
layer0_embed: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
do_sample: bool = True,
|
||||
temperature: float = 0.9,
|
||||
top_k: int = 50,
|
||||
top_p: float = 1.0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Predict residual codebooks 1..G-1 autoregressively via re-prefill."""
|
||||
bsz = int(layer0_code.shape[0])
|
||||
num_groups = self._num_groups
|
||||
device = layer0_code.device
|
||||
|
||||
# _setup_compile caches _model_dtype on first call; use it for buffers
|
||||
# so they always match model weight precision (#2385).
|
||||
self._setup_compile()
|
||||
dtype = self._model_dtype
|
||||
|
||||
padded_bsz = self._padded_bsz(bsz)
|
||||
self._ensure_buffers(device, dtype, padded_bsz)
|
||||
|
||||
proj_buf = self._proj_buf
|
||||
max_seq = num_groups + 1
|
||||
projection = self.small_to_mtp_projection
|
||||
model_fwd = self._compiled_model_fwd
|
||||
lm_heads = self._lm_heads_list
|
||||
codec_embeds = self._codec_embeds_list
|
||||
|
||||
# Zero the padded region of the buffer
|
||||
proj_buf[:padded_bsz].zero_()
|
||||
|
||||
# Fill buffer positions 0 (talker hidden) & 1 (layer0 embed)
|
||||
proj_buf[:bsz, 0, :] = projection(last_talker_hidden.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
|
||||
proj_buf[:bsz, 1, :] = projection(layer0_embed.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
|
||||
|
||||
# Get pre-computed pos_ids for this bucket
|
||||
full_pos_ids = self._bucket_pos_ids.get(padded_bsz)
|
||||
if full_pos_ids is None:
|
||||
full_pos_ids = (
|
||||
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1).contiguous()
|
||||
)
|
||||
|
||||
# Use captured CUDA graph if available, otherwise call compiled fn.
|
||||
cuda_graph_entry = self._cuda_graphs.get(padded_bsz)
|
||||
|
||||
# Prepare sampling parameters
|
||||
stored_mode = self._wrapper_config.sampling_mode == "stored"
|
||||
if stored_mode:
|
||||
s_top_k = self._top_k
|
||||
s_top_p = self._top_p
|
||||
else:
|
||||
use_sampling = do_sample and temperature > 0
|
||||
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
|
||||
if use_sampling and top_p != 1.0:
|
||||
raise NotImplementedError(
|
||||
"top_p sampling is not implemented for the vLLM-native code predictor; please set top_p=1.0."
|
||||
)
|
||||
|
||||
# Output codes -- shape depends on return mode
|
||||
if self._wrapper_config.return_proj_buf:
|
||||
all_codes = torch.empty(bsz, num_groups, 1, dtype=torch.int64, device=device)
|
||||
all_codes[:, 0] = layer0_code.reshape(bsz, -1)[:, :1]
|
||||
else:
|
||||
all_codes = torch.empty(bsz, num_groups, dtype=torch.long, device=device)
|
||||
all_codes[:, 0] = layer0_code.reshape(bsz)
|
||||
|
||||
# Autoregressive loop: predict layers 1..G-1
|
||||
for step in range(1, num_groups):
|
||||
# Run transformer (CUDA graph replay or compiled forward)
|
||||
if cuda_graph_entry is not None:
|
||||
cuda_graph_entry[0].replay()
|
||||
hidden_out = cuda_graph_entry[1]
|
||||
else:
|
||||
hidden_out = model_fwd(proj_buf[:padded_bsz, :max_seq, :], full_pos_ids)
|
||||
|
||||
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
|
||||
|
||||
# Sample next code
|
||||
if stored_mode:
|
||||
# "stored" mode: top-k -> top-p -> softmax -> multinomial
|
||||
if s_top_k > 0:
|
||||
topk_vals, _ = logits.topk(s_top_k, dim=-1)
|
||||
logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf"))
|
||||
if s_top_p < 1.0:
|
||||
sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
|
||||
sorted_probs = F.softmax(sorted_logits, dim=-1)
|
||||
cumulative_probs = sorted_probs.cumsum(dim=-1)
|
||||
remove_mask = (cumulative_probs - sorted_probs) >= s_top_p
|
||||
sorted_logits[remove_mask] = float("-inf")
|
||||
logits = sorted_logits.scatter(1, sorted_idx, sorted_logits)
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
code = torch.multinomial(probs, num_samples=1)
|
||||
else:
|
||||
# "per_call" mode: temperature-scaled + top-k
|
||||
if use_sampling:
|
||||
scaled = logits * inv_temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
probs = F.softmax(scaled, dim=-1)
|
||||
code = torch.multinomial(probs, num_samples=1)
|
||||
else:
|
||||
code = logits.argmax(dim=-1, keepdim=True)
|
||||
|
||||
# Store code
|
||||
if self._wrapper_config.return_proj_buf:
|
||||
all_codes[:, step] = code
|
||||
else:
|
||||
all_codes[:, step] = code.reshape(bsz)
|
||||
|
||||
# Embed predicted code -> project -> next buffer position
|
||||
if step < num_groups - 1 or self._wrapper_config.return_proj_buf:
|
||||
new_embed = codec_embeds[step - 1](code)
|
||||
proj_buf[:bsz, step + 1, :] = projection(new_embed.reshape(bsz, 1, -1)).reshape(bsz, -1)
|
||||
|
||||
if self._wrapper_config.return_proj_buf:
|
||||
return all_codes, proj_buf[:bsz].clone()
|
||||
return all_codes
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights directly (no fused projection remapping needed)."""
|
||||
loaded: set[str] = set()
|
||||
model_weights: list[tuple[str, torch.Tensor]] = []
|
||||
other_weights: list[tuple[str, torch.Tensor]] = []
|
||||
|
||||
for name, w in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if name.startswith("model."):
|
||||
model_weights.append((name[len("model.") :], w))
|
||||
else:
|
||||
other_weights.append((name, w))
|
||||
|
||||
loaded_model = self.model.load_weights(model_weights)
|
||||
loaded |= {f"model.{n}" for n in loaded_model}
|
||||
|
||||
params = dict(self.named_parameters(remove_duplicate=False))
|
||||
for name, w in other_weights:
|
||||
param = params.get(name)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, w)
|
||||
loaded.add(name)
|
||||
|
||||
return loaded
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CosyVoice3 pipeline topology (frozen).
|
||||
|
||||
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
|
||||
Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
|
||||
* ``sync_process_input_func`` runs when ``deploy.async_chunk=false``:
|
||||
stage 1 builds full-sequence flow input via ``text2flow``.
|
||||
* ``async_chunk_process_next_stage_input_func`` runs when
|
||||
``deploy.async_chunk=true``: stage 0 streams codec chunks to stage 1
|
||||
through the shared-memory connector.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
|
||||
|
||||
COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
model_type="cosyvoice3",
|
||||
model_arch="CosyVoice3Model",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="cosyvoice3_talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
sampling_constraints={
|
||||
# merged speech stop token (logsumexp of all 200 stop logits)
|
||||
"stop_token_ids": [6562],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="cosyvoice3_code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="latent",
|
||||
sync_process_input_func=f"{_PROC}.text2flow",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -649,6 +649,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
|
||||
input_embeds: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
text_step: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GPU fast-path: run Fast AR to predict residual codebook codes.
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fish Speech S2 Pro pipeline topology (frozen).
|
||||
|
||||
Stage 0: slow_ar — text → RVQ codec tokens (LLM autoregressive).
|
||||
Stage 1: dac_decoder — RVQ tokens → audio waveform (LLM_GENERATION).
|
||||
|
||||
The HF config top-level reports ``model_type = "fish_qwen3_omni"`` (the
|
||||
OmniConfig that bundles slow-AR and fast-AR sub-configs), which is why the
|
||||
registry key follows the HF name rather than the human-readable "fish_speech".
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech"
|
||||
|
||||
FISH_SPEECH_PIPELINE = PipelineConfig(
|
||||
model_type="fish_qwen3_omni",
|
||||
model_arch="FishSpeechSlowARForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="fish_speech_slow_ar",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.slow_ar_to_dac_decoder_async_chunk"),
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
# <|im_end|> — stop when the model emits end-of-turn.
|
||||
"stop_token_ids": [151645],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="dac_decoder",
|
||||
model_arch="FishSpeechDACDecoder",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -38,10 +38,7 @@ def _encode_plain_text(tokenizer: Any, text: str) -> list[int]:
|
||||
|
||||
|
||||
def _encode_control_token(tokenizer: Any, token: str) -> list[int]:
|
||||
vocab = tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else {}
|
||||
token_id = vocab.get(token)
|
||||
if token_id is None:
|
||||
token_id = tokenizer.convert_tokens_to_ids(token)
|
||||
token_id = tokenizer.convert_tokens_to_ids(token)
|
||||
if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None):
|
||||
raise ValueError(f"Fish Speech tokenizer is missing required control token: {token}")
|
||||
return [int(token_id)]
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-Image model compatible with HuggingFace weights."""
|
||||
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
@@ -127,6 +128,14 @@ class GlmImageDataParser(MultiModalDataParser):
|
||||
parsers["img2img"] = self._parse_image_data
|
||||
return parsers
|
||||
|
||||
def parse_mm_data(self, mm_data, **kwargs):
|
||||
# Normalize "img2img" to "image" so the rest of the pipeline
|
||||
# (mm_hashes, _merge_mm_kwargs) uses a single modality key.
|
||||
normalized = {}
|
||||
for k, v in mm_data.items():
|
||||
normalized["image" if k == "img2img" else k] = v
|
||||
return super().parse_mm_data(normalized, **kwargs)
|
||||
|
||||
|
||||
class GlmImageProcessingInfo(BaseProcessingInfo):
|
||||
"""
|
||||
@@ -346,6 +355,10 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
target_h = mm_kwargs.get("target_h", 1024) if mm_kwargs else 1024
|
||||
target_w = mm_kwargs.get("target_w", 1024) if mm_kwargs else 1024
|
||||
|
||||
logger.debug(
|
||||
f"_call_hf_processor: target dimensions for generation: {target_h}x{target_w}, mm_kwargs={mm_kwargs}"
|
||||
)
|
||||
|
||||
if not mm_data or not mm_data.get("images"):
|
||||
# Text-to-image mode
|
||||
if processor is not None:
|
||||
@@ -566,6 +579,58 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
tensor_type="pt",
|
||||
)
|
||||
|
||||
def _apply_hf_processor_text_only(
|
||||
self, prompt_text: str, hf_processor_mm_kwargs: Mapping[str, object], tokenization_kwargs: Mapping[str, object]
|
||||
) -> list[int]:
|
||||
prompt_ids, _, _ = super()._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt_text,
|
||||
mm_items=MultiModalDataItems({}),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
return prompt_ids
|
||||
|
||||
def _build_generation_grids(self, hf_processor_mm_kwargs: Mapping[str, object]) -> torch.Tensor:
|
||||
"""Build generation grids for M-RoPE decode positions.
|
||||
|
||||
For GLM-Image generation, decode order is:
|
||||
1) small preview grid
|
||||
2) large target grid
|
||||
3) EOS
|
||||
|
||||
We store grids as [large, small] to match HF processor behavior, and
|
||||
decode logic consumes them in reverse order.
|
||||
"""
|
||||
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("target_h") if isinstance(hf_processor_mm_kwargs.get("target_h"), int) else None
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("target_w") if isinstance(hf_processor_mm_kwargs.get("target_w"), int) else None
|
||||
)
|
||||
if target_h is None or target_w is None:
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("height") if isinstance(hf_processor_mm_kwargs.get("height"), int) else 1024
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("width") if isinstance(hf_processor_mm_kwargs.get("width"), int) else 1024
|
||||
)
|
||||
|
||||
factor = 32
|
||||
target_h = (target_h // factor) * factor
|
||||
target_w = (target_w // factor) * factor
|
||||
token_h = target_h // factor
|
||||
token_w = target_w // factor
|
||||
|
||||
ratio = token_h / token_w if token_w > 0 else 1.0
|
||||
small_token_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
|
||||
small_token_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
|
||||
|
||||
return torch.tensor(
|
||||
[[1, token_h, token_w], [1, small_token_h, small_token_w]],
|
||||
dtype=torch.long,
|
||||
)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
@@ -594,126 +659,145 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
|
||||
logger.debug(f"_apply_hf_processor_main: mm_counts={mm_counts}, num_images={num_images}")
|
||||
|
||||
if num_images == 0 or enable_hf_prompt_update:
|
||||
if num_images == 0 and isinstance(prompt, str):
|
||||
# t2i mode or normal flow - use parent implementation
|
||||
return super()._apply_hf_processor_main(
|
||||
prompt=prompt,
|
||||
prompt_ids = self._apply_hf_processor_text_only(
|
||||
prompt_text=prompt,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
enable_hf_prompt_update=enable_hf_prompt_update,
|
||||
)
|
||||
|
||||
# i2i mode with enable_hf_prompt_update=False (cache miss scenario)
|
||||
# We need to build prompt_ids with image placeholders
|
||||
logger.debug(f"_apply_hf_processor_main: i2i mode with enable_hf_prompt_update=False, num_images={num_images}")
|
||||
# t2i has no source images, so mm features cannot provide image_grid_thw.
|
||||
# Provide explicit generation grids for M-RoPE to avoid fallback token parsing
|
||||
# (which can degrade high-resolution spatial positions, e.g. 1920x1920).
|
||||
try:
|
||||
mrope_grid_thw = self._build_generation_grids(hf_processor_mm_kwargs)
|
||||
mm_processed_data["mrope_image_grid_thw"] = mrope_grid_thw
|
||||
logger.info(
|
||||
"_apply_hf_processor_main t2i: mrope_image_grid_thw=%s",
|
||||
mrope_grid_thw.tolist(),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("_apply_hf_processor_main t2i: failed to set mrope_image_grid_thw: %s", e)
|
||||
|
||||
# Get mm data from our overridden _apply_hf_processor_mm_only
|
||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
return prompt_ids, mm_processed_data, False
|
||||
|
||||
# i2i mode: use unified HF processor path only.
|
||||
# This avoids drift between duplicated manual/HF i2i implementations.
|
||||
logger.debug(
|
||||
"_apply_hf_processor_main: i2i mode (enable_hf_prompt_update=%s), num_images=%s",
|
||||
enable_hf_prompt_update,
|
||||
num_images,
|
||||
)
|
||||
|
||||
# In this path we do NOT call HF apply_chat_template, so we must still
|
||||
# provide full grids (source + target) for M-RoPE to compute decode positions.
|
||||
# Keep `image_grid_thw` source-only for MM batching/validation.
|
||||
try:
|
||||
source_grid_thw = mm_processed_data.get("image_grid_thw")
|
||||
if source_grid_thw is not None and isinstance(source_grid_thw, torch.Tensor):
|
||||
# Compute target grid following HF GlmImageProcessor: factor=32.
|
||||
# Prefer explicit target_h/target_w if present, otherwise fall back.
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("target_h")
|
||||
if isinstance(hf_processor_mm_kwargs.get("target_h"), int)
|
||||
else None
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("target_w")
|
||||
if isinstance(hf_processor_mm_kwargs.get("target_w"), int)
|
||||
else None
|
||||
)
|
||||
if target_h is None or target_w is None:
|
||||
# Some callers pass generation size as height/width.
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("height")
|
||||
if isinstance(hf_processor_mm_kwargs.get("height"), int)
|
||||
else 1024
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("width")
|
||||
if isinstance(hf_processor_mm_kwargs.get("width"), int)
|
||||
else 1024
|
||||
)
|
||||
if not isinstance(prompt, str):
|
||||
# Online OpenAI chat preprocessing can arrive here with tokenized
|
||||
# prompts (list[int]) before serving_chat replaces engine prompt
|
||||
# with the clean text prompt. Do not fail the whole request.
|
||||
logger.warning(
|
||||
"_apply_hf_processor_main i2i: got tokenized prompt type=%s; "
|
||||
"using compatibility path for preprocessing",
|
||||
type(prompt).__name__,
|
||||
)
|
||||
|
||||
factor = 32
|
||||
target_h = (target_h // factor) * factor
|
||||
target_w = (target_w // factor) * factor
|
||||
token_h = target_h // factor
|
||||
token_w = target_w // factor
|
||||
target_grid = torch.tensor([[1, token_h, token_w]], dtype=source_grid_thw.dtype)
|
||||
prompt_ids = list(prompt)
|
||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
mm_processed_data["mrope_image_grid_thw"] = torch.cat([source_grid_thw, target_grid], dim=0)
|
||||
except Exception:
|
||||
# Best-effort only; M-RoPE has additional fallbacks.
|
||||
pass
|
||||
|
||||
# Build prompt_ids with image placeholders
|
||||
# _apply_prompt_updates will replace each [image_token_id] with expanded tokens
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<|image|>")
|
||||
|
||||
if isinstance(prompt, str):
|
||||
# Match HF GlmImageProcessor behavior: append target grid tokens + BOS.
|
||||
# This helps M-RoPE/grid parsing and keeps i2i vs t2i behavior aligned.
|
||||
# Preserve full grids for M-RoPE decode (source + target), while
|
||||
# keeping image_grid_thw source-only for MM batching.
|
||||
try:
|
||||
grid_bos = getattr(tokenizer, "grid_bos_token", "")
|
||||
grid_eos = getattr(tokenizer, "grid_eos_token", "")
|
||||
bos = getattr(tokenizer, "bos_token", "")
|
||||
|
||||
# Use the same target sizes we used for mrope grids when available.
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("target_h")
|
||||
if isinstance(hf_processor_mm_kwargs.get("target_h"), int)
|
||||
else None
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("target_w")
|
||||
if isinstance(hf_processor_mm_kwargs.get("target_w"), int)
|
||||
else None
|
||||
)
|
||||
if target_h is None or target_w is None:
|
||||
source_grid_thw = mm_processed_data.get("image_grid_thw")
|
||||
if source_grid_thw is not None and isinstance(source_grid_thw, torch.Tensor):
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("height")
|
||||
if isinstance(hf_processor_mm_kwargs.get("height"), int)
|
||||
else 1024
|
||||
hf_processor_mm_kwargs.get("target_h")
|
||||
if isinstance(hf_processor_mm_kwargs.get("target_h"), int)
|
||||
else None
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("width")
|
||||
if isinstance(hf_processor_mm_kwargs.get("width"), int)
|
||||
else 1024
|
||||
hf_processor_mm_kwargs.get("target_w")
|
||||
if isinstance(hf_processor_mm_kwargs.get("target_w"), int)
|
||||
else None
|
||||
)
|
||||
if target_h is None or target_w is None:
|
||||
target_h = (
|
||||
hf_processor_mm_kwargs.get("height")
|
||||
if isinstance(hf_processor_mm_kwargs.get("height"), int)
|
||||
else 1024
|
||||
)
|
||||
target_w = (
|
||||
hf_processor_mm_kwargs.get("width")
|
||||
if isinstance(hf_processor_mm_kwargs.get("width"), int)
|
||||
else 1024
|
||||
)
|
||||
|
||||
factor = 32
|
||||
target_h = (target_h // factor) * factor
|
||||
target_w = (target_w // factor) * factor
|
||||
token_h = target_h // factor
|
||||
token_w = target_w // factor
|
||||
|
||||
expanded_prompt = f"{prompt}{grid_bos}{token_h} {token_w}{grid_eos}{bos}"
|
||||
text_ids = tokenizer.encode(expanded_prompt, add_special_tokens=False)
|
||||
factor = 32
|
||||
token_h = max(1, target_h // factor)
|
||||
token_w = max(1, target_w // factor)
|
||||
target_grid = torch.tensor([[1, token_h, token_w]], dtype=source_grid_thw.dtype)
|
||||
mm_processed_data["mrope_image_grid_thw"] = torch.cat([source_grid_thw, target_grid], dim=0)
|
||||
except Exception:
|
||||
text_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
pass
|
||||
|
||||
# Prompt updates will expand image placeholders in this compatibility path.
|
||||
return prompt_ids, mm_processed_data, False
|
||||
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_list = [images.get(i) for i in range(images.get_count())]
|
||||
if not image_list:
|
||||
raise ValueError("GLM-Image i2i requires at least one source image in mm_items")
|
||||
|
||||
hf_inputs = self._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data={"images": image_list},
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
tok_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
input_ids = hf_inputs.get("input_ids")
|
||||
if input_ids is None:
|
||||
raise ValueError("HF i2i processor returned no input_ids")
|
||||
|
||||
if isinstance(input_ids, torch.Tensor):
|
||||
prompt_ids = input_ids[0].tolist() if input_ids.dim() > 1 else input_ids.tolist()
|
||||
else:
|
||||
text_ids = list(prompt)
|
||||
prompt_ids = (
|
||||
input_ids[0]
|
||||
if isinstance(input_ids, list) and input_ids and isinstance(input_ids[0], list)
|
||||
else list(input_ids)
|
||||
)
|
||||
|
||||
# Prepend image placeholders - one per image
|
||||
prompt_ids = [image_token_id] * num_images + text_ids
|
||||
mm_processed_data = BatchFeature(dict(), tensor_type="pt")
|
||||
for key in ("pixel_values", "image_grid_thw", "mrope_image_grid_thw"):
|
||||
value = hf_inputs.get(key)
|
||||
if value is not None:
|
||||
mm_processed_data[key] = value
|
||||
|
||||
logger.debug(f"_apply_hf_processor_main: built prompt_ids with {num_images} image placeholders")
|
||||
image_grid_thw = mm_processed_data.get("image_grid_thw")
|
||||
mrope_grid_thw = mm_processed_data.get("mrope_image_grid_thw")
|
||||
hf_config = self.info.get_hf_config()
|
||||
image_token_id = getattr(hf_config, "image_token_id", 167855)
|
||||
image_token_count = prompt_ids.count(image_token_id)
|
||||
logger.info(
|
||||
"_apply_hf_processor_main i2i(HF): num_images=%s, prompt_len=%s, image_token_count=%s, "
|
||||
"source_grid_shape=%s, mrope_grid_shape=%s",
|
||||
num_images,
|
||||
len(prompt_ids),
|
||||
image_token_count,
|
||||
tuple(image_grid_thw.shape) if image_grid_thw is not None else None,
|
||||
tuple(mrope_grid_thw.shape) if mrope_grid_thw is not None else None,
|
||||
)
|
||||
|
||||
# Return is_update_applied=False so _apply_prompt_updates will expand the placeholders
|
||||
return prompt_ids, mm_processed_data, False
|
||||
# HF processor already expanded image placeholders in input_ids.
|
||||
return prompt_ids, mm_processed_data, True
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -2667,9 +2751,23 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP
|
||||
# Input format: "text<sop>H W<eop><sop>h w<eop><bos>" where <bos>=image_start_token_id=16384
|
||||
# For 1024x1024: H=32, W=32 (large), h=16, w=16 (small preview)
|
||||
if not image_grid_thw:
|
||||
# Preferred path for t2i: use explicit target size propagated from
|
||||
# serving/request sampling params. This avoids fragile grid parsing
|
||||
# from token IDs and matches HF processor grid construction.
|
||||
target_h = kwargs.get("target_h")
|
||||
target_w = kwargs.get("target_w")
|
||||
if isinstance(target_h, int) and isinstance(target_w, int) and target_h > 0 and target_w > 0:
|
||||
factor = 32
|
||||
token_h = target_h // factor
|
||||
token_w = target_w // factor
|
||||
ratio = token_h / token_w if token_w > 0 else 1.0
|
||||
small_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
|
||||
small_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
|
||||
image_grid_thw = [[1, token_h, token_w], [1, small_h, small_w]]
|
||||
|
||||
# Try to parse from kwargs (passed from processor)
|
||||
hf_config_arg = kwargs.get("hf_config")
|
||||
if hf_config_arg is not None and hasattr(hf_config_arg, "image_grid_thw"):
|
||||
if (not image_grid_thw) and hf_config_arg is not None and hasattr(hf_config_arg, "image_grid_thw"):
|
||||
image_grid_thw = hf_config_arg.image_grid_thw
|
||||
|
||||
# If still empty, try to infer from input tokens
|
||||
@@ -2723,19 +2821,29 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP
|
||||
prompt_ends_with_start = len(input_tokens) > 0 and input_tokens[-1] == image_start_token_id
|
||||
if prompt_ends_with_start and len(image_grid_thw) == num_source_images and num_source_images > 0:
|
||||
# i2i mode: source grids exist but no target grids
|
||||
# Parse target grids from prompt tokens or use defaults
|
||||
parsed_grids = self._parse_grid_from_tokens(input_tokens, hf_config)
|
||||
if parsed_grids:
|
||||
# parsed_grids contains all grids mentioned in prompt
|
||||
# For i2i, add only the generation target grids
|
||||
if len(parsed_grids) > num_source_images:
|
||||
image_grid_thw = list(image_grid_thw) + parsed_grids[num_source_images:]
|
||||
else:
|
||||
# Fallback: add default 1024x1024 generation grids (1 target for i2i)
|
||||
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
|
||||
# Prefer explicit target size propagated from request sampling params.
|
||||
# This avoids fragile grid parsing from token IDs for non-1024 i2i.
|
||||
target_h = kwargs.get("target_h")
|
||||
target_w = kwargs.get("target_w")
|
||||
if isinstance(target_h, int) and isinstance(target_w, int) and target_h > 0 and target_w > 0:
|
||||
factor = 32
|
||||
token_h = target_h // factor
|
||||
token_w = target_w // factor
|
||||
image_grid_thw = list(image_grid_thw) + [[1, token_h, token_w]]
|
||||
else:
|
||||
# Fallback to default 1024x1024 grids for generation
|
||||
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
|
||||
# Parse target grids from prompt tokens or use defaults
|
||||
parsed_grids = self._parse_grid_from_tokens(input_tokens, hf_config)
|
||||
if parsed_grids:
|
||||
# parsed_grids contains all grids mentioned in prompt
|
||||
# For i2i, add only the generation target grids
|
||||
if len(parsed_grids) > num_source_images:
|
||||
image_grid_thw = list(image_grid_thw) + parsed_grids[num_source_images:]
|
||||
else:
|
||||
# Fallback: add default 1024x1024 generation grid (1 target for i2i)
|
||||
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
|
||||
else:
|
||||
# Fallback to default 1024x1024 grid for generation
|
||||
image_grid_thw = list(image_grid_thw) + [[1, 32, 32]]
|
||||
|
||||
llm_pos_ids_list: list[torch.Tensor] = []
|
||||
|
||||
|
||||
@@ -77,7 +77,9 @@ from vllm.multimodal.processing import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.tokenizer import get_tokenizer
|
||||
from vllm.utils.tensor_schema import TensorSchema
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.autoencoder_kl_3d import AutoencoderKLConv3D
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.siglip2 import LightProjector, Siglip2VisionTransformer
|
||||
@@ -175,8 +177,11 @@ class HunyuanModel(HunYuanModel):
|
||||
return True
|
||||
return False
|
||||
|
||||
skipped_unexpected: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if contains_unexpected_keyword(name, unexpected_keywords):
|
||||
skipped_unexpected.add(name)
|
||||
continue
|
||||
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
@@ -362,6 +367,17 @@ class HunyuanModel(HunYuanModel):
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
if skipped_unexpected:
|
||||
logger.warning_once(
|
||||
"Skipped %d weights matching unexpected_keywords "
|
||||
"(e.g. vae, vision_model, patch_embed, timestep_emb). "
|
||||
"If upstream renamed components, these may be silently "
|
||||
"lost. Skipped names: %s",
|
||||
len(skipped_unexpected),
|
||||
sorted(skipped_unexpected)[:10],
|
||||
)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
@@ -1149,6 +1165,8 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
|
||||
HunyuanImage3Inputs: TypeAlias = HunyuanImage3PixelInputs
|
||||
|
||||
prefer_model_sampler = True
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -1199,6 +1217,10 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
# --- AR-stage components ---
|
||||
# These are needed for image encoding in the AR stage.
|
||||
# If a future text-only stage is added, gate on vllm_config.model_config.model_stage.
|
||||
|
||||
# vae
|
||||
self.vae = AutoencoderKLConv3D.from_config(config.vae)
|
||||
self.patch_embed = UNetDown(
|
||||
@@ -1226,6 +1248,63 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
self._mrope_joint_img_sep_token_id = tokenizer.convert_tokens_to_ids("<joint_img_sep>")
|
||||
self._mrope_max_num_patches = config.vit_processor.get("max_num_patches", 729)
|
||||
|
||||
# Special token IDs for logits processors (stage transitions).
|
||||
# These mirror the official tokenization_hunyuan_image_3.py setup.
|
||||
self._end_of_think_id = tokenizer.convert_tokens_to_ids("</think>")
|
||||
self._recaption_id = tokenizer.convert_tokens_to_ids("<recaption>")
|
||||
self._end_of_recaption_id = tokenizer.convert_tokens_to_ids("</recaption>")
|
||||
self._answer_id = tokenizer.convert_tokens_to_ids("<answer>")
|
||||
self._end_of_answer_id = tokenizer.convert_tokens_to_ids("</answer>")
|
||||
image_base_size = getattr(config, "image_base_size", 1024)
|
||||
self._size_token_id = tokenizer.convert_tokens_to_ids(f"<img_size_{image_base_size}>")
|
||||
self._start_ratio_id = tokenizer.convert_tokens_to_ids("<img_ratio_0>")
|
||||
self._end_ratio_id = tokenizer.convert_tokens_to_ids("<img_ratio_32>")
|
||||
ratio_33 = tokenizer.convert_tokens_to_ids("<img_ratio_33>")
|
||||
ratio_36 = tokenizer.convert_tokens_to_ids("<img_ratio_36>")
|
||||
self._ratio_other_slices = [(ratio_33, ratio_36 + 1)]
|
||||
# Build the full set of ratio token IDs for use as stop tokens.
|
||||
self._all_ratio_ids = set(range(self._start_ratio_id, self._end_ratio_id + 1))
|
||||
for s, e in self._ratio_other_slices:
|
||||
self._all_ratio_ids.update(range(s, e))
|
||||
|
||||
# Determine mode: comprehension (I2T/T2T) vs generation (IT2I/T2I).
|
||||
engine_output_type = getattr(vllm_config.model_config, "engine_output_type", None)
|
||||
self._is_comprehension = engine_output_type in (None, "text")
|
||||
|
||||
# For comprehension mode, block image generation tokens but allow
|
||||
# text structure tokens (<think>, <answer>, etc.) so the model can
|
||||
# follow its natural generation pattern. Stop tokens in YAML will
|
||||
# terminate at </answer> or EOS.
|
||||
self._blocked_token_ids: set[int] = set()
|
||||
if self._is_comprehension:
|
||||
self._blocked_token_ids.update(
|
||||
[
|
||||
self._mrope_boi_token_id, # <boi>
|
||||
self._mrope_eoi_token_id, # <eoi>
|
||||
self._size_token_id, # <img_size_*>
|
||||
]
|
||||
)
|
||||
self._blocked_token_ids.update(self._all_ratio_ids)
|
||||
|
||||
# For generation mode, build stage transition map.
|
||||
# Official logic: </think> → [<recaption>],
|
||||
# </recaption> → [<answer>, <boi>, <img_size_*>]
|
||||
# After <img_size_*>, restrict vocab to ratio tokens only.
|
||||
# Stage-transition forced sequences, keyed by trigger token.
|
||||
self._stage_transitions: dict[int, list[int]] = {}
|
||||
if not self._is_comprehension:
|
||||
self._stage_transitions[self._end_of_think_id] = [
|
||||
self._recaption_id,
|
||||
]
|
||||
self._stage_transitions[self._end_of_recaption_id] = [
|
||||
self._answer_id,
|
||||
self._mrope_boi_token_id,
|
||||
self._size_token_id,
|
||||
]
|
||||
|
||||
self._sampler: Sampler | None = None
|
||||
self._eos_token_id: int = tokenizer.eos_token_id
|
||||
|
||||
self._replace_rotary_embeddings()
|
||||
|
||||
def _replace_rotary_embeddings(self):
|
||||
@@ -1257,6 +1336,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
head_dim,
|
||||
rope_theta,
|
||||
)
|
||||
if replaced == 0:
|
||||
raise RuntimeError(
|
||||
"HunyuanImage3: _replace_rotary_embeddings replaced 0 layers. "
|
||||
"The custom interleaved 2D mRoPE is not active — model outputs "
|
||||
"will be incorrect. Check that model.layers[*].self_attn.rotary_emb exists."
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self,
|
||||
@@ -1274,6 +1359,10 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
if vit_pixel_values is None or vae_pixel_values is None:
|
||||
return None
|
||||
|
||||
# Handle empty batch (e.g., during profiling with 0 images / T2T mode)
|
||||
if vit_pixel_values.numel() == 0 or vae_pixel_values.numel() == 0:
|
||||
return None
|
||||
|
||||
return HunyuanImage3PixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values={
|
||||
@@ -1472,6 +1561,112 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Custom sampler — applies HunyuanImage3-specific logits processors
|
||||
# before the standard sampling step.
|
||||
#
|
||||
# Comprehension (I2T / T2T):
|
||||
# Block generation-specific special tokens so sampling can't
|
||||
# accidentally produce <answer>, <boi>, ratio tokens, etc.
|
||||
#
|
||||
# Generation (IT2I / T2I think):
|
||||
# 1. _StageTransitionLogitsProcessor — force token sequences at
|
||||
# transition boundaries (</think> → <recaption>, etc.)
|
||||
# 2. _ConditionalSliceVocabLogitsProcessor — after <img_size_*>,
|
||||
# restrict vocab to ratio tokens only (greedy).
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> SamplerOutput | None:
|
||||
if logits is None or logits.numel() == 0:
|
||||
return None
|
||||
|
||||
if self._sampler is None:
|
||||
self._sampler = Sampler()
|
||||
|
||||
min_score = torch.finfo(logits.dtype).min
|
||||
|
||||
assert logits.shape[0] == 1, f"HunyuanImage3 sampler requires max_num_seqs=1, got batch size {logits.shape[0]}"
|
||||
|
||||
for req_idx in range(logits.shape[0]):
|
||||
decoded_tokens: list[int] = (
|
||||
sampling_metadata.output_token_ids[req_idx] if req_idx < len(sampling_metadata.output_token_ids) else []
|
||||
)
|
||||
last_token = decoded_tokens[-1] if decoded_tokens else -1
|
||||
|
||||
if self._is_comprehension:
|
||||
for tid in self._blocked_token_ids:
|
||||
logits[req_idx, tid] = min_score
|
||||
else:
|
||||
forced = self._get_forced_token(decoded_tokens)
|
||||
if forced is not None:
|
||||
logits[req_idx].fill_(min_score)
|
||||
logits[req_idx, forced] = 0
|
||||
elif last_token == self._size_token_id:
|
||||
self._apply_ratio_restriction(logits, req_idx, min_score)
|
||||
elif last_token in self._all_ratio_ids:
|
||||
logits[req_idx].fill_(min_score)
|
||||
logits[req_idx, self._eos_token_id] = 0
|
||||
|
||||
return self._sampler(logits=logits, sampling_metadata=sampling_metadata)
|
||||
|
||||
def _get_forced_token(self, decoded_tokens: list[int]) -> int | None:
|
||||
"""Derive the next forced token from output history (stateless).
|
||||
|
||||
Scans decoded_tokens backwards for the most recent trigger token,
|
||||
then prefix-matches the forced sequence against what followed.
|
||||
Returns the next token to force, or None if the sequence is complete
|
||||
or history has diverged from the expected forced sequence.
|
||||
"""
|
||||
for i in range(len(decoded_tokens) - 1, -1, -1):
|
||||
trigger = decoded_tokens[i]
|
||||
if trigger not in self._stage_transitions:
|
||||
continue
|
||||
|
||||
forced_seq = self._stage_transitions[trigger]
|
||||
emitted = decoded_tokens[i + 1 :]
|
||||
|
||||
matched = 0
|
||||
for expected, actual in zip(forced_seq, emitted):
|
||||
if actual != expected:
|
||||
# History diverged from the expected forced sequence.
|
||||
# Stop applying transition forcing for safety.
|
||||
return None
|
||||
matched += 1
|
||||
|
||||
if matched < len(forced_seq):
|
||||
return forced_seq[matched]
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _apply_ratio_restriction(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
req_idx: int,
|
||||
min_score: float,
|
||||
) -> None:
|
||||
"""Port of official _ConditionalSliceVocabLogitsProcessor.__call__.
|
||||
|
||||
After the size token, only allow ratio tokens and pick greedily.
|
||||
"""
|
||||
original = logits[req_idx].clone()
|
||||
logits[req_idx].fill_(min_score)
|
||||
# Allow primary ratio range.
|
||||
logits[req_idx, self._start_ratio_id : self._end_ratio_id + 1] = original[
|
||||
self._start_ratio_id : self._end_ratio_id + 1
|
||||
]
|
||||
# Allow extra ratio slices.
|
||||
for s, e in self._ratio_other_slices:
|
||||
logits[req_idx, s:e] = original[s:e]
|
||||
# Force greedy: keep only the argmax.
|
||||
max_id = logits[req_idx].argmax().item()
|
||||
logits[req_idx].fill_(min_score)
|
||||
logits[req_idx, max_id] = 0
|
||||
|
||||
def make_empty_intermediate_tensors(
|
||||
self, batch_size: int, dtype: torch.dtype, device: torch.device
|
||||
) -> IntermediateTensors:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiMo Audio pipeline topology (frozen).
|
||||
|
||||
Stage 0: fused thinker+talker — multimodal understanding + text + RVQ codes.
|
||||
Stage 1: Code2Wav — RVQ codes → waveform.
|
||||
|
||||
MiMoAudioConfig inherits from Qwen2Config, so the HF ``model_type`` field
|
||||
reports ``qwen2`` — the registry's model_type-based auto-detect can't
|
||||
disambiguate. ``hf_architectures`` lets ``StageConfigFactory`` fall back to
|
||||
matching ``hf_config.architectures`` instead.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.mimo_audio"
|
||||
|
||||
MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
model_type="mimo_audio",
|
||||
# HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in
|
||||
# ``model_executor/models/registry.py``; it resolves to the internal
|
||||
# class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``.
|
||||
model_arch="MiMoAudioModel",
|
||||
hf_architectures=("MiMoAudioModel",),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="fused_thinker_talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
from .ming_flash_omni import MingFlashOmniForConditionalGeneration
|
||||
from .ming_flash_omni_thinker import (
|
||||
MingFlashOmniThinkerDummyInputsBuilder,
|
||||
MingFlashOmniThinkerForConditionalGeneration,
|
||||
MingFlashOmniThinkerMultiModalProcessor,
|
||||
MingFlashOmniThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MingFlashOmniForConditionalGeneration",
|
||||
"MingFlashOmniThinkerForConditionalGeneration",
|
||||
"MingFlashOmniThinkerProcessingInfo",
|
||||
"MingFlashOmniThinkerMultiModalProcessor",
|
||||
"MingFlashOmniThinkerDummyInputsBuilder",
|
||||
]
|
||||
@@ -0,0 +1,246 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright 2024 ANT Group and the HuggingFace Inc. team.
|
||||
# Copyright (c) 2022 OpenAI
|
||||
# Adapted from Ming repository modeling_whisper_encoder.py
|
||||
# https://github.com/inclusionAI/Ming
|
||||
|
||||
import operator
|
||||
from collections.abc import Iterable
|
||||
from itertools import accumulate
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func
|
||||
from vllm_omni.model_executor.models.whisper_utils import Conv1d, Linear, sinusoids
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MultiHeadAttention(nn.Module):
|
||||
"""Multi-head attention with packed sequence support.
|
||||
Adapted from Qwen3-TTS WhisperEncoder.
|
||||
"""
|
||||
|
||||
def __init__(self, n_state: int, n_head: int, use_flash_attn: bool = True):
|
||||
super().__init__()
|
||||
self.n_head = n_head
|
||||
self.query = Linear(n_state, n_state)
|
||||
self.key = Linear(n_state, n_state, bias=False)
|
||||
self.value = Linear(n_state, n_state)
|
||||
self.out = Linear(n_state, n_state)
|
||||
|
||||
if use_flash_attn and not HAS_FLASH_ATTN:
|
||||
logger.warning("flash-attn is not available. Fallback to manual PyTorch version")
|
||||
self.use_flash_attn = use_flash_attn and HAS_FLASH_ATTN
|
||||
|
||||
def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
|
||||
"""Forward pass with packed sequence support.
|
||||
|
||||
Args:
|
||||
x: [total_tokens, n_state] packed sequence
|
||||
cu_seqlens: [num_seqs + 1] cumulative sequence lengths, e.g. [0, len1, len1+len2, ...]
|
||||
|
||||
Returns:
|
||||
[total_tokens, n_state] attention output
|
||||
"""
|
||||
q = self.query(x)
|
||||
k = self.key(x)
|
||||
v = self.value(x)
|
||||
|
||||
n_ctx, n_state = q.shape
|
||||
head_dim = n_state // self.n_head
|
||||
|
||||
q = q.view(n_ctx, self.n_head, head_dim)
|
||||
k = k.view(n_ctx, self.n_head, head_dim)
|
||||
v = v.view(n_ctx, self.n_head, head_dim)
|
||||
|
||||
# Try flash attention varlen
|
||||
if self.use_flash_attn and cu_seqlens is not None and q.dtype in [torch.float16, torch.bfloat16]:
|
||||
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
|
||||
attn_output = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen)
|
||||
else:
|
||||
attn_output = self._manual_attention(q, k, v, cu_seqlens)
|
||||
|
||||
# Reshape back: [T, H, D] -> [T, H*D]
|
||||
attn_output = attn_output.contiguous().view(n_ctx, n_state)
|
||||
return self.out(attn_output)
|
||||
|
||||
def _manual_attention(
|
||||
self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, cu_seqlens: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Manual attention for variable-length sequences (fallback)."""
|
||||
_, n_head, head_dim = q.shape
|
||||
scale = head_dim**-0.5
|
||||
|
||||
# Unpack sequences and pad to max length
|
||||
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
|
||||
batch_size = len(seqlens)
|
||||
max_seqlen = max(seqlens)
|
||||
|
||||
# Create padded tensors
|
||||
q_padded = torch.zeros(batch_size, max_seqlen, n_head, head_dim, dtype=q.dtype, device=q.device)
|
||||
k_padded = torch.zeros_like(q_padded)
|
||||
v_padded = torch.zeros_like(q_padded)
|
||||
|
||||
# Fill with actual sequences
|
||||
for i in range(batch_size):
|
||||
start_idx = cu_seqlens[i]
|
||||
end_idx = cu_seqlens[i + 1]
|
||||
seq_len = seqlens[i]
|
||||
q_padded[i, :seq_len] = q[start_idx:end_idx]
|
||||
k_padded[i, :seq_len] = k[start_idx:end_idx]
|
||||
v_padded[i, :seq_len] = v[start_idx:end_idx]
|
||||
|
||||
# Transpose for attention: [B, H, T, D]
|
||||
q_padded = q_padded.transpose(1, 2)
|
||||
k_padded = k_padded.transpose(1, 2)
|
||||
v_padded = v_padded.transpose(1, 2)
|
||||
|
||||
# Create attention mask for variable lengths: 0 for valid positions, -inf for padding
|
||||
padding_mask = (
|
||||
torch.arange(max_seqlen, device=q.device)[None, :] >= torch.tensor(seqlens, device=q.device)[:, None]
|
||||
)
|
||||
attn_mask = torch.zeros(batch_size, 1, 1, max_seqlen, dtype=q.dtype, device=q.device)
|
||||
attn_mask = attn_mask.masked_fill(padding_mask.unsqueeze(1).unsqueeze(2), -torch.finfo(q.dtype).max)
|
||||
|
||||
# Compute attention
|
||||
attn_scores = torch.matmul(q_padded, k_padded.transpose(-2, -1)) * scale
|
||||
attn_scores = attn_scores + attn_mask
|
||||
attn_weights = F.softmax(attn_scores, dim=-1)
|
||||
context = torch.matmul(attn_weights, v_padded)
|
||||
|
||||
# Transpose back: [B, H, T, D] -> [B, T, H, D]
|
||||
context = context.transpose(1, 2).contiguous()
|
||||
output_packed = torch.cat([context[i, : seqlens[i]] for i in range(batch_size)], dim=0)
|
||||
|
||||
return output_packed
|
||||
|
||||
|
||||
class ResidualAttentionBlock(nn.Module):
|
||||
"""Whisper-style residual attention block with packed sequence support.
|
||||
|
||||
Adapted from
|
||||
https://github.com/openai/whisper/blob/v20250625/whisper/model.py
|
||||
vllm_omni/model_executor/models/qwen3_tts/tokenizer_25hz/vq/whisper_encoder.py
|
||||
"""
|
||||
|
||||
def __init__(self, n_state: int, n_head: int, use_flash_attn: bool = True):
|
||||
super().__init__()
|
||||
self.attn = MultiHeadAttention(n_state, n_head, use_flash_attn=use_flash_attn)
|
||||
self.attn_ln = nn.LayerNorm(n_state)
|
||||
|
||||
n_mlp = n_state * 4
|
||||
self.mlp = nn.Sequential(
|
||||
Linear(n_state, n_mlp),
|
||||
nn.GELU(),
|
||||
Linear(n_mlp, n_state),
|
||||
)
|
||||
self.mlp_ln = nn.LayerNorm(n_state)
|
||||
|
||||
def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
|
||||
x = x + self.attn(self.attn_ln(x), cu_seqlens=cu_seqlens)
|
||||
x = x + self.mlp(self.mlp_ln(x))
|
||||
return x
|
||||
|
||||
|
||||
class WhisperAudioEncoder(nn.Module):
|
||||
"""Whisper audio encoder for Ming with packed sequence support.
|
||||
|
||||
Adapted from
|
||||
https://github.com/openai/whisper/blob/v20250625/whisper/model.py
|
||||
vllm_omni/model_executor/models/qwen3_tts/tokenizer_25hz/vq/whisper_encoder.py
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
n_mels: int = 128,
|
||||
n_ctx: int = 15000,
|
||||
n_state: int = 1280,
|
||||
n_head: int = 20,
|
||||
n_layer: int = 32,
|
||||
use_flash_attn: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
|
||||
self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
|
||||
# self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state))
|
||||
self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state))
|
||||
self.blocks = nn.ModuleList(
|
||||
[ResidualAttentionBlock(n_state, n_head, use_flash_attn=use_flash_attn) for _ in range(n_layer)]
|
||||
)
|
||||
self.ln_post = nn.LayerNorm(n_state)
|
||||
self.audio_emb_dim = n_state
|
||||
|
||||
self.n_layer = n_layer
|
||||
self.n_mels = n_mels
|
||||
self.use_flash_attn = use_flash_attn
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x_list: list[torch.Tensor],
|
||||
audio_lens: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass with packed sequence format for variable-length inputs.
|
||||
|
||||
Args:
|
||||
x_list: List of [n_mels, T_i] mel spectrogram features for each audio
|
||||
audio_lens: List of original audio lengths in frames
|
||||
|
||||
Returns:
|
||||
[total_T', n_state] packed encoded audio features, where
|
||||
total_T' is the sum of all encoded sequence lengths
|
||||
"""
|
||||
# Cast inputs to model dtype
|
||||
target_dtype = self.conv1.weight.dtype
|
||||
x_list = [x.to(target_dtype) for x in x_list]
|
||||
|
||||
encoded_list = []
|
||||
encoded_lens = []
|
||||
for mel_spec in x_list:
|
||||
# mel_spec: [n_mels, T] - process through conv layers
|
||||
x = mel_spec.unsqueeze(0) # [1, n_mels, T]
|
||||
x = F.gelu(self.conv1(x))
|
||||
x = F.gelu(self.conv2(x))
|
||||
x = x.squeeze(0).transpose(0, 1) # [T', n_state]
|
||||
|
||||
# Add positional embedding
|
||||
seq_len = x.shape[0]
|
||||
positional_embedding = self.positional_embedding[:seq_len, :]
|
||||
x = (x + positional_embedding).to(x.dtype)
|
||||
|
||||
encoded_list.append(x)
|
||||
encoded_lens.append(seq_len)
|
||||
|
||||
x_packed = torch.cat(encoded_list, dim=0) # [total_T', n_state]
|
||||
|
||||
cu_seqlens = list(accumulate(encoded_lens, func=operator.add, initial=0))
|
||||
cu_seqlens = torch.tensor(cu_seqlens, device=x_packed.device, dtype=torch.int32)
|
||||
|
||||
for block in self.blocks:
|
||||
x_packed = block(x_packed, cu_seqlens=cu_seqlens)
|
||||
|
||||
x_packed = self.ln_post(x_packed)
|
||||
return x_packed
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict: dict[str, torch.Tensor] = {
|
||||
**dict(self.named_parameters(remove_duplicate=False)),
|
||||
**dict(self.named_buffers()),
|
||||
}
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if name not in params_dict:
|
||||
logger.warning("Skipping unknown audio encoder weight: %s", name)
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
@@ -0,0 +1,223 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright 2024 ANT Group and the HuggingFace Inc. team. All rights reserved.
|
||||
# Adapted from Ming repository modeling_bailingmm2.py
|
||||
# https://github.com/inclusionAI/Ming
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Ming-flash-omni-2.0 unified model (thinker + imagegen + talker)."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.model_executor.models.utils import (
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config, MingFlashOmniConfig
|
||||
|
||||
from .ming_flash_omni_thinker import (
|
||||
MingFlashOmniThinkerDummyInputsBuilder,
|
||||
MingFlashOmniThinkerMultiModalProcessor,
|
||||
MingFlashOmniThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MingFlashOmniThinkerMultiModalProcessor,
|
||||
info=MingFlashOmniThinkerProcessingInfo,
|
||||
dummy_inputs=MingFlashOmniThinkerDummyInputsBuilder,
|
||||
)
|
||||
class MingFlashOmniForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
CustomProcessMixin,
|
||||
):
|
||||
"""Unified Ming-flash-omni-2.0 model combining thinker, imagegen, and talker."""
|
||||
|
||||
supports_multimodal = True
|
||||
requires_raw_input_tokens: bool = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.config = config
|
||||
|
||||
if isinstance(config, MingFlashOmniConfig):
|
||||
thinker_config = config.thinker_config
|
||||
else:
|
||||
thinker_config = config
|
||||
|
||||
self.thinker_config: BailingMM2Config = thinker_config
|
||||
self.model_stage = vllm_config.model_config.model_stage
|
||||
|
||||
if self.model_stage == "thinker":
|
||||
thinker_vllm_config = vllm_config.with_hf_config(
|
||||
thinker_config, architectures=["MingFlashOmniThinkerForConditionalGeneration"]
|
||||
)
|
||||
self.thinker = init_vllm_registered_model(
|
||||
vllm_config=thinker_vllm_config,
|
||||
prefix=maybe_prefix(prefix, "thinker"),
|
||||
architectures=["MingFlashOmniThinkerForConditionalGeneration"],
|
||||
)
|
||||
self.model = self.thinker
|
||||
self.imagegen = None
|
||||
self.talker = None
|
||||
|
||||
elif self.model_stage == "imagegen":
|
||||
# TODO: Implement image generator stage
|
||||
raise NotImplementedError(
|
||||
"Image generation stage is not yet implemented. Please use model_stage='thinker' for now."
|
||||
)
|
||||
|
||||
elif self.model_stage == "talker":
|
||||
# TODO: Implement talker (TTS) stage
|
||||
raise NotImplementedError(
|
||||
"Talker (TTS) stage is not yet implemented. Please use model_stage='thinker' for now."
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid model_stage: {self.model_stage}. Must be one of: 'thinker', 'imagegen', 'talker'"
|
||||
)
|
||||
|
||||
# Set up intermediate tensors
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> OmniOutput:
|
||||
return self.model.forward(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata=None,
|
||||
) -> torch.Tensor | None:
|
||||
if hasattr(self.model, "compute_logits"):
|
||||
return self.model.compute_logits(hidden_states, sampling_metadata)
|
||||
return None
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata,
|
||||
):
|
||||
if hasattr(self.model, "sample"):
|
||||
return self.model.sample(logits, sampling_metadata)
|
||||
raise NotImplementedError("sample method not available on current stage")
|
||||
|
||||
def get_mrope_input_positions(self, *args, **kwargs):
|
||||
if hasattr(self.model, "get_mrope_input_positions"):
|
||||
return self.model.get_mrope_input_positions(*args, **kwargs)
|
||||
raise NotImplementedError("get_mrope_input_positions not available on current stage")
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loaded_weights = set()
|
||||
thinker_weights = []
|
||||
imagegen_weights = []
|
||||
talker_weights = []
|
||||
|
||||
for name, value in weights:
|
||||
if name.startswith("thinker."):
|
||||
thinker_weights.append((name, value))
|
||||
elif name.startswith("imagegen."):
|
||||
imagegen_weights.append((name, value))
|
||||
elif name.startswith("talker."):
|
||||
talker_weights.append((name, value))
|
||||
else:
|
||||
# Weights without prefix go to thinker by default
|
||||
thinker_weights.append((name, value))
|
||||
|
||||
if self.model_stage == "thinker" and thinker_weights:
|
||||
# Remove "thinker." prefix before loading
|
||||
thinker_weights_stripped = [
|
||||
(name.replace("thinker.", "", 1) if name.startswith("thinker.") else name, value)
|
||||
for name, value in thinker_weights
|
||||
]
|
||||
thinker_loaded = self.thinker.load_weights(thinker_weights_stripped)
|
||||
thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
|
||||
loaded_weights.update(thinker_loaded)
|
||||
|
||||
# TODO: Load imagegen weights when implemented
|
||||
# TODO: Load talker weights when implemented
|
||||
|
||||
return loaded_weights
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="thinker.language_model",
|
||||
connector=["thinker.linear_proj.", "thinker.linear_proj_audio."],
|
||||
tower_model=["thinker.vision.", "thinker.audio."],
|
||||
)
|
||||
|
||||
@property
|
||||
def sampler(self):
|
||||
if hasattr(self.model, "sampler"):
|
||||
return self.model.sampler
|
||||
return None
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings=None,
|
||||
*,
|
||||
is_multimodal=None,
|
||||
) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(
|
||||
input_ids,
|
||||
multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
return self.model.embed_multimodal(**kwargs)
|
||||
@@ -0,0 +1,893 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright 2024 ANT Group and the HuggingFace Inc. team.
|
||||
# Adapted from Ming repository modeling_bailingmm2.py and processing_bailingmm2.py
|
||||
# https://github.com/inclusionAI/Ming
|
||||
|
||||
"""Ming-flash-omni-2.0 Thinker stage implementation (multimodal understanding)."""
|
||||
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from typing import Annotated, Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.qwen2_5_vl import (
|
||||
Qwen2_5_VLImageInputs,
|
||||
Qwen2_5_VLImagePixelInputs,
|
||||
Qwen2_5_VLVideoInputs,
|
||||
Qwen2_5_VLVideoPixelInputs,
|
||||
)
|
||||
from vllm.model_executor.models.qwen2_vl import (
|
||||
Qwen2VLProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
AudioProcessorItems,
|
||||
ImageProcessorItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
VideoProcessorItems,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config
|
||||
from vllm_omni.transformers_utils.processors.ming import (
|
||||
PLACEHOLDER_AUDIO_TOKEN_IN_TEXT,
|
||||
PLACEHOLDER_IMAGE_TOKEN_IN_TEXT,
|
||||
PLACEHOLDER_VIDEO_TOKEN_IN_TEXT,
|
||||
MingFlashOmniProcessor,
|
||||
MingWhisperFeatureExtractor,
|
||||
)
|
||||
|
||||
from .audio_encoder import WhisperAudioEncoder
|
||||
from .modeling_bailing_moe_v2 import BailingMoeV2ForCausalLM
|
||||
from .projectors import AudioProjector, VisionProjector
|
||||
from .vision_encoder import MingVisionEncoder
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MingAudioInput(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- l: Total audio frames (clips concatenated along the time axis)
|
||||
- nm: Number of mel bins
|
||||
- N: Max number of audio clips per batch item
|
||||
"""
|
||||
|
||||
audio_feats: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("b", "l", "nm"),
|
||||
]
|
||||
|
||||
audio_feats_lengths: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("b", "N"),
|
||||
]
|
||||
|
||||
|
||||
class MingFlashOmniThinkerProcessingInfo(Qwen2VLProcessingInfo):
|
||||
def get_hf_config(self) -> BailingMM2Config:
|
||||
return self.ctx.get_hf_config(BailingMM2Config)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
return self.ctx.get_hf_processor(MingFlashOmniProcessor, **kwargs)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
# See `_normalize_audio_tensor` in vllm_omni/transformers_utils/processors/ming.py
|
||||
return 1
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None, "video": None, "audio": None}
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
mm_counts = mm_counts or {}
|
||||
requested_modalities = {m for m, c in mm_counts.items() if c > 0}
|
||||
mm_max_tokens: dict[str, int] = {}
|
||||
|
||||
if requested_modalities & {"image", "video"}:
|
||||
vl_tokens = super().get_mm_max_tokens_per_item(
|
||||
seq_len=seq_len,
|
||||
mm_counts=mm_counts,
|
||||
)
|
||||
mm_max_tokens.update({m: vl_tokens[m] for m in ["image", "video"] if m in requested_modalities})
|
||||
|
||||
if "audio" in requested_modalities:
|
||||
# TODO: consider computing from audio config
|
||||
mm_max_tokens["audio"] = 3000
|
||||
|
||||
return mm_max_tokens
|
||||
|
||||
def get_feature_extractor(self, **kwargs: object) -> MingWhisperFeatureExtractor:
|
||||
hf_processor = self.get_hf_processor(**kwargs)
|
||||
feature_extractor = hf_processor.audio_processor
|
||||
assert isinstance(feature_extractor, MingWhisperFeatureExtractor)
|
||||
return feature_extractor
|
||||
|
||||
def get_data_parser(self):
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
return MultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
|
||||
|
||||
class MingFlashOmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[MingFlashOmniThinkerProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
|
||||
audio_token: str = hf_processor.audio_token
|
||||
image_token: str = hf_processor.image_token
|
||||
video_token: str = hf_processor.video_token
|
||||
|
||||
return image_token * num_images + video_token * num_videos + audio_token * num_audios
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions] | None = None,
|
||||
) -> MultiModalDataDict:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
|
||||
# Default dimensions for dummy data
|
||||
image_width, image_height = 448, 448
|
||||
video_width, video_height = 448, 448
|
||||
num_frames = 8
|
||||
audio_duration = 3.0 # seconds
|
||||
sample_rate = 16000
|
||||
|
||||
audio_length = int(audio_duration * sample_rate)
|
||||
|
||||
mm_data: MultiModalDataDict = {
|
||||
"image": self._get_dummy_images(
|
||||
width=image_width,
|
||||
height=image_height,
|
||||
num_images=num_images,
|
||||
),
|
||||
"video": self._get_dummy_videos(
|
||||
width=video_width,
|
||||
height=video_height,
|
||||
num_frames=num_frames,
|
||||
num_videos=num_videos,
|
||||
),
|
||||
"audio": [(np.random.randn(audio_length).astype(np.float32), sample_rate) for _ in range(num_audios)],
|
||||
}
|
||||
|
||||
return mm_data
|
||||
|
||||
|
||||
class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashOmniThinkerProcessingInfo]):
|
||||
"""Multimodal processor for Ming-flash-omni Thinker stage.
|
||||
|
||||
Handles preprocessing of 1) image, 2) video, and 3) audio inputs,
|
||||
and expands placeholder tokens to the correct number of patch tokens.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
# might want to add a fallback to resolve token ids
|
||||
# vocab = tokenizer.get_vocab()
|
||||
thinker_config = self.info.get_hf_config()
|
||||
|
||||
# patch/delimiter token IDs (used in replacement sequences)
|
||||
image_start_token_id = thinker_config.llm_config.image_start_token
|
||||
image_patch_token_id = thinker_config.llm_config.image_patch_token
|
||||
image_end_token_id = thinker_config.llm_config.image_end_token
|
||||
|
||||
video_start_token_id = thinker_config.llm_config.video_start_token
|
||||
frame_patch_token_id = thinker_config.llm_config.video_patch_token
|
||||
video_end_token_id = thinker_config.llm_config.video_end_token
|
||||
|
||||
audio_start_token_id = thinker_config.llm_config.audio_start_token
|
||||
audio_patch_token_id = thinker_config.llm_config.audio_patch_token
|
||||
audio_end_token_id = thinker_config.llm_config.audio_end_token
|
||||
|
||||
vision_config = thinker_config.vision_config
|
||||
spatial_merge_size = vision_config.spatial_merge_size if vision_config else 2
|
||||
|
||||
newline_token_ids: list[int] = tokenizer.encode("\n", add_special_tokens=False)
|
||||
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
|
||||
def get_replacement_image(item_idx: int) -> PromptUpdateDetails:
|
||||
"""Generate token sequence for an image."""
|
||||
grid_thw = out_mm_data.get("image_grid_thw")
|
||||
if grid_thw is None:
|
||||
raise ValueError(
|
||||
"image_grid_thw missing from processor output; "
|
||||
"cannot determine image patch count for prompt replacement."
|
||||
)
|
||||
if isinstance(grid_thw, torch.Tensor):
|
||||
thw = grid_thw[item_idx]
|
||||
num_patches = int(thw.prod().item()) // (spatial_merge_size**2)
|
||||
else:
|
||||
thw = grid_thw[item_idx]
|
||||
num_patches = (thw[0] * thw[1] * thw[2]) // (spatial_merge_size**2)
|
||||
|
||||
# Build token sequence: <image> <imagePatch>*N </image> \n
|
||||
# the newline token is added in purpose from original model processing
|
||||
tokens: list[int] = []
|
||||
tokens.append(image_start_token_id)
|
||||
tokens.extend([image_patch_token_id] * num_patches)
|
||||
tokens.append(image_end_token_id)
|
||||
# Refer to Ming's BailingMM2Processor._expand_image_tokens
|
||||
# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/processing_bailingmm2.py
|
||||
tokens.extend(newline_token_ids)
|
||||
|
||||
# Only <imagePatch> tokens receive multimodal embeddings
|
||||
return PromptUpdateDetails.select_token_id(tokens, image_patch_token_id)
|
||||
|
||||
def get_replacement_video(item_idx: int) -> PromptUpdateDetails:
|
||||
"""Generate token sequence for a video."""
|
||||
grid_thw = out_mm_data.get("video_grid_thw", None)
|
||||
if grid_thw is None:
|
||||
raise ValueError(
|
||||
"video_grid_thw missing from processor output; "
|
||||
"cannot determine video patch count for prompt replacement."
|
||||
)
|
||||
if isinstance(grid_thw, torch.Tensor):
|
||||
thw = grid_thw[item_idx]
|
||||
num_patches = int(thw.prod().item()) // (spatial_merge_size**2)
|
||||
else:
|
||||
thw = grid_thw[item_idx]
|
||||
num_patches = (thw[0] * thw[1] * thw[2]) // (spatial_merge_size**2)
|
||||
|
||||
# Build token sequence: <video> <framePatch>*N </video> \n
|
||||
# the newline token is added in purpose from original model processing
|
||||
tokens: list[int] = []
|
||||
tokens.append(video_start_token_id)
|
||||
tokens.extend([frame_patch_token_id] * num_patches)
|
||||
tokens.append(video_end_token_id)
|
||||
tokens.extend(newline_token_ids)
|
||||
|
||||
# Only <framePatch> tokens receive multimodal embeddings
|
||||
return PromptUpdateDetails.select_token_id(tokens, frame_patch_token_id)
|
||||
|
||||
def get_replacement_audio(item_idx: int) -> PromptUpdateDetails:
|
||||
"""Generate token sequence for an audio."""
|
||||
encoder_feats_lengths = out_mm_data.get("encoder_feats_lengths", None)
|
||||
if encoder_feats_lengths is None:
|
||||
raise ValueError(
|
||||
"encoder_feats_lengths missing from processor output; "
|
||||
"cannot determine audio patch count for prompt replacement."
|
||||
)
|
||||
if isinstance(encoder_feats_lengths, torch.Tensor):
|
||||
num_patches = int(encoder_feats_lengths[item_idx].item())
|
||||
else:
|
||||
num_patches = encoder_feats_lengths[item_idx]
|
||||
|
||||
# Build token sequence: <audio> <audioPatch>*N </audio>
|
||||
tokens: list[int] = []
|
||||
tokens.append(audio_start_token_id)
|
||||
tokens.extend([audio_patch_token_id] * num_patches)
|
||||
tokens.append(audio_end_token_id)
|
||||
|
||||
# Only <audioPatch> tokens receive multimodal embeddings
|
||||
return PromptUpdateDetails.select_token_id(tokens, audio_patch_token_id)
|
||||
|
||||
# Build prompt updates and process replacement
|
||||
updates: list[PromptUpdate] = []
|
||||
|
||||
if "image" in mm_items and mm_items.get_items("image", ImageProcessorItems):
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=PLACEHOLDER_IMAGE_TOKEN_IN_TEXT,
|
||||
replacement=get_replacement_image,
|
||||
)
|
||||
)
|
||||
if "video" in mm_items and mm_items.get_items("video", VideoProcessorItems):
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=PLACEHOLDER_VIDEO_TOKEN_IN_TEXT,
|
||||
replacement=get_replacement_video,
|
||||
)
|
||||
)
|
||||
if "audio" in mm_items and mm_items.get_items("audio", AudioProcessorItems):
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=PLACEHOLDER_AUDIO_TOKEN_IN_TEXT,
|
||||
replacement=get_replacement_audio,
|
||||
)
|
||||
)
|
||||
return updates
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
config: dict[str, MultiModalFieldConfig] = {}
|
||||
|
||||
# Image fields, pixel_values is flat (concatenated patches from all images)
|
||||
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
|
||||
if "pixel_values" in hf_inputs:
|
||||
image_sizes = image_grid_thw.prod(-1)
|
||||
config["pixel_values"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"image",
|
||||
image_sizes,
|
||||
)
|
||||
if "image_grid_thw" in hf_inputs:
|
||||
config["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
|
||||
# Video fields, same flat layout as images
|
||||
video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
|
||||
if "pixel_values_videos" in hf_inputs:
|
||||
video_sizes = video_grid_thw.prod(-1)
|
||||
config["pixel_values_videos"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"video",
|
||||
video_sizes,
|
||||
)
|
||||
if "video_grid_thw" in hf_inputs:
|
||||
config["video_grid_thw"] = MultiModalFieldConfig.batched("video")
|
||||
|
||||
# Audio fields
|
||||
if "audio_feats" in hf_inputs:
|
||||
config["audio_feats"] = MultiModalFieldConfig.batched("audio")
|
||||
if "audio_feats_lengths" in hf_inputs:
|
||||
config["audio_feats_lengths"] = MultiModalFieldConfig.batched("audio")
|
||||
if "encoder_feats_lengths" in hf_inputs:
|
||||
config["encoder_feats_lengths"] = MultiModalFieldConfig.batched("audio")
|
||||
if "placeholder_audio_loc_lens" in hf_inputs:
|
||||
config["placeholder_audio_loc_lens"] = MultiModalFieldConfig.batched("audio")
|
||||
|
||||
return config
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call sub-processors for multimodal inputs and tokenize.
|
||||
|
||||
We call the image/audio sub-processors directly (instead of going
|
||||
through `MingFlashOmniProcessor.__call__`) so that the high-level
|
||||
placeholder tokens remain **unexpanded** in the tokenized output.
|
||||
"""
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
data: dict[str, object] = {}
|
||||
|
||||
images = mm_data.get("images", None)
|
||||
if images is not None:
|
||||
image_outputs = hf_processor.image_processor(
|
||||
images=images,
|
||||
videos=None,
|
||||
return_tensors="pt",
|
||||
)
|
||||
data.update(image_outputs)
|
||||
|
||||
videos = mm_data.get("videos", None)
|
||||
if videos is not None:
|
||||
video_outputs = hf_processor.image_processor(
|
||||
images=None,
|
||||
videos=videos,
|
||||
return_tensors="pt",
|
||||
)
|
||||
# Rename keys to distinguish from images
|
||||
if "pixel_values" in video_outputs:
|
||||
video_outputs["pixel_values_videos"] = video_outputs.pop("pixel_values")
|
||||
if "image_grid_thw" in video_outputs:
|
||||
video_outputs["video_grid_thw"] = video_outputs.pop("image_grid_thw")
|
||||
data.update(video_outputs)
|
||||
|
||||
audios = mm_data.get("audios", None)
|
||||
if audios is not None:
|
||||
# vLLM's AudioProcessorItems provides raw numpy arrays (already resampled).
|
||||
# MingWhisperAudioProcessor expects (waveform, sr) tuples,
|
||||
# so wrap them with the target sample rate.
|
||||
target_sr = hf_processor.audio_processor.sampling_rate
|
||||
audio_tuples = [(a, target_sr) if not isinstance(a, tuple) else a for a in audios]
|
||||
|
||||
audio_outputs = hf_processor.audio_processor(
|
||||
audio_tuples,
|
||||
return_tensors="pt",
|
||||
)
|
||||
data.update(audio_outputs)
|
||||
|
||||
# Tokenize text with placeholders still intact
|
||||
text_outputs = tokenizer(prompt, return_tensors="pt", **tok_kwargs)
|
||||
data.update(text_outputs)
|
||||
|
||||
return BatchFeature(data=data)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MingFlashOmniThinkerMultiModalProcessor,
|
||||
info=MingFlashOmniThinkerProcessingInfo,
|
||||
dummy_inputs=MingFlashOmniThinkerDummyInputsBuilder,
|
||||
)
|
||||
class MingFlashOmniThinkerForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
CustomProcessMixin,
|
||||
):
|
||||
"""Ming Thinker stage: multimodal understanding
|
||||
(text + image + video + audio) -> text generation.
|
||||
"""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={"model.": "language_model."},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
# vllm_omni/transformers_utils/processors/ming.py
|
||||
if modality.startswith("image"):
|
||||
return "<IMAGE>"
|
||||
elif modality.startswith("video"):
|
||||
return "<VIDEO>"
|
||||
elif modality.startswith("audio"):
|
||||
return "<AUDIO>"
|
||||
|
||||
raise ValueError("Only image, video, or audio modality is supported")
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
thinker_config: BailingMM2Config = config
|
||||
if (
|
||||
thinker_config.llm_config is None
|
||||
or thinker_config.vision_config is None
|
||||
or thinker_config.audio_config is None
|
||||
):
|
||||
raise ValueError(
|
||||
"MingFlashOmniThinker requires `llm_config`, `vision_config`, and `audio_config` in `thinker_config`."
|
||||
)
|
||||
|
||||
llm_config = thinker_config.llm_config
|
||||
|
||||
self.config = llm_config
|
||||
self.thinker_config = thinker_config
|
||||
self.have_multimodal_outputs = True
|
||||
|
||||
# Initialize LLM as a component
|
||||
with self._mark_language_model(vllm_config):
|
||||
llm_vllm_config = vllm_config.with_hf_config(llm_config)
|
||||
self.language_model = BailingMoeV2ForCausalLM(
|
||||
vllm_config=llm_vllm_config, prefix=maybe_prefix(prefix, "llm")
|
||||
)
|
||||
|
||||
# Ming thinker is inherently multimodal; initialize both towers eagerly.
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
self.vision = MingVisionEncoder(
|
||||
vision_config=thinker_config.vision_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "vision"),
|
||||
)
|
||||
self.linear_proj = VisionProjector(
|
||||
vision_dim=self.vision.image_emb_dim,
|
||||
llm_dim=llm_config.hidden_size,
|
||||
mlp_depth=getattr(thinker_config, "mlp_depth", 2),
|
||||
)
|
||||
logger.info("Initialized MingVisionEncoder and VisionProjector")
|
||||
|
||||
audio_cfg = thinker_config.audio_config
|
||||
whisper_cfg = getattr(audio_cfg, "whisper_encoder_config", {}) or {}
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
self.audio = WhisperAudioEncoder(
|
||||
**whisper_cfg,
|
||||
use_flash_attn=True,
|
||||
)
|
||||
self.linear_proj_audio = AudioProjector(
|
||||
audio_dim=self.audio.audio_emb_dim,
|
||||
llm_dim=llm_config.hidden_size,
|
||||
ds_kernel_size=getattr(audio_cfg, "ds_kernel_size", 3),
|
||||
ds_stride=getattr(audio_cfg, "ds_stride", 2),
|
||||
mlp_depth=getattr(thinker_config, "mlp_depth", 1),
|
||||
)
|
||||
logger.info("Initialized WhisperAudioEncoder and AudioProjector")
|
||||
|
||||
# Expose interfaces
|
||||
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
|
||||
|
||||
logger.info("MingFlashOmniThinker initialized with vision and audio towers")
|
||||
|
||||
def extract_image_feature(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor:
|
||||
"""Extract and project image features.
|
||||
|
||||
Args:
|
||||
pixel_values: Flattened pixel values from vision processor.
|
||||
grid_thw: [num_images, 3] tensor of (t, h, w) grid dimensions.
|
||||
|
||||
Returns:
|
||||
[seq_len, hidden_size] L2-normalized image embeddings.
|
||||
"""
|
||||
if self.vision is None:
|
||||
raise ValueError("Vision encoder not initialized")
|
||||
|
||||
with torch.amp.autocast(pixel_values.device.type, dtype=torch.bfloat16):
|
||||
image_embeds = self.vision(pixel_values, grid_thw=grid_thw)
|
||||
|
||||
if self.vision.use_deepstack:
|
||||
image_embeds = image_embeds[:, : self.vision.image_emb_dim]
|
||||
|
||||
image_embeds = self.linear_proj(image_embeds)
|
||||
image_embeds = F.normalize(image_embeds, dim=-1)
|
||||
return image_embeds
|
||||
|
||||
def extract_audio_feature(
|
||||
self, audio_feats: torch.Tensor, audio_feats_lengths: torch.Tensor
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Extract and project audio features.
|
||||
|
||||
Args:
|
||||
audio_feats: [B, L_total, n_mels] wrapped mel features — multiple audio
|
||||
clips per batch item are concatenated along the time dimension
|
||||
(time-first, as produced by MingWhisperFeatureExtractor).
|
||||
audio_feats_lengths: [B, N] lengths of each audio clip per batch item.
|
||||
N is the max number of clips per item; zero-padded entries are skipped.
|
||||
|
||||
Returns:
|
||||
Tuple of per-clip [T'_i, hidden_size] projected audio embeddings.
|
||||
"""
|
||||
if self.audio is None:
|
||||
raise ValueError("Audio encoder not initialized")
|
||||
|
||||
# Unwrap packed [B, L_total, n_mels] into a list of [n_mels, T'_i] tensors,
|
||||
# one per audio clip, as expected by WhisperAudioEncoder.
|
||||
x_list: list[torch.Tensor] = []
|
||||
audio_lens: list[int] = []
|
||||
for i in range(audio_feats_lengths.shape[0]):
|
||||
feat_index = 0
|
||||
for j in range(audio_feats_lengths.shape[1]):
|
||||
feat_len = int(audio_feats_lengths[i, j].item())
|
||||
if feat_len == 0:
|
||||
break
|
||||
mel_seg = audio_feats[i, feat_index : feat_index + feat_len].transpose(0, 1)
|
||||
x_list.append(mel_seg)
|
||||
audio_lens.append(feat_len)
|
||||
feat_index += feat_len
|
||||
|
||||
audio_packed = self.audio(x_list, audio_lens)
|
||||
|
||||
# Compute per-clip lengths after Whisper Conv1d (kernel=3, stride=2, pad=1)
|
||||
encoded_lens = [(audio_len - 3 + 2) // 2 + 1 for audio_len in audio_lens]
|
||||
|
||||
# Project packed
|
||||
proj_packed, proj_lens = self.linear_proj_audio.forward_packed(audio_packed, encoded_lens)
|
||||
|
||||
normalize = getattr(self.thinker_config.audio_config, "norm_query_embeds", False)
|
||||
if normalize:
|
||||
proj_packed = F.normalize(proj_packed, dim=-1)
|
||||
|
||||
proj_packed = proj_packed.to(audio_feats.dtype)
|
||||
|
||||
# Split into per-clip tensors
|
||||
return proj_packed.split(proj_lens)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
"""Parse and validate multimodal kwargs into per-modality dicts."""
|
||||
mm_input_by_modality: dict[str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | MingAudioInput] = {}
|
||||
|
||||
for key in kwargs:
|
||||
if key == "pixel_values" and "image" not in mm_input_by_modality:
|
||||
pixel_values = kwargs.get("pixel_values")
|
||||
image_grid_thw = kwargs.get("image_grid_thw")
|
||||
if pixel_values is not None and image_grid_thw is not None:
|
||||
mm_input_by_modality["image"] = Qwen2_5_VLImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values, # type: ignore[arg-type]
|
||||
image_grid_thw=image_grid_thw, # type: ignore[arg-type]
|
||||
)
|
||||
elif key == "pixel_values_videos" and "video" not in mm_input_by_modality:
|
||||
pixel_values_videos = kwargs.get("pixel_values_videos")
|
||||
video_grid_thw = kwargs.get("video_grid_thw")
|
||||
second_per_grid_ts = kwargs.get("second_per_grid_ts")
|
||||
if pixel_values_videos is not None and video_grid_thw is not None:
|
||||
mm_input_by_modality["video"] = Qwen2_5_VLVideoPixelInputs(
|
||||
type="pixel_values_videos",
|
||||
pixel_values_videos=pixel_values_videos, # type: ignore[arg-type]
|
||||
video_grid_thw=video_grid_thw, # type: ignore[arg-type]
|
||||
second_per_grid_ts=second_per_grid_ts, # type: ignore[arg-type]
|
||||
)
|
||||
elif key == "audio_feats" and "audio" not in mm_input_by_modality:
|
||||
audio_feats = kwargs.get("audio_feats")
|
||||
audio_feats_lengths = kwargs.get("audio_feats_lengths")
|
||||
if audio_feats is not None and audio_feats_lengths is not None:
|
||||
mm_input_by_modality["audio"] = MingAudioInput(
|
||||
audio_feats=audio_feats, # type: ignore[arg-type]
|
||||
audio_feats_lengths=audio_feats_lengths, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
return mm_input_by_modality
|
||||
|
||||
def _process_image_input(self, image_input: Qwen2_5_VLImageInputs) -> list[torch.Tensor]:
|
||||
# Splits the flat [total_tokens, D] output of extract_image_feature
|
||||
# into one tensor per image.
|
||||
pixel_values = image_input["pixel_values"]
|
||||
image_grid_thw = image_input["image_grid_thw"]
|
||||
image_embeds = self.extract_image_feature(pixel_values, image_grid_thw)
|
||||
merge_unit = self.thinker_config.vision_config.spatial_merge_size**2
|
||||
sizes = (image_grid_thw.prod(dim=-1) // merge_unit).tolist()
|
||||
return list(image_embeds.split([int(s) for s in sizes], dim=0))
|
||||
|
||||
def _process_video_input(self, video_input: Qwen2_5_VLVideoInputs) -> list[torch.Tensor]:
|
||||
pixel_values_videos = video_input["pixel_values_videos"]
|
||||
video_grid_thw = video_input["video_grid_thw"]
|
||||
video_embeds = self.extract_image_feature(pixel_values_videos, video_grid_thw)
|
||||
merge_unit = self.thinker_config.vision_config.spatial_merge_size**2
|
||||
sizes = (video_grid_thw.prod(dim=-1) // merge_unit).tolist()
|
||||
return list(video_embeds.split([int(s) for s in sizes], dim=0))
|
||||
|
||||
def _process_audio_input(self, audio_input: MingAudioInput) -> list[torch.Tensor]:
|
||||
return list(self.extract_audio_feature(audio_input["audio_feats"], audio_input["audio_feats_lengths"]))
|
||||
|
||||
def _compute_modality_masks(self, input_ids: torch.Tensor) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
"""Compute vision and audio MoE-routing masks from input_ids.
|
||||
|
||||
Returns:
|
||||
Tuple of (vision_mask, audio_mask), each shape [seq_len] bool.
|
||||
"""
|
||||
llm_config = self.config
|
||||
|
||||
# vision mask
|
||||
vision_mask = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
image_token = llm_config.image_patch_token
|
||||
video_token = llm_config.video_patch_token
|
||||
vision_mask = vision_mask | (input_ids == image_token)
|
||||
vision_mask = vision_mask | (input_ids == video_token)
|
||||
|
||||
# audio mask
|
||||
audio_token = llm_config.audio_patch_token
|
||||
audio_mask = input_ids == audio_token
|
||||
|
||||
return vision_mask, audio_mask
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
if not mm_input_by_modality:
|
||||
return []
|
||||
|
||||
# preserve the order of modalities
|
||||
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
||||
|
||||
for modality, mm_input in mm_input_by_modality.items():
|
||||
if modality == "image":
|
||||
multimodal_embeddings += tuple(self._process_image_input(mm_input)) # type: ignore[arg-type]
|
||||
elif modality == "video":
|
||||
multimodal_embeddings += tuple(self._process_video_input(mm_input)) # type: ignore[arg-type]
|
||||
elif modality == "audio":
|
||||
multimodal_embeddings += tuple(self._process_audio_input(mm_input)) # type: ignore[arg-type]
|
||||
|
||||
return multimodal_embeddings
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.model.word_embeddings(input_ids)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
|
||||
assert is_multimodal is not None, "`is_multimodal` mask required when `multimodal_embeddings` provided"
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> OmniOutput:
|
||||
# Compute MoE modality masks on every device
|
||||
image_mask, audio_mask = self._compute_modality_masks(input_ids)
|
||||
hidden_states = self.language_model.forward(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
image_mask=image_mask,
|
||||
audio_mask=audio_mask,
|
||||
)
|
||||
|
||||
# Capture embeddings for downstream stages
|
||||
multimodal_outputs = {
|
||||
"final_hidden_states": hidden_states,
|
||||
}
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=hidden_states,
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states, sampling_metadata)
|
||||
|
||||
def sample(self, logits: torch.Tensor, sampling_metadata):
|
||||
return self.language_model.sample(logits, sampling_metadata)
|
||||
|
||||
@property
|
||||
def sampler(self):
|
||||
return self.language_model.sampler
|
||||
|
||||
def iter_mm_features(
|
||||
self,
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
) -> Iterator[tuple[int, str, dict[str, Any]]]:
|
||||
"""Iterate over image/video features sorted by token position.
|
||||
|
||||
Yields: (offset, modality, feature_data) where feature_data contains:
|
||||
- image: {"grid_t", "grid_h", "grid_w", "second_per_grid_t"}
|
||||
- video: {"grid_t", "grid_h", "grid_w", "second_per_grid_t"}
|
||||
|
||||
Audio features are not yielded: Ming assigns them sequential
|
||||
text positions (same T/H/W value) rather than 3D grid positions.
|
||||
"""
|
||||
spatial_merge_size = self.config.spatial_merge_size
|
||||
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
if mm_feature.data is None:
|
||||
continue
|
||||
|
||||
offset = mm_feature.mm_position.offset
|
||||
modality = mm_feature.modality
|
||||
|
||||
if modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
yield (
|
||||
offset,
|
||||
"image",
|
||||
{
|
||||
"grid_t": int(t),
|
||||
"grid_h": int(h) // spatial_merge_size,
|
||||
"grid_w": int(w) // spatial_merge_size,
|
||||
"second_per_grid_t": 0.0,
|
||||
},
|
||||
)
|
||||
elif modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
second_per_grid_t = 1.0
|
||||
spgt_field = mm_feature.data.get("second_per_grid_ts")
|
||||
if spgt_field is not None:
|
||||
second_per_grid_t = float(spgt_field.data.item())
|
||||
yield (
|
||||
offset,
|
||||
"video",
|
||||
{
|
||||
"grid_t": int(t),
|
||||
"grid_h": int(h) // spatial_merge_size,
|
||||
"grid_w": int(w) // spatial_merge_size,
|
||||
"second_per_grid_t": second_per_grid_t,
|
||||
},
|
||||
)
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec] | None = None,
|
||||
**kwargs: object,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Compute M-RoPE input positions using mm_features directly."""
|
||||
llm_config = self.config
|
||||
tokens_per_second: int = getattr(llm_config, "tokens_per_second", 2)
|
||||
seq_len = len(input_tokens)
|
||||
|
||||
llm_pos_ids_list: list[np.ndarray] = []
|
||||
st = 0 # index of next unprocessed token
|
||||
|
||||
for patch_offset, _modality, data in self.iter_mm_features(mm_features or []):
|
||||
text_len = patch_offset - st
|
||||
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
|
||||
if text_len > 0:
|
||||
llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx)
|
||||
st_idx += text_len
|
||||
|
||||
# 3-D grid positions for patch tokens
|
||||
grid_t: int = data["grid_t"]
|
||||
grid_h: int = data["grid_h"]
|
||||
grid_w: int = data["grid_w"]
|
||||
second_per_grid_t: float = data["second_per_grid_t"]
|
||||
|
||||
t_raw = np.arange(grid_t)
|
||||
if second_per_grid_t > 0:
|
||||
t_index = (t_raw * second_per_grid_t * tokens_per_second).astype(np.int64)
|
||||
else:
|
||||
t_index = t_raw.astype(np.int64)
|
||||
t_index = np.repeat(t_index, grid_h * grid_w)
|
||||
|
||||
h_index = np.tile(np.arange(grid_h).repeat(grid_w), grid_t)
|
||||
w_index = np.tile(np.arange(grid_w), grid_t * grid_h)
|
||||
|
||||
llm_pos_ids_list.append(np.stack([t_index, h_index, w_index]) + st_idx)
|
||||
|
||||
num_patches = grid_t * grid_h * grid_w
|
||||
st = patch_offset + num_patches
|
||||
|
||||
if st < seq_len:
|
||||
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
|
||||
tail_len = seq_len - st
|
||||
llm_pos_ids_list.append(np.broadcast_to(np.arange(tail_len), (3, tail_len)) + st_idx)
|
||||
|
||||
if llm_pos_ids_list:
|
||||
position_ids = torch.from_numpy(np.concatenate(llm_pos_ids_list, axis=1).astype(np.int64)) # (3, seq_len)
|
||||
else:
|
||||
# text-only, simple sequential positions
|
||||
position_ids = torch.arange(seq_len, dtype=torch.long).unsqueeze(0).expand(3, -1)
|
||||
|
||||
mrope_position_delta = int(position_ids.max().item()) + 1 - seq_len
|
||||
return position_ids, mrope_position_delta
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -0,0 +1,896 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright 2023 Antgroup and The HuggingFace Inc. team. All rights reserved.
|
||||
# Adapted from Ming
|
||||
# https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, SharedFusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead, VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMoeV2Config
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MingVideoRopeMRotaryEmbedding(MRotaryEmbedding):
|
||||
"""MRotaryEmbedding with Ming's video_rope cos/sin interleaving.
|
||||
|
||||
Unlike standard mrope which maps contiguous frequency sections to T/H/W,
|
||||
video_rope alternates H/W frequencies element-wise in the spatial section
|
||||
and places temporal frequencies at the end:
|
||||
Standard mrope: [T T T ... H H H ... W W W ...]
|
||||
Video rope: [H W H W ... H W ... T T T ...]
|
||||
|
||||
Refer to Ming's BailingMoeV2RotaryEmbedding3D
|
||||
https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py#L174
|
||||
"""
|
||||
|
||||
def _remap_video_rope(
|
||||
self,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Remap 3D cos/sin to video_rope interleaved layout.
|
||||
|
||||
Args:
|
||||
cos, sin: [3, num_tokens, rotary_dim // 2]
|
||||
Returns:
|
||||
cos, sin: [num_tokens, rotary_dim // 2]
|
||||
|
||||
Refer to Ming's apply_3d_rotary_pos_emb
|
||||
https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py#L226
|
||||
"""
|
||||
assert self.mrope_section is not None
|
||||
hw_size = self.mrope_section[1] + self.mrope_section[2]
|
||||
|
||||
result_cos = torch.empty_like(cos[0])
|
||||
result_sin = torch.empty_like(sin[0])
|
||||
|
||||
# Spatial frequencies: even indices from H (dim 1), odd from W (dim 2)
|
||||
result_cos[:, 0:hw_size:2] = cos[1, :, 0:hw_size:2]
|
||||
result_cos[:, 1:hw_size:2] = cos[2, :, 1:hw_size:2]
|
||||
result_sin[:, 0:hw_size:2] = sin[1, :, 0:hw_size:2]
|
||||
result_sin[:, 1:hw_size:2] = sin[2, :, 1:hw_size:2]
|
||||
|
||||
# Temporal frequencies at the end
|
||||
result_cos[:, hw_size:] = cos[0, :, hw_size:]
|
||||
result_sin[:, hw_size:] = sin[0, :, hw_size:]
|
||||
|
||||
return result_cos, result_sin
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert positions.ndim == 1 or positions.ndim == 2
|
||||
assert key is not None
|
||||
|
||||
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
|
||||
num_tokens = positions.shape[-1]
|
||||
cos_sin = cos_sin_cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
|
||||
if positions.ndim == 2:
|
||||
cos, sin = self._remap_video_rope(cos, sin)
|
||||
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
query_rot = self.apply_rotary_emb.forward_native(query_rot, cos, sin)
|
||||
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
|
||||
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
key_rot = self.apply_rotary_emb.forward_native(key_rot, cos, sin)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
|
||||
return query, key
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# No custom Triton kernel for video_rope; fall back to native for 3D
|
||||
# TODO: Consider custom optimization
|
||||
if positions.ndim == 2:
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
return super().forward_cuda(positions, query, key, offsets)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
|
||||
|
||||
class BailingMoeV2MLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: BailingMoeV2Config,
|
||||
intermediate_size: int,
|
||||
hidden_act: str = "silu",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
self.intermediate_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(f"Unsupported activation: {hidden_act}")
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class BailingMoeV2Gate(nn.Module):
|
||||
"""MoE routing gate with grouped expert selection."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: BailingMoeV2Config,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.num_experts = config.num_experts
|
||||
|
||||
self.n_group = config.n_group
|
||||
self.topk_group = config.topk_group
|
||||
|
||||
self.gating_dim = config.hidden_size
|
||||
|
||||
self.gate = ReplicatedLinear(
|
||||
self.gating_dim,
|
||||
self.num_experts,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
|
||||
self.expert_bias = nn.Parameter(torch.zeros(self.num_experts), requires_grad=False)
|
||||
|
||||
def group_limited_topk(self, scores: torch.Tensor):
|
||||
"""Group-limited top-k selection for expert routing."""
|
||||
num_tokens, _ = scores.size()
|
||||
# Organize experts into groups
|
||||
group_scores = scores.view(num_tokens, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
|
||||
group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
|
||||
group_mask = torch.zeros_like(group_scores)
|
||||
group_mask.scatter_(1, group_idx, 1)
|
||||
|
||||
# Mask experts based on selected groups
|
||||
score_mask = (
|
||||
group_mask.unsqueeze(-1)
|
||||
.expand(num_tokens, self.n_group, self.num_experts // self.n_group)
|
||||
.reshape(num_tokens, -1)
|
||||
)
|
||||
|
||||
masked_scores = scores.masked_fill(~score_mask.bool(), float("-inf"))
|
||||
probs, top_indices = torch.topk(masked_scores, k=self.top_k, dim=-1, sorted=False)
|
||||
|
||||
return probs, top_indices
|
||||
|
||||
def forward(self, hidden_states):
|
||||
# compute gating score
|
||||
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
logits, _ = self.gate(hidden_states)
|
||||
|
||||
logits = logits.float()
|
||||
scores = torch.sigmoid(logits)
|
||||
|
||||
scores_for_routing = scores + self.expert_bias
|
||||
_, topk_idx = self.group_limited_topk(scores_for_routing)
|
||||
|
||||
scores = torch.gather(scores, dim=1, index=topk_idx).type_as(logits)
|
||||
|
||||
topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores
|
||||
topk_weight = topk_weight * self.routed_scaling_factor
|
||||
|
||||
return topk_idx, topk_weight, logits
|
||||
|
||||
|
||||
def _unpack_multi_routing(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Stateless routing function that unpacks pre-computed routing results.
|
||||
|
||||
Used as `custom_routing_function` for `FusedMoE`. The caller is expected
|
||||
to pack (topk_weight, topk_idx) into `gating_output` before
|
||||
calling FusedMoE.forward(), and this function unpacks them.
|
||||
|
||||
Args:
|
||||
gating_output: [num_tokens, top_k * 2]
|
||||
- [:, :top_k], topk_weight (float)
|
||||
- [:, top_k:], topk_idx (float, cast back to int)
|
||||
"""
|
||||
topk_weight = gating_output[:, :topk].contiguous()
|
||||
topk_idx = gating_output[:, topk:]
|
||||
return topk_weight.to(torch.float32), topk_idx.to(torch.int32)
|
||||
|
||||
|
||||
class BailingMoeV2SparseMoeBlock(nn.Module):
|
||||
"""Sparse MoE block with MultiRouter support for multimodal routing.
|
||||
|
||||
Keep the custom multi-router gating logic external.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: BailingMoeV2Config,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.num_experts_per_tok = config.num_experts_per_tok
|
||||
|
||||
if isinstance(self.config.num_shared_experts, int) and self.config.num_shared_experts > 0:
|
||||
self.shared_experts = BailingMoeV2MLP(
|
||||
config=self.config,
|
||||
intermediate_size=self.config.moe_intermediate_size * self.config.num_shared_experts,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = SharedFusedMoE(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
custom_routing_function=_unpack_multi_routing,
|
||||
renormalize=False, # we handle normalization in the gate
|
||||
reduce_results=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
|
||||
self.experts.expert_mapping = FusedMoE.make_expert_params_mapping(
|
||||
self.experts,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=config.num_experts,
|
||||
)
|
||||
|
||||
self.router_type = self.config.router_type
|
||||
if self.router_type == "topN":
|
||||
self.gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.gate")
|
||||
elif self.router_type == "MultiRouter":
|
||||
self.gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.gate")
|
||||
self.image_gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.image_gate")
|
||||
self.audio_gate = BailingMoeV2Gate(self.config, quant_config, prefix=f"{prefix}.audio_gate")
|
||||
else:
|
||||
raise ValueError(f"Unsupported router_type: {self.router_type}")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_mask(
|
||||
mask: torch.Tensor,
|
||||
bsz: int,
|
||||
seq_len: int,
|
||||
name: str,
|
||||
) -> torch.Tensor:
|
||||
"""Validate and reshape a modality mask to [bsz*seq_len, 1] bool."""
|
||||
N = bsz * seq_len
|
||||
if mask.ndim == 1:
|
||||
# vLLM path: flat tokens [N]
|
||||
assert mask.shape[0] == N, f"{name} length {mask.shape[0]} != N={N}"
|
||||
elif mask.ndim == 2:
|
||||
assert mask.shape == (bsz, seq_len), f"{name} shape {mask.shape} != ({bsz}, {seq_len})"
|
||||
elif mask.ndim == 3:
|
||||
assert mask.shape == (bsz, seq_len, 1), f"{name} shape {mask.shape} != ({bsz}, {seq_len}, 1)"
|
||||
else:
|
||||
raise ValueError(f"Unsupported {name} shape: {mask.shape}")
|
||||
|
||||
return mask.reshape(N, 1).bool()
|
||||
|
||||
def forward(self, hidden_states, image_mask: torch.Tensor, audio_mask: torch.Tensor):
|
||||
# TODO(yuanheng-zhao): revise the shapes in the flow
|
||||
assert 2 <= hidden_states.dim() <= 3, f"{self.__class__.__name__} only supports 2D or 3D inputs"
|
||||
input_is_2d = hidden_states.ndim == 2
|
||||
if input_is_2d:
|
||||
hidden_states = hidden_states.unsqueeze(0)
|
||||
|
||||
bsz, seq_len, h = hidden_states.shape
|
||||
|
||||
if self.router_type == "MultiRouter":
|
||||
image_mask = self._normalize_mask(image_mask, bsz, seq_len, "image_mask").to(hidden_states.device)
|
||||
audio_mask = self._normalize_mask(audio_mask, bsz, seq_len, "audio_mask").to(hidden_states.device)
|
||||
|
||||
# if image_mask is not None and audio_mask is not None:
|
||||
# assert torch.logical_and(image_mask, audio_mask).sum() == 0
|
||||
|
||||
image_topk_idx, image_topk_weight, _ = self.image_gate(hidden_states)
|
||||
audio_topk_idx, audio_topk_weight, _ = self.audio_gate(hidden_states)
|
||||
topk_idx, topk_weight, _ = self.gate(hidden_states)
|
||||
|
||||
topk_idx = torch.where(image_mask, image_topk_idx, topk_idx)
|
||||
topk_weight = torch.where(image_mask, image_topk_weight, topk_weight)
|
||||
topk_idx = torch.where(audio_mask, audio_topk_idx, topk_idx)
|
||||
topk_weight = torch.where(audio_mask, audio_topk_weight, topk_weight)
|
||||
else:
|
||||
topk_idx, topk_weight, _ = self.gate(hidden_states)
|
||||
|
||||
# Pack pre-computed routing into a single tensor
|
||||
packed_routing = torch.cat(
|
||||
[
|
||||
topk_weight.to(hidden_states.dtype),
|
||||
topk_idx.to(hidden_states.dtype),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
# SharedFusedMoE expects 2D hidden_states
|
||||
hidden_states_2d = hidden_states.view(-1, h)
|
||||
result = self.experts(hidden_states_2d, packed_routing)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
shared_output, fused_out = result
|
||||
else:
|
||||
shared_output, fused_out = None, result
|
||||
|
||||
final_hidden_states = fused_out + shared_output if shared_output is not None else fused_out
|
||||
|
||||
final_hidden_states = final_hidden_states.view(bsz, seq_len, h)
|
||||
|
||||
return final_hidden_states.squeeze(0) if input_is_2d else final_hidden_states
|
||||
|
||||
|
||||
class BailingMoeV2Attention(nn.Module):
|
||||
"""Multi-headed attention using vLLM's Attention layer with 3D RoPE support."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: BailingMoeV2Config,
|
||||
layer_idx: int,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_kv_heads = config.num_key_value_heads
|
||||
self.head_dim = config.head_dim
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
assert self.num_heads % tp_size == 0
|
||||
self.num_heads = self.num_heads // tp_size
|
||||
self.num_kv_heads = max(1, self.num_kv_heads // tp_size)
|
||||
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
partial_rotary_factor = config.partial_rotary_factor
|
||||
self.rope_dim = int(self.head_dim * partial_rotary_factor)
|
||||
|
||||
total_num_heads = config.num_attention_heads
|
||||
total_num_kv_heads = config.num_key_value_heads
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
total_num_heads,
|
||||
total_num_kv_heads,
|
||||
bias=config.use_qkv_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
|
||||
self.dense = RowParallelLinear(
|
||||
total_num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
bias=config.use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.dense",
|
||||
)
|
||||
|
||||
# apply vLLM RMSNorm here rather than BailingMoeV2RMSNorm, diff might exist
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
# 3D Rotary embeddings for multimodal
|
||||
if config.rope_scaling is None:
|
||||
raise ValueError("rope_scaling must not be None")
|
||||
|
||||
rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
|
||||
mrope_section = config.rope_scaling.get("mrope_section", [8, 12, 12])
|
||||
|
||||
if rope_type == "video_rope":
|
||||
# Ming-specific video_rope with custom H/W interleaving
|
||||
self.rotary_emb = MingVideoRopeMRotaryEmbedding(
|
||||
head_size=self.head_dim,
|
||||
rotary_dim=self.rope_dim,
|
||||
max_position_embeddings=config.max_position_embeddings,
|
||||
base=config.rope_theta,
|
||||
is_neox_style=True,
|
||||
dtype=torch.get_default_dtype(),
|
||||
mrope_section=mrope_section,
|
||||
)
|
||||
else:
|
||||
# Standard m_rope (rope_type "3D", "default", or None)
|
||||
rope_scaling = dict(config.rope_scaling)
|
||||
rope_scaling["rope_type"] = "default" # normalize for get_rope dispatch
|
||||
rope_scaling["mrope_section"] = mrope_section
|
||||
self.rotary_emb = get_rope(
|
||||
head_size=self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
is_neox_style=True,
|
||||
rope_parameters={
|
||||
"rope_theta": config.rope_theta,
|
||||
"partial_rotary_factor": config.partial_rotary_factor,
|
||||
**rope_scaling,
|
||||
},
|
||||
)
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass for attention with 3D RoPE.
|
||||
|
||||
Args:
|
||||
positions: Position IDs, shape (3, num_tokens) for 3D rope
|
||||
or (num_tokens,) for text-only
|
||||
hidden_states: Input hidden states, shape (num_tokens, hidden_size)
|
||||
|
||||
Returns:
|
||||
Attention output tensor, shape (num_tokens, hidden_size)
|
||||
"""
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
num_tokens = q.shape[0]
|
||||
q = self.q_norm(q.view(num_tokens, self.num_heads, self.head_dim)).view(num_tokens, self.q_size)
|
||||
k = self.k_norm(k.view(num_tokens, self.num_kv_heads, self.head_dim)).view(num_tokens, self.kv_size)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
output, _ = self.dense(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class BailingMoeV2DecoderLayer(nn.Module):
|
||||
"""Decoder layer with attention and MoE MLP."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: BailingMoeV2Config,
|
||||
layer_idx: int,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
self.attention = BailingMoeV2Attention(
|
||||
config=config,
|
||||
layer_idx=layer_idx,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attention",
|
||||
)
|
||||
|
||||
# MLP or MoE based on layer index
|
||||
if config.num_experts is not None and layer_idx >= config.first_k_dense_replace:
|
||||
self.mlp = BailingMoeV2SparseMoeBlock(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.is_moe = True
|
||||
else:
|
||||
self.mlp = BailingMoeV2MLP(
|
||||
config=config,
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.is_moe = False
|
||||
|
||||
# apply vLLM RMSNorm to replace BailingMoeV2RMSNorm, diff might exist
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
image_mask: torch.Tensor | None = None,
|
||||
audio_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Forward pass for decoder layer.
|
||||
|
||||
Args:
|
||||
positions: Position IDs
|
||||
hidden_states: Input hidden states
|
||||
residual: Residual connection from previous layer
|
||||
image_mask: Mask for image tokens (for MultiRouter MoE)
|
||||
audio_mask: Mask for audio tokens (for MultiRouter MoE)
|
||||
|
||||
Returns:
|
||||
Tuple of (hidden_states, residual)
|
||||
"""
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
hidden_states = self.attention(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
|
||||
if self.is_moe:
|
||||
hidden_states = self.mlp(hidden_states, image_mask, audio_mask)
|
||||
else:
|
||||
# Dense MLP only takes hidden_states (no routing masks)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"image_mask": 0,
|
||||
"audio_mask": 0,
|
||||
}
|
||||
)
|
||||
class BailingMoeV2Model(nn.Module):
|
||||
"""BailingMoeV2 Model adapted from:
|
||||
|
||||
Ming repo BailingMoeV2Model
|
||||
https://github.com/inclusionAI/Ming/blob/2a0c02ae3130190160c215f89fce7de3005db483/modeling_bailing_moe_v2.py
|
||||
vLLM repo BailingMoeModel
|
||||
https://github.com/vllm-project/vllm/blob/7291d1b288558d48508e1a17c37b0aa170332264/vllm/model_executor/models/bailing_moe.py
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# BailingMoeV2Config
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.padding_idx = config.pad_token_id
|
||||
self.vocab_size = config.vocab_size
|
||||
self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False)
|
||||
|
||||
if get_pp_group().is_first_rank or (self.tie_word_embeddings and get_pp_group().is_last_rank):
|
||||
self.word_embeddings = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.word_embeddings",
|
||||
)
|
||||
else:
|
||||
self.word_embeddings = PPMissingLayer()
|
||||
|
||||
# Decoder layers with later pipeline parallelism support
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: BailingMoeV2DecoderLayer(
|
||||
config=config,
|
||||
layer_idx=int(prefix.split(".")[-1]),
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
# apply vLLM RMSNorm to replace BailingMoeV2RMSNorm, diff might exist
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.word_embeddings
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
image_mask: torch.Tensor | None = None,
|
||||
audio_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
for layer in self.layers[self.start_layer : self.end_layer]:
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
residual,
|
||||
image_mask=image_mask,
|
||||
audio_mask=audio_mask,
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states, "residual": residual})
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class BailingMoeV2ForCausalLM(nn.Module, CustomProcessMixin):
|
||||
"""BailingMoeV2 model for causal language modeling, adapted for vLLM.
|
||||
|
||||
Inherits from CustomProcessMixin to support custom preprocessing and postprocessing
|
||||
for integration with omni model pipelines.
|
||||
"""
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
# BailingMoeV2Config
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.model = BailingMoeV2Model(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
)
|
||||
|
||||
self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if self.tie_word_embeddings:
|
||||
self.lm_head.weight = self.model.word_embeddings.weight
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.sampler = Sampler()
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
image_mask: torch.Tensor | None = None,
|
||||
audio_mask: torch.Tensor | None = None,
|
||||
):
|
||||
hidden_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
image_mask=image_mask,
|
||||
audio_mask=audio_mask,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, sampling_metadata)
|
||||
return logits
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata,
|
||||
) -> SamplerOutput | None:
|
||||
next_tokens = self.sampler(logits, sampling_metadata)
|
||||
return next_tokens
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, weight_name, shard_id)
|
||||
# BailingMoE stores fused QKV in checkpoint as query_key_value
|
||||
("qkv_proj", "query_key_value", None),
|
||||
# Dense MLP and shared_experts gate/up are stored separately
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
# Gate router linear layers: checkpoint `{r}.weight` -> model `{r}.gate.weight`
|
||||
gate_name_mapper = WeightsMapper(
|
||||
orig_to_new_substr={f".{r}.weight": f".{r}.gate.weight" for r in ("gate", "image_gate", "audio_gate")}
|
||||
)
|
||||
|
||||
# FusedMoE expert params mapping is identical across all MoE layers
|
||||
expert_params_mapping: list[tuple[str, str, int, str]] = []
|
||||
for layer in self.model.layers:
|
||||
if hasattr(layer, "mlp") and hasattr(layer.mlp, "experts"):
|
||||
expert_params_mapping = layer.mlp.experts.expert_mapping
|
||||
break
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in gate_name_mapper.apply(weights):
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name or "mlp.experts" in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
for param_name, weight_name, expert_id, shard_id in expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, name, shard_id=shard_id, expert_id=expert_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from Ming repository modeling_bailingmm2.py
|
||||
# https://github.com/inclusionAI/Ming
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Transpose(nn.Module):
|
||||
"""Used in nn.Sequential pipelines."""
|
||||
|
||||
def __init__(self, dim0: int, dim1: int):
|
||||
super().__init__()
|
||||
self.dim0 = dim0
|
||||
self.dim1 = dim1
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x.transpose(self.dim0, self.dim1)
|
||||
|
||||
|
||||
class VisionProjector(nn.Module):
|
||||
"""MLP projector from vision encoder output to LLM hidden space.
|
||||
|
||||
Args:
|
||||
vision_dim: Vision encoder output dimension (out_hidden_size).
|
||||
llm_dim: LLM hidden dimension.
|
||||
mlp_depth: Number of linear layers (>= 1).
|
||||
"""
|
||||
|
||||
def __init__(self, vision_dim: int, llm_dim: int, mlp_depth: int = 1):
|
||||
super().__init__()
|
||||
layers: list[nn.Module] = [nn.Linear(vision_dim, llm_dim)]
|
||||
for _ in range(1, mlp_depth):
|
||||
layers.append(nn.GELU())
|
||||
layers.append(nn.Linear(llm_dim, llm_dim))
|
||||
self.proj = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Project vision features.
|
||||
|
||||
Args:
|
||||
x: [seq_len, vision_dim] or [B, seq_len, vision_dim]
|
||||
|
||||
Returns:
|
||||
Projected features with last dim = llm_dim.
|
||||
"""
|
||||
return self.proj(x)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if not name.startswith("proj."):
|
||||
name = f"proj.{name}"
|
||||
if name not in params_dict:
|
||||
logger.warning("Skipping unknown vision projector weight: %s", name)
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class AudioProjector(nn.Module):
|
||||
"""Projector for audio features.
|
||||
|
||||
Args:
|
||||
audio_dim: Audio encoder output dimension (n_state).
|
||||
llm_dim: LLM hidden dimension.
|
||||
ds_kernel_size: Conv1d kernel size for downsampling.
|
||||
ds_stride: Conv1d stride for downsampling.
|
||||
mlp_depth: Total number of projection layers (>= 1).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_dim: int,
|
||||
llm_dim: int,
|
||||
ds_kernel_size: int = 3,
|
||||
ds_stride: int = 2,
|
||||
mlp_depth: int = 1,
|
||||
):
|
||||
super().__init__()
|
||||
self.ds_kernel_size = ds_kernel_size
|
||||
self.ds_stride = ds_stride
|
||||
|
||||
layers: list[nn.Module] = [
|
||||
nn.Conv1d(
|
||||
audio_dim,
|
||||
llm_dim,
|
||||
kernel_size=ds_kernel_size,
|
||||
stride=ds_stride,
|
||||
padding=ds_kernel_size // 2,
|
||||
),
|
||||
Transpose(-1, -2), # [B, llm_dim, T'] -> [B, T', llm_dim]
|
||||
]
|
||||
for _ in range(1, mlp_depth):
|
||||
layers.append(nn.GELU())
|
||||
layers.append(nn.Linear(llm_dim, llm_dim))
|
||||
self.proj = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Project audio features with temporal downsampling.
|
||||
|
||||
Args:
|
||||
x: [B, T, audio_dim] audio encoder output (channel-last).
|
||||
|
||||
Returns:
|
||||
[B, T', llm_dim] projected features (channel-last),
|
||||
where T' = (T - ds_kernel_size + 2*(ds_kernel_size//2)) // ds_stride + 1.
|
||||
"""
|
||||
# Conv1d expects [B, C, T], so transpose input
|
||||
x = x.transpose(-1, -2) # [B, audio_dim, T]
|
||||
return self.proj(x)
|
||||
|
||||
def forward_packed(
|
||||
self,
|
||||
packed: torch.Tensor,
|
||||
encoded_lens: list[int],
|
||||
) -> tuple[torch.Tensor, list[int]]:
|
||||
"""Project packed audio features from the Whisper encoder.
|
||||
|
||||
Args:
|
||||
packed: [total_T', audio_dim] packed encoder output.
|
||||
encoded_lens: Length of each clip after Whisper encoding.
|
||||
|
||||
Returns:
|
||||
Tuple of:
|
||||
- [total_T'', llm_dim] packed projected features.
|
||||
- List of projected lengths per clip.
|
||||
"""
|
||||
conv1d = self.proj[0]
|
||||
mlp = self.proj[2:]
|
||||
|
||||
# Split packed tensor per clip for Conv1d
|
||||
segments = packed.split(encoded_lens)
|
||||
conv_segments = []
|
||||
proj_lens: list[int] = []
|
||||
for seg in segments:
|
||||
out = conv1d(seg.transpose(0, 1).unsqueeze(0)) # [1, llm_dim, T'_i]
|
||||
out = out.squeeze(0).transpose(0, 1) # [T'_i, llm_dim]
|
||||
conv_segments.append(out)
|
||||
proj_lens.append(out.shape[0])
|
||||
|
||||
packed_proj = torch.cat(conv_segments, dim=0) # [total_T'', llm_dim]
|
||||
packed_proj = mlp(packed_proj)
|
||||
return packed_proj, proj_lens
|
||||
|
||||
def compute_output_length(self, input_length: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute output sequence length after Conv1d downsampling.
|
||||
|
||||
Args:
|
||||
input_length: Original mel spectrogram lengths.
|
||||
|
||||
Returns:
|
||||
Output lengths after both convolutions.
|
||||
"""
|
||||
length = (input_length - 3 + 2 * 1) // 2 + 1
|
||||
length = (length - self.ds_kernel_size + 2 * (self.ds_kernel_size // 2)) // self.ds_stride + 1
|
||||
return length
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if not name.startswith("proj."):
|
||||
name = f"proj.{name}"
|
||||
if name not in params_dict:
|
||||
logger.warning("Skipping unknown audio projector weight: %s", name)
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
@@ -0,0 +1,125 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Adapted from Ming repository qwen3_moe_vit.py
|
||||
# https://github.com/inclusionAI/Ming
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3Omni_VisionTransformer,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _adapt_vision_config(vision_config):
|
||||
# Adapt Ming's Qwen3VLMoeVisionConfig to be compatible with vLLM's
|
||||
# Qwen3Omni_VisionTransformer expectations.
|
||||
if not hasattr(vision_config, "image_size") or vision_config.image_size is None:
|
||||
if hasattr(vision_config, "num_position_embeddings") and vision_config.num_position_embeddings:
|
||||
import math
|
||||
|
||||
num_grid = int(math.sqrt(vision_config.num_position_embeddings))
|
||||
vision_config.image_size = num_grid * vision_config.patch_size
|
||||
else:
|
||||
vision_config.image_size = vision_config.patch_size * 14 # fallback
|
||||
|
||||
if not hasattr(vision_config, "apply_vit_abs_pos_embed"):
|
||||
vision_config.apply_vit_abs_pos_embed = True
|
||||
|
||||
return vision_config
|
||||
|
||||
|
||||
class MingVisionEncoder(nn.Module):
|
||||
"""**Wrapper** around vLLM's Qwen3Omni_VisionTransformer for Ming."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"deepstack_merger_list.": "merger_list.",
|
||||
"merger.norm.": "merger.ln_q.",
|
||||
"merger.linear_fc1.": "merger.mlp.0.",
|
||||
"merger.linear_fc2.": "merger.mlp.2.",
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
adapted_config = _adapt_vision_config(vision_config)
|
||||
norm_eps = 1e-6
|
||||
self.encoder = Qwen3Omni_VisionTransformer(
|
||||
vision_config=adapted_config,
|
||||
norm_eps=norm_eps,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.encoder",
|
||||
)
|
||||
self.image_emb_dim = vision_config.out_hidden_size
|
||||
self.use_deepstack = (
|
||||
hasattr(vision_config, "deepstack_visual_indexes") and vision_config.deepstack_visual_indexes is not None
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.encoder.dtype
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return self.encoder.device
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""forward method of the vision encoder.
|
||||
|
||||
Args:
|
||||
pixel_values: Flattened pixel values.
|
||||
grid_thw: [num_images, 3] tensor of (t, h, w) grid sizes.
|
||||
|
||||
Returns:
|
||||
If deepstack is enabled, returns concatenated multi-scale features
|
||||
along the feature dim: [seq_len, hidden_size * (1 + num_deepstack)].
|
||||
Otherwise returns [seq_len, hidden_size].
|
||||
"""
|
||||
return self.encoder(pixel_values, grid_thw=grid_thw)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
import re
|
||||
|
||||
def _remap_merger_list_inner(name: str) -> str:
|
||||
name = re.sub(r"(merger_list\.\d+)\.norm\.", r"\1.ln_q.", name)
|
||||
name = re.sub(r"(merger_list\.\d+)\.linear_fc1\.", r"\1.mlp.0.", name)
|
||||
name = re.sub(r"(merger_list\.\d+)\.linear_fc2\.", r"\1.mlp.2.", name)
|
||||
|
||||
return name
|
||||
|
||||
remapped_weights = self.hf_to_vllm_mapper.apply(weights)
|
||||
remapped_weights = ((_remap_merger_list_inner(name), tensor) for name, tensor in remapped_weights)
|
||||
loaded_params = self.encoder.load_weights(remapped_weights)
|
||||
|
||||
loaded_params = {f"encoder.{loaded_param}" for loaded_param in loaded_params}
|
||||
|
||||
return loaded_params
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen2.5-Omni pipeline topology (frozen).
|
||||
|
||||
Stage 0: Thinker — multimodal understanding + text generation
|
||||
Stage 1: Talker — text embeddings → speech tokens
|
||||
Stage 2: Code2Wav — speech tokens → audio waveform
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen2_5_omni"
|
||||
|
||||
QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
model_type="qwen2_5_omni",
|
||||
model_arch="Qwen2_5OmniForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(0,),
|
||||
engine_output_type="latent",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [8294],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=2,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(1,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Single-stage thinker-only variant for the abort test.
|
||||
QWEN2_5_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
|
||||
model_type="qwen2_5_omni_thinker_only",
|
||||
model_arch="Qwen2_5OmniForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen3-Omni-MoE pipeline topology (frozen).
|
||||
|
||||
Stage 0: Thinker — multimodal understanding + text generation
|
||||
Stage 1: Talker — text embeddings → RVQ codec codes
|
||||
Stage 2: Code2Wav — RVQ codes → audio waveform
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni"
|
||||
|
||||
QWEN3_OMNI_PIPELINE = PipelineConfig(
|
||||
model_type="qwen3_omni_moe",
|
||||
model_arch="Qwen3OmniMoeForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
hf_config_name="thinker_config",
|
||||
engine_output_type="latent",
|
||||
custom_process_next_stage_input_func=(f"{_PROC}.thinker2talker_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(0,),
|
||||
hf_config_name="talker_config",
|
||||
engine_output_type="latent",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
custom_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [2150],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=2,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(1,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
hf_config_name="thinker_config",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -180,6 +180,11 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
"trailing_text_hidden",
|
||||
"tts_pad_embed_projected",
|
||||
}
|
||||
# Keys that need to be accumulated across streaming inputs
|
||||
self.streaming_accumulated_keys: set[str] = {
|
||||
"thinker_prefill_embeddings",
|
||||
"thinker_hidden_states",
|
||||
}
|
||||
|
||||
elif self.model_stage == "code2wav":
|
||||
self.enable_update_additional_information = True
|
||||
@@ -684,6 +689,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
input_embeds: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
text_step: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
):
|
||||
# TODO(Peiqi): not support intermediate_tensors now
|
||||
input_ids = safe_tensor_reshape(input_ids, (input_ids.shape[0], -1))
|
||||
@@ -891,10 +897,11 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
Returns:
|
||||
(input_ids, input_embeds) for talker
|
||||
"""
|
||||
target_len = thinker_result_ids.shape[-1]
|
||||
im_start_indexes = torch.cat(
|
||||
(
|
||||
torch.nonzero(input_ids[0] == self.config.im_start_token_id).squeeze(),
|
||||
torch.tensor([thinker_result_ids.shape[-1]], device=input_ids.device, dtype=input_ids.dtype),
|
||||
torch.tensor([target_len], device=input_ids.device, dtype=input_ids.dtype),
|
||||
),
|
||||
dim=-1,
|
||||
) # Shape [n_starts + 1]; Take batch 0 since batched inference is not supported here.
|
||||
@@ -1029,8 +1036,35 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
return last_talker_hidden, text_step, update_dict
|
||||
|
||||
def _get_talker_user_parts(self, im_start_index, segment_end_index, multimodal_mask, thinker_hidden, thinker_embed):
|
||||
clamped = min(
|
||||
segment_end_index,
|
||||
multimodal_mask.shape[0],
|
||||
thinker_hidden.shape[0],
|
||||
thinker_embed.shape[0],
|
||||
)
|
||||
if clamped < segment_end_index:
|
||||
logger.warning(
|
||||
"_get_talker_user_parts: segment_end_index %d clamped to %d "
|
||||
"(embed=%d, hidden=%d, mask=%d). "
|
||||
"This usually means _merge_pd_embeddings failed to merge "
|
||||
"prefill embeddings – check PD prefill_mm keys.",
|
||||
segment_end_index,
|
||||
clamped,
|
||||
thinker_embed.shape[0],
|
||||
thinker_hidden.shape[0],
|
||||
multimodal_mask.shape[0],
|
||||
)
|
||||
segment_end_index = clamped
|
||||
seg_len = segment_end_index - im_start_index
|
||||
if seg_len <= 0:
|
||||
return torch.empty(
|
||||
(0, self.config.talker_config.text_config.hidden_size),
|
||||
device=thinker_hidden.device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
user_talker_part = torch.empty(
|
||||
(segment_end_index - im_start_index, self.config.talker_config.text_config.hidden_size),
|
||||
(seg_len, self.config.talker_config.text_config.hidden_size),
|
||||
device=thinker_hidden.device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
@@ -1,510 +1,28 @@
|
||||
"""Qwen3-Omni Code Predictor -- optimized re-prefill, no KV cache.
|
||||
"""Qwen3-Omni Code Predictor -- thin wrapper over CodePredictorWrapper."""
|
||||
|
||||
* SDPA attention (F.scaled_dot_product_attention) with native GQA support
|
||||
* HF-compatible numerics (float32 RMSNorm, float32 RoPE, separate linear layers)
|
||||
* Per-call embedding buffer to avoid cross-request aliasing
|
||||
* Pre-allocated position_ids (read-only, safe to persist)
|
||||
* torch.compile (epilogue_fusion=False) on inner transformer by default
|
||||
* Inline sampling (top-k + top-p) -- no custom op overhead
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
|
||||
CodePredictorWrapper,
|
||||
CodePredictorWrapperConfig,
|
||||
)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# HF-numerics-compatible layers for code predictor
|
||||
# ===================================================================
|
||||
#
|
||||
# These use plain PyTorch ops (nn.Linear, manual RMSNorm in float32,
|
||||
# rotate_half RoPE) to produce outputs numerically identical to the
|
||||
# HuggingFace reference. vLLM's fused kernels (RMSNorm, QKVParallel,
|
||||
# get_rope) introduce small precision differences that compound across
|
||||
# the autoregressive steps of the code predictor, causing severe
|
||||
# audio quality degradation.
|
||||
#
|
||||
# See: https://github.com/vllm-project/vllm-omni/issues/2274
|
||||
class Qwen3OmniMoeTalkerCodePredictor(CodePredictorWrapper):
|
||||
"""Qwen3-Omni code predictor (no CUDA graphs, VocabParallelEmbedding)."""
|
||||
|
||||
|
||||
class _RMSNorm(nn.Module):
|
||||
"""RMSNorm matching HuggingFace's implementation exactly.
|
||||
|
||||
Computes variance in float32 to avoid bfloat16 precision loss.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
return self.weight * hidden_states.to(input_dtype)
|
||||
|
||||
|
||||
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Rotates half the hidden dims of the input."""
|
||||
x1 = x[..., : x.shape[-1] // 2]
|
||||
x2 = x[..., x.shape[-1] // 2 :]
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
class _RotaryEmbedding(nn.Module):
|
||||
"""RoPE matching HuggingFace's implementation exactly.
|
||||
|
||||
Forces float32 computation for cos/sin, matching HF's torch.autocast(enabled=False).
|
||||
"""
|
||||
|
||||
def __init__(self, config) -> None:
|
||||
super().__init__()
|
||||
head_dim = getattr(
|
||||
config,
|
||||
"head_dim",
|
||||
config.hidden_size // config.num_attention_heads,
|
||||
)
|
||||
rope_theta = getattr(config, "rope_theta", 10000.0)
|
||||
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# position_ids: [batch, seq_len]
|
||||
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
||||
position_ids_expanded = position_ids[:, None, :].float()
|
||||
|
||||
# Force float32 (matching HF)
|
||||
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
|
||||
with torch.autocast(device_type=device_type, enabled=False):
|
||||
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
||||
emb = torch.cat((freqs, freqs), dim=-1)
|
||||
cos = emb.cos()
|
||||
sin = emb.sin()
|
||||
|
||||
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
||||
|
||||
|
||||
class Qwen3OmniCodePredictorAttention(nn.Module):
|
||||
"""Multi-head self-attention for code predictor.
|
||||
|
||||
Uses ``F.scaled_dot_product_attention`` with HF-compatible RoPE and RMSNorm.
|
||||
No KV cache -- the code predictor always re-prefills the full (short)
|
||||
sequence each AR step.
|
||||
|
||||
Input : [B, seq_len, hidden_size]
|
||||
Output: [B, seq_len, hidden_size]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
cp_cfg = config.code_predictor_config
|
||||
self.num_heads = cp_cfg.num_attention_heads
|
||||
self.num_kv_heads = cp_cfg.num_key_value_heads
|
||||
self.head_dim = getattr(
|
||||
cp_cfg,
|
||||
"head_dim",
|
||||
cp_cfg.hidden_size // cp_cfg.num_attention_heads,
|
||||
)
|
||||
self.hidden_size = cp_cfg.hidden_size
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self._use_gqa = self.num_kv_heads != self.num_heads
|
||||
|
||||
# Separate q/k/v projections matching HF (no fused packing)
|
||||
self.q_proj = nn.Linear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.head_dim,
|
||||
bias=False,
|
||||
)
|
||||
self.k_proj = nn.Linear(
|
||||
self.hidden_size,
|
||||
self.num_kv_heads * self.head_dim,
|
||||
bias=False,
|
||||
)
|
||||
self.v_proj = nn.Linear(
|
||||
self.hidden_size,
|
||||
self.num_kv_heads * self.head_dim,
|
||||
bias=False,
|
||||
)
|
||||
self.o_proj = nn.Linear(
|
||||
self.num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
)
|
||||
self.q_norm = _RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps)
|
||||
self.k_norm = _RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim)
|
||||
hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim)
|
||||
|
||||
q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)
|
||||
k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2)
|
||||
v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2)
|
||||
|
||||
cos, sin = position_embeddings
|
||||
# cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads
|
||||
cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]
|
||||
sin = sin.unsqueeze(1)
|
||||
q = (q * cos) + (_rotate_half(q) * sin)
|
||||
k = (k * cos) + (_rotate_half(k) * sin)
|
||||
|
||||
attn_out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
scale=self.scaling,
|
||||
is_causal=True,
|
||||
enable_gqa=self._use_gqa,
|
||||
)
|
||||
|
||||
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
|
||||
output = self.o_proj(attn_out)
|
||||
return output
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# MLP
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class Qwen3OmniCodePredictorMLP(nn.Module):
|
||||
"""SiLU-gated MLP for code predictor, matching HF's implementation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.code_predictor_config.hidden_size
|
||||
intermediate_size = config.code_predictor_config.intermediate_size
|
||||
|
||||
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
||||
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
||||
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Decoder Layer
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class Qwen3OmniCodePredictorDecoderLayer(nn.Module):
|
||||
"""Transformer decoder layer (SDPA, no KV cache)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.self_attn = Qwen3OmniCodePredictorAttention(
|
||||
config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
self.mlp = Qwen3OmniCodePredictorMLP(
|
||||
config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
cp_cfg = config.code_predictor_config
|
||||
self.input_layernorm = _RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps)
|
||||
self.post_attention_layernorm = _RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(hidden_states, position_embeddings)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Base Transformer Model (re-prefill, no KV cache)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
"""Inner transformer for code predictor.
|
||||
|
||||
Signature: ``forward(inputs_embeds, position_ids) -> hidden_states``
|
||||
-- plain Tensor in, plain Tensor out (no namedtuple).
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config.code_predictor_config
|
||||
self.config = config
|
||||
|
||||
self.codec_embedding = nn.ModuleList(
|
||||
[VocabParallelEmbedding(config.vocab_size, config.hidden_size) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
Qwen3OmniCodePredictorDecoderLayer(
|
||||
vllm_config.model_config.hf_config,
|
||||
prefix=f"{prefix}.layers.{idx}",
|
||||
)
|
||||
for idx in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.norm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.rotary_emb = _RotaryEmbedding(config)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = inputs_embeds
|
||||
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(hidden_states, position_embeddings)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Code Predictor Wrapper (optimized re-prefill, persistent buffers)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
|
||||
"""Optimized code predictor -- re-prefill approach, no KV cache.
|
||||
|
||||
Each AR step forwards the full growing sequence (len 2 -> num_code_groups+1)
|
||||
through the transformer. The extra O(T^2) FLOPs are negligible for
|
||||
short sequences, and this avoids all KV-cache management overhead.
|
||||
|
||||
Optimizations:
|
||||
1. Per-call embedding buffer -- avoids cross-request aliasing.
|
||||
2. Pre-allocated position_ids -- no torch.arange per step.
|
||||
3. Cached module references -- bypass ModuleList indexing.
|
||||
4. torch.compile on inner transformer.
|
||||
5. Inline sampling (top-k + top-p) -- no custom op overhead.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.prefix = prefix
|
||||
|
||||
self.num_code_groups = config.code_predictor_config.num_code_groups
|
||||
self._hidden_size = config.code_predictor_config.hidden_size
|
||||
|
||||
self.model = Qwen3OmniCodePredictorBaseModel(
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
cp_config = vllm_config.model_config.hf_config.code_predictor_config
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
cp_config=cp_config,
|
||||
wrapper_config=CodePredictorWrapperConfig(
|
||||
use_cuda_graphs=False,
|
||||
use_parallel_embedding=True,
|
||||
use_projection=False,
|
||||
return_proj_buf=True,
|
||||
sampling_mode="stored",
|
||||
),
|
||||
talker_hidden_size=cp_config.hidden_size,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
# One lm_head per residual layer (layers 1 .. G-1)
|
||||
self.lm_head = nn.ModuleList(
|
||||
[
|
||||
nn.Linear(
|
||||
config.code_predictor_config.hidden_size,
|
||||
config.code_predictor_config.vocab_size,
|
||||
bias=False,
|
||||
)
|
||||
for _ in range(self.num_code_groups - 1)
|
||||
]
|
||||
)
|
||||
|
||||
self.set_sampling_params()
|
||||
|
||||
# Lazily initialised position ids (read-only, safe to persist)
|
||||
self._pos_ids: torch.Tensor | None = None
|
||||
|
||||
# Cached plain-list refs (set once)
|
||||
self._lm_heads: list | None = None
|
||||
self._codec_embeds: list | None = None
|
||||
|
||||
# Model forward (optionally compiled)
|
||||
self._model_fwd: object | None = None
|
||||
|
||||
def set_sampling_params(self, top_k: int = 50, top_p: float = 0.8):
|
||||
"""Configure sampling parameters to maintain consistency with previous implementation."""
|
||||
self._top_k = top_k
|
||||
self._top_p = top_p
|
||||
logger.debug(f"Sampling parameters updated: top_k={top_k}, top_p={top_p}s")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lazy-init helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_pos_ids(self, device: torch.device) -> None:
|
||||
if self._pos_ids is not None and self._pos_ids.device == device:
|
||||
return
|
||||
max_seq = self.num_code_groups + 1
|
||||
# [1, max_seq] for HF-style RoPE (will be expanded to [bsz, seq_len] at use)
|
||||
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device).unsqueeze(0)
|
||||
|
||||
def _ensure_cached_refs(self) -> None:
|
||||
if self._lm_heads is not None:
|
||||
return
|
||||
self._lm_heads = list(self.lm_head)
|
||||
self._codec_embeds = list(self.model.codec_embedding)
|
||||
|
||||
def _ensure_model_fwd(self) -> None:
|
||||
if self._model_fwd is not None:
|
||||
return
|
||||
if current_omni_platform.supports_torch_inductor():
|
||||
# torch.compile fuses RMSNorm/RoPE in ways that lose float32
|
||||
# precision, compounding across AR steps. Use epilogue_fusion=False
|
||||
# to disable the problematic fusions while still getting kernel
|
||||
# fusion benefits for the linear layers and SDPA.
|
||||
self._model_fwd = torch.compile(
|
||||
self.model.forward,
|
||||
dynamic=True,
|
||||
options={
|
||||
"epilogue_fusion": False,
|
||||
},
|
||||
)
|
||||
logger.info("code_predictor: torch.compile enabled (no epilogue fusion)")
|
||||
else:
|
||||
self._model_fwd = self.model.forward
|
||||
logger.info("code_predictor: using eager mode (no torch.compile)")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Forward -- re-prefill + inline sampling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
layer0_code: torch.Tensor,
|
||||
layer0_embed: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Predict residual codebooks 1..G-1 autoregressively via re-prefill.
|
||||
|
||||
Args:
|
||||
layer0_code: [bsz, 1] int64
|
||||
layer0_embed: [bsz, 1, hidden_size]
|
||||
last_talker_hidden: [bsz, 1, hidden_size]
|
||||
|
||||
Returns:
|
||||
all_codes: [bsz, num_code_groups, 1]
|
||||
proj_buf: [bsz, num_code_groups + 1, hidden_size]
|
||||
pos 0 = last_talker_hidden (NOT a codec embed)
|
||||
pos 1 = layer0_embed
|
||||
pos 2.. = `codec_embedding[i](predicted_code_i)`
|
||||
"""
|
||||
bsz = int(layer0_code.shape[0])
|
||||
device = layer0_code.device
|
||||
dtype = last_talker_hidden.dtype
|
||||
num_groups = self.num_code_groups
|
||||
|
||||
# Lazy init (read-only caches only)
|
||||
self._ensure_pos_ids(device)
|
||||
self._ensure_model_fwd()
|
||||
self._ensure_cached_refs()
|
||||
|
||||
# Allocate proj_buf locally each call to avoid cross-call aliasing
|
||||
max_seq = num_groups + 1
|
||||
proj_buf = torch.zeros(bsz, max_seq, self._hidden_size, dtype=dtype, device=device)
|
||||
pos_ids = self._pos_ids
|
||||
model_fwd = self._model_fwd
|
||||
lm_heads = self._lm_heads
|
||||
codec_embeds = self._codec_embeds
|
||||
|
||||
# Output codes
|
||||
all_codes = torch.empty(bsz, num_groups, 1, dtype=torch.int64, device=device)
|
||||
all_codes[:, 0] = layer0_code
|
||||
|
||||
# Fill buffer positions 0 & 1
|
||||
proj_buf[:bsz, 0:1, :] = last_talker_hidden
|
||||
proj_buf[:bsz, 1:2, :] = layer0_embed
|
||||
|
||||
# Autoregressive loop: predict layers 1..G-1
|
||||
for step in range(1, num_groups):
|
||||
seq_len = step + 1
|
||||
projected = proj_buf[:bsz, :seq_len, :]
|
||||
# position_ids: [batch, seq_len] for HF-style RoPE
|
||||
step_pos_ids = pos_ids[:, :seq_len].expand(bsz, -1)
|
||||
|
||||
hidden_out = model_fwd(projected, step_pos_ids)
|
||||
|
||||
# Inline sampling: top-k -> top-p -> softmax -> multinomial
|
||||
logits = lm_heads[step - 1](hidden_out[:, -1, :]) # [bsz, vocab]
|
||||
if self._top_k > 0:
|
||||
topk_vals, _ = logits.topk(self._top_k, dim=-1)
|
||||
logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf"))
|
||||
if self._top_p < 1.0:
|
||||
sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
|
||||
cumulative_probs = F.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
|
||||
# Remove tokens with cumulative probability above top_p
|
||||
remove_mask = cumulative_probs - F.softmax(sorted_logits, dim=-1) >= self._top_p
|
||||
sorted_logits[remove_mask] = float("-inf")
|
||||
logits = sorted_logits.scatter(1, sorted_idx, sorted_logits)
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
code = torch.multinomial(probs, num_samples=1) # [bsz, 1]
|
||||
|
||||
all_codes[:, step] = code
|
||||
|
||||
# Embed predicted code -> next buffer position
|
||||
new_embed = codec_embeds[step - 1](code) # [batch, 1, hidden_size]
|
||||
proj_buf[:bsz, step + 1 : step + 2, :] = new_embed
|
||||
|
||||
return all_codes, proj_buf[:bsz]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights directly (no fused projection remapping needed).
|
||||
|
||||
Since we use separate nn.Linear for q/k/v/o and gate/up/down,
|
||||
weight names match the HF checkpoint directly.
|
||||
"""
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Skip rotary embeddings
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
param = params_dict.get(name)
|
||||
if param is None:
|
||||
continue
|
||||
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen3-TTS pipeline: Talker (text → RVQ codec) → Code2Wav (codec → audio).
|
||||
|
||||
Chunked vs end-to-end mode is dispatched from ``deploy.async_chunk``.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_tts"
|
||||
|
||||
QWEN3_TTS_PIPELINE = PipelineConfig(
|
||||
model_type="qwen3_tts",
|
||||
# Pipeline-level default; the code2wav stage overrides per-stage below.
|
||||
model_arch="Qwen3TTSTalkerForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="qwen3_tts",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [2150],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="Qwen3TTSCode2Wav",
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
sampling_constraints={"detokenize": True},
|
||||
extras={"tts_args": {"max_instructions_length": 500}},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -1,93 +0,0 @@
|
||||
model_type: qwen3_tts
|
||||
async_chunk: true
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: qwen3_tts
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
input_sources: []
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
max_num_seqs: 10
|
||||
model_arch: Qwen3TTSTalkerForConditionalGeneration
|
||||
hf_overrides:
|
||||
architectures: [Qwen3TTSTalkerForConditionalGeneration]
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.08
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 512
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 1
|
||||
model_stage: code2wav
|
||||
stage_type: llm
|
||||
input_sources: [0]
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen3TTSCode2Wav
|
||||
hf_overrides:
|
||||
architectures: [Qwen3TTSCode2Wav]
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.08
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 65536
|
||||
max_model_len: 65536
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
tts_args:
|
||||
max_instructions_length: 500
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
# Match the decoder sliding attention window to avoid chunk-boundary noise.
|
||||
codec_left_context_frames: 72
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
window_size: -1
|
||||
@@ -1,318 +1,27 @@
|
||||
"""Qwen3-TTS Code Predictor -- thin wrapper over CodePredictorWrapper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
)
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
|
||||
CodePredictorBaseModel,
|
||||
CodePredictorWrapper,
|
||||
CodePredictorWrapperConfig,
|
||||
)
|
||||
|
||||
from .configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig, Qwen3TTSTalkerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# Backward-compat alias used by tests
|
||||
Qwen3TTSTalkerCodePredictorModelVLLM = CodePredictorBaseModel
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# HF-numerics-compatible layers for code predictor
|
||||
# ===================================================================
|
||||
#
|
||||
# These use plain PyTorch ops (nn.Linear, manual RMSNorm in float32,
|
||||
# rotate_half RoPE) to produce outputs numerically identical to the
|
||||
# HuggingFace reference. vLLM's fused kernels (RMSNorm, QKVParallel,
|
||||
# get_rope) introduce small precision differences that compound across
|
||||
# the 15 autoregressive steps of the code predictor, causing severe
|
||||
# audio quality degradation (UTMOS ~4.26 → ~2.66).
|
||||
#
|
||||
# See: https://github.com/vllm-project/vllm-omni/issues/2274
|
||||
|
||||
|
||||
class _RMSNorm(nn.Module):
|
||||
"""RMSNorm matching HuggingFace's Qwen3TTSRMSNorm exactly.
|
||||
|
||||
Computes variance in float32 to avoid bfloat16 precision loss.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
return self.weight * hidden_states.to(input_dtype)
|
||||
|
||||
|
||||
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Rotates half the hidden dims of the input."""
|
||||
x1 = x[..., : x.shape[-1] // 2]
|
||||
x2 = x[..., x.shape[-1] // 2 :]
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
class _RotaryEmbedding(nn.Module):
|
||||
"""RoPE matching HuggingFace's Qwen3TTSRotaryEmbedding exactly.
|
||||
|
||||
Forces float32 computation for cos/sin, matching HF's torch.autocast(enabled=False).
|
||||
"""
|
||||
|
||||
def __init__(self, config: Qwen3TTSTalkerCodePredictorConfig) -> None:
|
||||
super().__init__()
|
||||
head_dim = getattr(
|
||||
config,
|
||||
"head_dim",
|
||||
config.hidden_size // config.num_attention_heads,
|
||||
)
|
||||
# Standard default RoPE
|
||||
rope_theta = getattr(config, "rope_theta", 10000.0)
|
||||
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# position_ids: [batch, seq_len]
|
||||
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
||||
position_ids_expanded = position_ids[:, None, :].float()
|
||||
|
||||
# Force float32 (matching HF)
|
||||
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
|
||||
with torch.autocast(device_type=device_type, enabled=False):
|
||||
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
||||
emb = torch.cat((freqs, freqs), dim=-1)
|
||||
cos = emb.cos()
|
||||
sin = emb.sin()
|
||||
|
||||
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
||||
|
||||
|
||||
class _CodePredictorAttention(nn.Module):
|
||||
"""Standalone multi-head attention for code predictor.
|
||||
|
||||
Uses F.scaled_dot_product_attention with HF-compatible RoPE and RMSNorm.
|
||||
Input: [B, seq_len, hidden_size], output: [B, seq_len, hidden_size].
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3TTSTalkerCodePredictorConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_kv_heads = config.num_key_value_heads
|
||||
self.head_dim = getattr(
|
||||
config,
|
||||
"head_dim",
|
||||
config.hidden_size // config.num_attention_heads,
|
||||
)
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self._use_gqa = self.num_kv_heads != self.num_heads
|
||||
|
||||
# Separate q/k/v projections matching HF (no fused packing)
|
||||
self.q_proj = nn.Linear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.head_dim,
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
)
|
||||
self.k_proj = nn.Linear(
|
||||
self.hidden_size,
|
||||
self.num_kv_heads * self.head_dim,
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
)
|
||||
self.v_proj = nn.Linear(
|
||||
self.hidden_size,
|
||||
self.num_kv_heads * self.head_dim,
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
)
|
||||
self.o_proj = nn.Linear(
|
||||
self.num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
)
|
||||
self.q_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim)
|
||||
hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim)
|
||||
|
||||
q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)
|
||||
k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2)
|
||||
v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2)
|
||||
|
||||
cos, sin = position_embeddings
|
||||
# cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads
|
||||
cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]
|
||||
sin = sin.unsqueeze(1)
|
||||
q = (q * cos) + (_rotate_half(q) * sin)
|
||||
k = (k * cos) + (_rotate_half(k) * sin)
|
||||
|
||||
attn_out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
scale=self.scaling,
|
||||
is_causal=True,
|
||||
enable_gqa=self._use_gqa,
|
||||
)
|
||||
|
||||
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
|
||||
output = self.o_proj(attn_out)
|
||||
return output
|
||||
|
||||
|
||||
class _CodePredictorMLP(nn.Module):
|
||||
"""SiLU-gated MLP for code predictor, matching HF's Qwen3TTSTalkerTextMLP."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3TTSTalkerCodePredictorConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
||||
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
||||
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
|
||||
|
||||
|
||||
class _CodePredictorDecoderLayer(nn.Module):
|
||||
"""Transformer decoder layer for code predictor (SDPA, no KV cache)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3TTSTalkerCodePredictorConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.self_attn = _CodePredictorAttention(config, prefix=f"{prefix}.self_attn")
|
||||
self.mlp = _CodePredictorMLP(config, prefix=f"{prefix}.mlp")
|
||||
self.input_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(hidden_states, position_embeddings)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Code Predictor Transformer Model
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class Qwen3TTSTalkerCodePredictorModelVLLM(nn.Module):
|
||||
"""Transformer model for the code predictor (re-prefill, no KV cache)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3TTSTalkerCodePredictorConfig,
|
||||
*,
|
||||
talker_hidden_size: int | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[_CodePredictorDecoderLayer(config, prefix=f"{prefix}.layers.{i}") for i in range(config.num_hidden_layers)]
|
||||
)
|
||||
self.norm = _RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.rotary_emb = _RotaryEmbedding(config)
|
||||
|
||||
# Codec embeddings: one per residual group. Stored in talker hidden dim
|
||||
# (some checkpoints use talker_hidden_size != code_predictor hidden_size).
|
||||
emb_dim = int(talker_hidden_size) if talker_hidden_size is not None else int(config.hidden_size)
|
||||
self.codec_embedding = nn.ModuleList(
|
||||
[nn.Embedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.codec_embedding
|
||||
|
||||
def forward(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = inputs_embeds
|
||||
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(hidden_states, position_embeddings)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
param = params_dict.get(name)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Code Predictor Wrapper (optimized re-prefill + torch.compile)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
|
||||
"""vLLM-native code_predictor for the AR talker (residual codebooks).
|
||||
|
||||
Re-prefill approach: each AR step forwards the full growing sequence
|
||||
through the 5-layer transformer. No KV cache needed. This trades
|
||||
~O(T^2) extra attention FLOPs (negligible for T=16, 5 layers) for
|
||||
zero KV cache management overhead and a simpler execution model.
|
||||
|
||||
Uses HF-compatible layers (plain nn.Linear, float32 RMSNorm, rotate_half
|
||||
RoPE) to ensure numerical fidelity with the reference implementation.
|
||||
Precision matters here because small errors compound across 15 AR steps.
|
||||
|
||||
Optimizations preserved:
|
||||
1. torch.compile on model forward -- fuses small kernel launches.
|
||||
2. Pre-allocated embedding buffer [B, max_seq, H] -- no torch.cat per step.
|
||||
3. Projection caching -- each token projected once and cached.
|
||||
4. Pre-allocated position_ids -- no torch.arange per step.
|
||||
5. Inline sampling -- no custom op / forward_context overhead.
|
||||
6. Cached module references -- bypass nn.Module.__call__ overhead.
|
||||
7. CUDA graphs per batch-size bucket.
|
||||
"""
|
||||
class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(CodePredictorWrapper):
|
||||
"""Qwen3-TTS code predictor (CUDA graphs, per-call sampling, projection)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -322,250 +31,24 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
|
||||
talker_config: Qwen3TTSTalkerConfig,
|
||||
prefix: str = "code_predictor",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._vllm_config = vllm_config
|
||||
self.config = config
|
||||
self.talker_config = talker_config
|
||||
|
||||
self.model = Qwen3TTSTalkerCodePredictorModelVLLM(
|
||||
config,
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
cp_config=config,
|
||||
wrapper_config=CodePredictorWrapperConfig(
|
||||
use_cuda_graphs=True,
|
||||
use_parallel_embedding=False,
|
||||
use_projection=(config.hidden_size != talker_config.hidden_size),
|
||||
return_proj_buf=False,
|
||||
sampling_mode="per_call",
|
||||
),
|
||||
talker_hidden_size=int(talker_config.hidden_size),
|
||||
prefix=f"{prefix}.model",
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.lm_head = nn.ModuleList(
|
||||
[nn.Linear(config.hidden_size, config.vocab_size, bias=False) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
|
||||
if config.hidden_size != talker_config.hidden_size:
|
||||
self.small_to_mtp_projection = nn.Linear(talker_config.hidden_size, config.hidden_size, bias=True)
|
||||
else:
|
||||
self.small_to_mtp_projection = nn.Identity()
|
||||
|
||||
self._num_groups = int(config.num_code_groups)
|
||||
self._talker_hidden = int(talker_config.hidden_size)
|
||||
self._cp_hidden = int(config.hidden_size)
|
||||
|
||||
# Pre-allocated buffers (lazily initialized on first forward).
|
||||
self._proj_buf: torch.Tensor | None = None
|
||||
self._model_dtype: torch.dtype | None = None
|
||||
|
||||
# torch.compile + warmup state (lazily initialized in _setup_compile).
|
||||
self._compiled_model_fwd = None
|
||||
self._bucket_sizes: list[int] = []
|
||||
self._bucket_pos_ids: dict[int, torch.Tensor] = {}
|
||||
self._lm_heads_list: list[nn.Module] | None = None
|
||||
self._codec_embeds_list: list[nn.Module] | None = None
|
||||
self._cuda_graphs: dict[int, tuple[torch.cuda.CUDAGraph, torch.Tensor]] = {}
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.model.get_input_embeddings()
|
||||
# Store talker_config for backward compat (accessed by some callers)
|
||||
self.talker_config = talker_config
|
||||
self._vllm_config = vllm_config
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with vllm config context (required for VocabParallelEmbedding)."""
|
||||
with set_current_vllm_config(self._vllm_config):
|
||||
loaded: set[str] = set()
|
||||
model_weights: list[tuple[str, torch.Tensor]] = []
|
||||
other_weights: list[tuple[str, torch.Tensor]] = []
|
||||
for name, w in weights:
|
||||
if name.startswith("model."):
|
||||
model_weights.append((name[len("model.") :], w))
|
||||
else:
|
||||
other_weights.append((name, w))
|
||||
|
||||
loaded_model = self.model.load_weights(model_weights)
|
||||
loaded |= {f"model.{n}" for n in loaded_model}
|
||||
|
||||
params = dict(self.named_parameters(remove_duplicate=False))
|
||||
for name, w in other_weights:
|
||||
if name not in params:
|
||||
continue
|
||||
default_weight_loader(params[name], w)
|
||||
loaded.add(name)
|
||||
|
||||
return loaded
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Pre-allocated buffer management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_buffers(self, device: torch.device, dtype: torch.dtype) -> None:
|
||||
max_seq = self._num_groups + 1
|
||||
if self._proj_buf is not None and self._proj_buf.device == device and self._proj_buf.dtype == dtype:
|
||||
return
|
||||
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
|
||||
self._proj_buf = torch.zeros(
|
||||
max_bsz,
|
||||
max_seq,
|
||||
self._cp_hidden,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def _setup_compile(self) -> None:
|
||||
"""Lazily set up torch.compile with manual CUDA graph capture."""
|
||||
if self._compiled_model_fwd is not None:
|
||||
return
|
||||
# Cache model parameter dtype so forward() doesn't need to query it
|
||||
# on every call. Also ensures warmup buffers match model precision
|
||||
# even when upstream modules produce a different dtype (#2385).
|
||||
self._model_dtype = next(self.model.parameters()).dtype
|
||||
self._lm_heads_list = list(self.lm_head)
|
||||
self._codec_embeds_list = list(self.model.codec_embedding)
|
||||
if not current_omni_platform.supports_torch_inductor():
|
||||
logger.warning_once("code_predictor: torch.compile disabled")
|
||||
self._compiled_model_fwd = self.model.forward
|
||||
return
|
||||
|
||||
# torch.compile fuses RMSNorm/RoPE in ways that lose float32
|
||||
# precision, compounding across 15 AR steps. Use torch.compile
|
||||
# with options that disable the problematic fusions while still
|
||||
# getting kernel fusion benefits for the linear layers and SDPA.
|
||||
self._compiled_model_fwd = torch.compile(
|
||||
self.model.forward,
|
||||
dynamic=False,
|
||||
options={
|
||||
"epilogue_fusion": False,
|
||||
},
|
||||
)
|
||||
self._warmup_buckets()
|
||||
self._capture_cuda_graphs()
|
||||
logger.info("code_predictor: torch.compile (no epilogue fusion) + CUDA graphs")
|
||||
|
||||
def _padded_bsz(self, bsz: int) -> int:
|
||||
for bucket in self._bucket_sizes:
|
||||
if bsz <= bucket:
|
||||
return bucket
|
||||
return bsz
|
||||
|
||||
def _warmup_buckets(self) -> None:
|
||||
"""Warmup power-of-2 batch-size buckets to front-load Inductor compilation."""
|
||||
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
|
||||
bucket_sizes = [1 << i for i in range(max_bsz.bit_length()) if (1 << i) <= max_bsz]
|
||||
if max_bsz not in bucket_sizes:
|
||||
bucket_sizes.append(max_bsz)
|
||||
self._bucket_sizes = sorted(bucket_sizes)
|
||||
|
||||
max_seq = self._num_groups + 1
|
||||
device = next(self.model.parameters()).device
|
||||
|
||||
# Ensure proj_buf matches model parameter dtype to avoid dtype
|
||||
# mismatch during warmup compilation (see #2385).
|
||||
self._ensure_buffers(device, self._model_dtype)
|
||||
proj_buf = self._proj_buf
|
||||
for bsz in self._bucket_sizes:
|
||||
# position_ids: [batch, seq_len] for HF-style RoPE
|
||||
pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1)
|
||||
self._bucket_pos_ids[bsz] = pos_ids
|
||||
for _ in range(3):
|
||||
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
|
||||
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
|
||||
|
||||
def _capture_cuda_graphs(self) -> None:
|
||||
"""Capture a CUDA graph per bucket using vLLM's global graph pool."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pool = current_platform.get_global_graph_pool()
|
||||
|
||||
max_seq = self._num_groups + 1
|
||||
proj_buf = self._proj_buf
|
||||
|
||||
for bsz in self._bucket_sizes:
|
||||
static_input = proj_buf[:bsz, :max_seq, :]
|
||||
pos_ids = self._bucket_pos_ids[bsz]
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, pool=pool):
|
||||
static_output = self._compiled_model_fwd(static_input, pos_ids)
|
||||
|
||||
self._cuda_graphs[bsz] = (g, static_output)
|
||||
|
||||
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Optimized forward: re-prefill + torch.compile + projection cache
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
layer0_code: torch.Tensor,
|
||||
layer0_embed: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
do_sample: bool = True,
|
||||
temperature: float = 0.9,
|
||||
top_k: int = 50,
|
||||
top_p: float = 1.0,
|
||||
) -> torch.Tensor:
|
||||
"""Predict residual codebooks 1..Q-1 autoregressively via re-prefill.
|
||||
|
||||
torch.compile fuses the ~60 small kernel launches per step into fewer
|
||||
fused kernels, reducing kernel launch overhead by ~75%.
|
||||
|
||||
Projection caching: each token is projected once via small_to_mtp_projection
|
||||
and cached in _proj_buf, avoiding redundant re-projection of past tokens.
|
||||
"""
|
||||
bsz = int(layer0_code.shape[0])
|
||||
num_groups = self._num_groups
|
||||
device = layer0_code.device
|
||||
|
||||
all_codes = torch.empty(bsz, num_groups, dtype=torch.long, device=device)
|
||||
all_codes[:, 0] = layer0_code.reshape(bsz)
|
||||
|
||||
# _setup_compile caches _model_dtype on first call; use it for buffers
|
||||
# so they always match model weight precision (#2385).
|
||||
self._setup_compile()
|
||||
dtype = self._model_dtype
|
||||
self._ensure_buffers(device, dtype)
|
||||
|
||||
proj_buf = self._proj_buf
|
||||
max_seq = self._num_groups + 1
|
||||
|
||||
projection = self.small_to_mtp_projection
|
||||
model_fwd = self._compiled_model_fwd
|
||||
lm_heads = self._lm_heads_list
|
||||
codec_embeds = self._codec_embeds_list
|
||||
|
||||
use_sampling = do_sample and temperature > 0
|
||||
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
|
||||
if use_sampling and top_p != 1.0:
|
||||
raise NotImplementedError(
|
||||
"top_p sampling is not implemented for the vLLM-native code predictor; please set top_p=1.0."
|
||||
)
|
||||
|
||||
padded_bsz = self._padded_bsz(bsz)
|
||||
proj_buf[:padded_bsz].zero_()
|
||||
|
||||
proj_buf[:bsz, 0, :] = projection(last_talker_hidden.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
|
||||
proj_buf[:bsz, 1, :] = projection(layer0_embed.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
|
||||
full_pos_ids = self._bucket_pos_ids.get(padded_bsz)
|
||||
if full_pos_ids is None:
|
||||
full_pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1)
|
||||
|
||||
# Use captured CUDA graph if available, otherwise call compiled fn.
|
||||
cuda_graph_entry = self._cuda_graphs.get(padded_bsz)
|
||||
|
||||
for step in range(1, num_groups):
|
||||
if cuda_graph_entry is not None:
|
||||
cuda_graph_entry[0].replay()
|
||||
hidden_out = cuda_graph_entry[1]
|
||||
else:
|
||||
hidden_out = model_fwd(proj_buf[:padded_bsz, :max_seq, :], full_pos_ids)
|
||||
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
|
||||
|
||||
if use_sampling:
|
||||
scaled = logits * inv_temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
probs = F.softmax(scaled, dim=-1)
|
||||
next_ids = torch.multinomial(probs, num_samples=1)
|
||||
else:
|
||||
next_ids = logits.argmax(dim=-1, keepdim=True)
|
||||
|
||||
all_codes[:, step] = next_ids.reshape(bsz)
|
||||
|
||||
if step < num_groups - 1:
|
||||
new_embed = codec_embeds[step - 1](next_ids)
|
||||
proj_buf[:bsz, step + 1, :] = projection(new_embed.reshape(bsz, 1, -1)).reshape(bsz, -1)
|
||||
|
||||
return all_codes
|
||||
return super().load_weights(weights)
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.qwen3 import Qwen3Model
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper, maybe_prefix
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
@@ -414,6 +415,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
# In-memory LRU cache for voice extraction artifacts (Base voice clone).
|
||||
self._voice_cache = VoiceEmbeddingCache()
|
||||
raw_subtalker_sampling = getattr(vllm_config.model_config, "subtalker_sampling_params", None)
|
||||
self._subtalker_sampling_params: dict[str, Any] = (
|
||||
dict(raw_subtalker_sampling) if isinstance(raw_subtalker_sampling, Mapping) else {}
|
||||
)
|
||||
|
||||
# -------------------- vLLM required hooks --------------------
|
||||
|
||||
@@ -1094,9 +1099,8 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
# Resample to 24kHz for speaker encoder.
|
||||
target_sr = int(getattr(self.config.speaker_encoder_config, "sample_rate", 24000))
|
||||
if sr != target_sr:
|
||||
from vllm.multimodal.audio import resample_audio_resampy
|
||||
|
||||
wav = resample_audio_resampy(wav.astype(np.float32), orig_sr=int(sr), target_sr=target_sr)
|
||||
resampler = AudioResampler(target_sr=target_sr)
|
||||
wav = resampler.resample(wav.astype(np.float32), orig_sr=int(sr))
|
||||
sr = target_sr
|
||||
|
||||
# Follow official implementation: mel_spectrogram expects 24kHz.
|
||||
@@ -1638,6 +1642,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
input_embeds: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
text_step: torch.Tensor,
|
||||
do_sample: bool | None = None,
|
||||
temperature: float | None = None,
|
||||
top_k: int | None = None,
|
||||
top_p: float | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GPU fast-path used by OmniGPUModelRunner to predict residual codebooks (1..Q-1).
|
||||
Returns (inputs_embeds, audio_codes) for the current step."""
|
||||
@@ -1656,15 +1664,24 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
audio_codes = input_ids.reshape(bsz, 1)
|
||||
return (last_id_hidden + text_step).reshape(bsz, -1), audio_codes
|
||||
|
||||
# Predict residual codes (1..Q-1) with HF reference sampling params.
|
||||
subtalker_params = self._subtalker_sampling_params
|
||||
if do_sample is None:
|
||||
do_sample = bool(subtalker_params.get("do_sample", True))
|
||||
if temperature is None:
|
||||
temperature = float(subtalker_params.get("temperature", 0.9))
|
||||
if top_k is None:
|
||||
top_k = int(subtalker_params.get("top_k", 50))
|
||||
if top_p is None:
|
||||
top_p = float(subtalker_params.get("top_p", 1.0))
|
||||
|
||||
audio_codes = self.code_predictor(
|
||||
layer0_code=input_ids.reshape(bsz, 1),
|
||||
layer0_embed=last_id_hidden,
|
||||
last_talker_hidden=past_hidden,
|
||||
do_sample=True,
|
||||
temperature=0.9,
|
||||
top_k=50,
|
||||
top_p=1.0,
|
||||
do_sample=do_sample,
|
||||
temperature=temperature,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
) # [B, Q]
|
||||
|
||||
# Map invalid layer-0 ids (e.g. EOS) to PAD=0 so SpeechTokenizer sees only real codes.
|
||||
|
||||
@@ -22,7 +22,7 @@ import soundfile as sf
|
||||
import torch
|
||||
from torch.nn.utils.rnn import pad_sequence
|
||||
from transformers import AutoConfig, AutoFeatureExtractor, AutoModel
|
||||
from vllm.multimodal.audio import resample_audio_resampy
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
from vllm.multimodal.media.audio import load_audio as _load_audio_file
|
||||
|
||||
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
|
||||
@@ -161,7 +161,8 @@ class Qwen3TTSTokenizer:
|
||||
audio = np.mean(audio, axis=-1)
|
||||
|
||||
if sr != target_sr:
|
||||
audio = resample_audio_resampy(audio, orig_sr=sr, target_sr=target_sr)
|
||||
resampler = AudioResampler(target_sr=target_sr)
|
||||
audio = resampler.resample(audio, orig_sr=sr)
|
||||
|
||||
return audio.astype(np.float32)
|
||||
|
||||
@@ -209,7 +210,8 @@ class Qwen3TTSTokenizer:
|
||||
if a.ndim > 1:
|
||||
a = np.mean(a, axis=-1)
|
||||
if int(sr) != target_sr:
|
||||
a = resample_audio_resampy(a.astype(np.float32), orig_sr=int(sr), target_sr=target_sr)
|
||||
resampler = AudioResampler(target_sr=target_sr)
|
||||
a = resampler.resample(a.astype(np.float32), orig_sr=int(sr))
|
||||
out.append(a.astype(np.float32))
|
||||
return out
|
||||
|
||||
|
||||
@@ -17,17 +17,17 @@ import operator
|
||||
from itertools import accumulate
|
||||
|
||||
import onnxruntime
|
||||
import sox
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torchaudio.compliance.kaldi as kaldi
|
||||
from torch import Tensor
|
||||
|
||||
from vllm_omni.utils.audio import mel_filter_bank
|
||||
from vllm_omni.model_executor.models.whisper_utils import Conv1d, ConvTranspose1d
|
||||
from vllm_omni.utils.audio import mel_filter_bank, peak_normalize
|
||||
|
||||
from .core_vq import DistributedGroupResidualVectorQuantization
|
||||
from .whisper_encoder import Conv1d, ConvTranspose1d, WhisperEncoder
|
||||
from .whisper_encoder import WhisperEncoder
|
||||
|
||||
|
||||
def dynamic_range_compression_torch(x, c=1, clip_val=1e-5):
|
||||
@@ -153,9 +153,6 @@ class XVectorExtractor(nn.Module):
|
||||
audio_codec_with_xvector, sess_options=option, providers=providers
|
||||
)
|
||||
|
||||
self.tfm = sox.Transformer()
|
||||
self.tfm.norm(db_level=-6)
|
||||
|
||||
self.mel_ext = MelSpectrogramFeatures(
|
||||
filter_length=1024,
|
||||
hop_length=160,
|
||||
@@ -183,8 +180,7 @@ class XVectorExtractor(nn.Module):
|
||||
return norm_embedding.numpy(), ref_mel.permute(0, 2, 1).squeeze(0).numpy()
|
||||
|
||||
def sox_norm(self, audio):
|
||||
wav_norm = self.tfm.build_array(input_array=audio, sample_rate_in=16000)
|
||||
return wav_norm
|
||||
return peak_normalize(audio, db_level=-6)
|
||||
|
||||
|
||||
class WhisperEncoderVQ(WhisperEncoder):
|
||||
|
||||
@@ -23,6 +23,7 @@ import torch.nn.functional as F
|
||||
from torch import Tensor, nn
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func
|
||||
from vllm_omni.model_executor.models.whisper_utils import Conv1d, Linear, sinusoids
|
||||
from vllm_omni.utils.audio import mel_filter_bank
|
||||
|
||||
N_FFT = 400
|
||||
@@ -102,30 +103,6 @@ def get_mel_audio(audio, padding=False, audio_vq_ds_rate=1, n_mels=128):
|
||||
return mel
|
||||
|
||||
|
||||
def sinusoids(length, channels, max_timescale=10000):
|
||||
"""Returns sinusoids for positional embedding"""
|
||||
assert channels % 2 == 0
|
||||
log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
|
||||
inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
|
||||
scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
|
||||
return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
|
||||
|
||||
|
||||
class Conv1d(nn.Conv1d):
|
||||
def _conv_forward(self, x: Tensor, weight: Tensor, bias: Tensor | None) -> Tensor:
|
||||
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
|
||||
|
||||
|
||||
class ConvTranspose1d(nn.ConvTranspose1d):
|
||||
def _conv_forward(self, x: Tensor, weight: Tensor, bias: Tensor | None) -> Tensor:
|
||||
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
|
||||
|
||||
|
||||
class Linear(nn.Linear):
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return F.linear(x, self.weight.to(x.dtype), None if self.bias is None else self.bias.to(x.dtype))
|
||||
|
||||
|
||||
class MultiHeadAttention(nn.Module):
|
||||
def __init__(self, n_state: int, n_head: int, use_flash_attention: bool = True):
|
||||
super().__init__()
|
||||
|
||||
@@ -145,6 +145,12 @@ _OMNI_MODELS = {
|
||||
"fish_speech_dac_decoder",
|
||||
"FishSpeechDACDecoder",
|
||||
),
|
||||
## VoxCPM
|
||||
"VoxCPMForConditionalGeneration": (
|
||||
"voxcpm",
|
||||
"voxcpm",
|
||||
"VoxCPMForConditionalGeneration",
|
||||
),
|
||||
## VoxCPM2
|
||||
"VoxCPM2TalkerForConditionalGeneration": (
|
||||
"voxcpm2",
|
||||
@@ -174,6 +180,23 @@ _OMNI_MODELS = {
|
||||
"dynin_omni",
|
||||
"DyninOmniForConditionalGeneration",
|
||||
),
|
||||
## Ming-flash-omni-2.0
|
||||
"MingFlashOmniForConditionalGeneration": (
|
||||
"ming_flash_omni",
|
||||
"ming_flash_omni",
|
||||
"MingFlashOmniForConditionalGeneration",
|
||||
),
|
||||
"MingFlashOmniThinkerForConditionalGeneration": (
|
||||
"ming_flash_omni",
|
||||
"ming_flash_omni_thinker",
|
||||
"MingFlashOmniThinkerForConditionalGeneration",
|
||||
),
|
||||
# Alias: HF repo currently ships this architecture name in config.json
|
||||
"BailingMM2NativeForConditionalGeneration": (
|
||||
"ming_flash_omni",
|
||||
"ming_flash_omni",
|
||||
"MingFlashOmniForConditionalGeneration",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from .configuration_voxcpm import VoxCPMConfig
|
||||
from .voxcpm import VoxCPMForConditionalGeneration
|
||||
|
||||
__all__ = [
|
||||
"VoxCPMConfig",
|
||||
"VoxCPMForConditionalGeneration",
|
||||
]
|
||||
@@ -0,0 +1,3 @@
|
||||
from vllm_omni.transformers_utils.configs.voxcpm import VoxCPMConfig
|
||||
|
||||
__all__ = ["VoxCPMConfig"]
|
||||
@@ -0,0 +1,886 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import warnings
|
||||
import wave
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from tqdm import tqdm
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
from .voxcpm_loader import (
|
||||
_build_prompt_cache_with_soundfile,
|
||||
_device_to_string,
|
||||
_force_cuda_available_for_npu,
|
||||
_import_voxcpm_audio_vae_classes,
|
||||
_import_voxcpm_base_model_class,
|
||||
_is_torchcodec_load_error,
|
||||
_normalize_dtype_name,
|
||||
_prepare_runtime_model_dir,
|
||||
_resolve_runtime_device,
|
||||
)
|
||||
from .voxcpm_runtime_utils import resolve_voxcpm_model_dir
|
||||
from .voxcpm_stage_wrappers import _DirectVoxCPMAudioVAE, _DirectVoxCPMLatentGenerator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_VOXCPM_LATENT_MAGIC = 131071
|
||||
|
||||
|
||||
def _make_voxcpm_model_for_omni(base: type[Any]) -> type[Any]:
|
||||
"""Subclass upstream VoxCPMModel: local ``_inference`` + ``latents_only`` prompt-cache generation."""
|
||||
|
||||
from voxcpm.model.utils import get_dtype
|
||||
|
||||
class VoxCPMModelForOmni(base):
|
||||
@torch.inference_mode()
|
||||
def build_prompt_cache(self, *args: Any, **kwargs: Any):
|
||||
try:
|
||||
return super().build_prompt_cache(*args, **kwargs)
|
||||
except (ImportError, ModuleNotFoundError, RuntimeError) as exc:
|
||||
if not _is_torchcodec_load_error(exc):
|
||||
raise
|
||||
return _build_prompt_cache_with_soundfile(self, *args, **kwargs)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _inference(
|
||||
self,
|
||||
text: torch.Tensor,
|
||||
text_mask: torch.Tensor,
|
||||
feat: torch.Tensor,
|
||||
feat_mask: torch.Tensor,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 3,
|
||||
) -> Generator[tuple[torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]:
|
||||
B, _, _, _ = feat.shape
|
||||
|
||||
feat_embed = self.feat_encoder(feat)
|
||||
feat_embed = self.enc_to_lm_proj(feat_embed)
|
||||
|
||||
scale_emb = self.config.lm_config.scale_emb if self.config.lm_config.use_mup else 1.0
|
||||
text_embed = self.base_lm.embed_tokens(text) * scale_emb
|
||||
combined_embed = text_mask.unsqueeze(-1) * text_embed + feat_mask.unsqueeze(-1) * feat_embed
|
||||
|
||||
prefix_feat_cond = feat[:, -1, ...]
|
||||
pred_feat_seq: list[torch.Tensor] = []
|
||||
|
||||
audio_patch_count = int(feat_mask.sum().item())
|
||||
if audio_patch_count > 0:
|
||||
context_len = min(streaming_prefix_len - 1, audio_patch_count)
|
||||
prompt_context_patches = list(feat[:, -context_len:, :, :].split(1, dim=1))
|
||||
pred_feat_seq = prompt_context_patches + pred_feat_seq
|
||||
|
||||
enc_outputs, kv_cache_tuple = self.base_lm(
|
||||
inputs_embeds=combined_embed,
|
||||
is_causal=True,
|
||||
)
|
||||
self.base_lm.kv_cache.fill_caches(kv_cache_tuple)
|
||||
|
||||
enc_outputs = self.fsq_layer(enc_outputs) * feat_mask.unsqueeze(-1) + enc_outputs * text_mask.unsqueeze(-1)
|
||||
lm_hidden = enc_outputs[:, -1, :]
|
||||
|
||||
residual_enc_outputs, residual_kv_cache_tuple = self.residual_lm(
|
||||
inputs_embeds=enc_outputs + feat_mask.unsqueeze(-1) * feat_embed,
|
||||
is_causal=True,
|
||||
)
|
||||
self.residual_lm.kv_cache.fill_caches(residual_kv_cache_tuple)
|
||||
residual_hidden = residual_enc_outputs[:, -1, :]
|
||||
|
||||
for step_idx in tqdm(range(max_len)):
|
||||
dit_hidden = self.lm_to_dit_proj(lm_hidden) + self.res_to_dit_proj(residual_hidden)
|
||||
pred_feat = self.feat_decoder(
|
||||
mu=dit_hidden,
|
||||
patch_size=self.patch_size,
|
||||
cond=prefix_feat_cond.transpose(1, 2).contiguous(),
|
||||
n_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
).transpose(1, 2)
|
||||
|
||||
curr_embed = self.enc_to_lm_proj(self.feat_encoder(pred_feat.unsqueeze(1)))
|
||||
pred_feat_seq.append(pred_feat.unsqueeze(1))
|
||||
prefix_feat_cond = pred_feat
|
||||
|
||||
if streaming:
|
||||
pred_feat_chunk = torch.cat(pred_feat_seq[-streaming_prefix_len:], dim=1)
|
||||
feat_pred = rearrange(pred_feat_chunk, "b t p d -> b d (t p)", b=B, p=self.patch_size)
|
||||
yield feat_pred, pred_feat_seq
|
||||
|
||||
stop_flag = self.stop_head(self.stop_actn(self.stop_proj(lm_hidden))).argmax(dim=-1)[0].cpu().item()
|
||||
if step_idx > min_len and stop_flag == 1:
|
||||
break
|
||||
|
||||
lm_hidden = self.base_lm.forward_step(
|
||||
curr_embed[:, 0, :],
|
||||
torch.tensor([self.base_lm.kv_cache.step()], device=curr_embed.device),
|
||||
).clone()
|
||||
lm_hidden = self.fsq_layer(lm_hidden)
|
||||
residual_hidden = self.residual_lm.forward_step(
|
||||
lm_hidden + curr_embed[:, 0, :],
|
||||
torch.tensor([self.residual_lm.kv_cache.step()], device=curr_embed.device),
|
||||
).clone()
|
||||
|
||||
if not streaming:
|
||||
pred_feat_seq_cat = torch.cat(pred_feat_seq, dim=1)
|
||||
feat_pred = rearrange(pred_feat_seq_cat, "b t p d -> b d (t p)", b=B, p=self.patch_size)
|
||||
yield feat_pred, pred_feat_seq_cat.squeeze(0).cpu()
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate_latents_with_prompt_cache(
|
||||
self,
|
||||
target_text: str,
|
||||
prompt_cache: dict,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming_prefix_len: int = 3,
|
||||
) -> tuple[None, torch.Tensor, torch.Tensor]:
|
||||
return next(
|
||||
self._generate_with_prompt_cache(
|
||||
target_text=target_text,
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming=False,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
latents_only=True,
|
||||
)
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate_latents_with_prompt_cache_streaming(
|
||||
self,
|
||||
target_text: str,
|
||||
prompt_cache: dict,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming_prefix_len: int = 3,
|
||||
) -> Generator[tuple[None, torch.Tensor, torch.Tensor], None, None]:
|
||||
return self._generate_with_prompt_cache(
|
||||
target_text=target_text,
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming=True,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
latents_only=True,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _generate_with_prompt_cache(
|
||||
self,
|
||||
target_text: str,
|
||||
prompt_cache: dict,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 3,
|
||||
latents_only: bool = False,
|
||||
) -> Generator[tuple[torch.Tensor | None, torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]:
|
||||
if retry_badcase and streaming:
|
||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||
retry_badcase = False
|
||||
if prompt_cache is None:
|
||||
prompt_audio_feat = torch.empty((0, self.patch_size, self.audio_vae.latent_dim), dtype=torch.float32)
|
||||
text = target_text
|
||||
else:
|
||||
prompt_audio_feat = prompt_cache["audio_feat"]
|
||||
prompt_text = prompt_cache["prompt_text"]
|
||||
text = prompt_text + target_text
|
||||
|
||||
text_token = torch.LongTensor(self.text_tokenizer(text))
|
||||
text_token = torch.cat(
|
||||
[
|
||||
text_token,
|
||||
torch.tensor([self.audio_start_token], dtype=torch.int32, device=text_token.device),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
target_text_token = torch.LongTensor(self.text_tokenizer(target_text))
|
||||
|
||||
audio_length = prompt_audio_feat.size(0)
|
||||
text_length = text_token.shape[0]
|
||||
text_pad_token = torch.zeros(audio_length, dtype=torch.int32, device=text_token.device)
|
||||
audio_pad_feat = torch.zeros(
|
||||
(text_token.shape[0], self.patch_size, self.audio_vae.latent_dim),
|
||||
dtype=torch.float32,
|
||||
device=text_token.device,
|
||||
)
|
||||
text_token = torch.cat([text_token, text_pad_token])
|
||||
audio_feat = torch.cat([audio_pad_feat, prompt_audio_feat], dim=0)
|
||||
text_mask = (
|
||||
torch.cat([torch.ones(text_length), torch.zeros(audio_length)]).type(torch.int32).to(text_token.device)
|
||||
)
|
||||
audio_mask = (
|
||||
torch.cat([torch.zeros(text_length), torch.ones(audio_length)]).type(torch.int32).to(text_token.device)
|
||||
)
|
||||
|
||||
text_token = text_token.unsqueeze(0).to(self.device)
|
||||
text_mask = text_mask.unsqueeze(0).to(self.device)
|
||||
audio_feat = audio_feat.unsqueeze(0).to(self.device).to(get_dtype(self.config.dtype))
|
||||
audio_mask = audio_mask.unsqueeze(0).to(self.device)
|
||||
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
retry_badcase_times = 0
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
audio_feat,
|
||||
audio_mask,
|
||||
min_len=min_len,
|
||||
max_len=min(int(target_text_length * retry_badcase_ratio_threshold + 10), max_len),
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
streaming=streaming,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
)
|
||||
if streaming:
|
||||
patch_len = self.patch_size * self.chunk_size
|
||||
for latent_pred, pred_audio_feat in inference_result:
|
||||
if latents_only:
|
||||
decode_audio = None
|
||||
yield (decode_audio, target_text_token, latent_pred)
|
||||
else:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
break
|
||||
|
||||
latent_pred, pred_audio_feat = next(inference_result)
|
||||
if retry_badcase and pred_audio_feat.shape[0] >= target_text_length * retry_badcase_ratio_threshold:
|
||||
ratio = pred_audio_feat.shape[0] / target_text_length
|
||||
print(f" Badcase detected, audio_text_ratio={ratio}, retrying...", file=sys.stderr)
|
||||
retry_badcase_times += 1
|
||||
continue
|
||||
break
|
||||
|
||||
if not streaming:
|
||||
if latents_only:
|
||||
decode_audio = None
|
||||
else:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
patch_len = self.patch_size * self.chunk_size
|
||||
if audio_mask.sum().item() > 0:
|
||||
decode_audio = decode_audio[..., patch_len * (streaming_prefix_len - 1) :].squeeze(1).cpu()
|
||||
else:
|
||||
decode_audio = decode_audio[..., :].squeeze(1).cpu()
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
|
||||
VoxCPMModelForOmni.__name__ = "VoxCPMModelForOmni"
|
||||
VoxCPMModelForOmni.__qualname__ = "VoxCPMModelForOmni"
|
||||
return VoxCPMModelForOmni
|
||||
|
||||
|
||||
def _import_voxcpm_model_class() -> type[Any]:
|
||||
base = _import_voxcpm_base_model_class()
|
||||
return _make_voxcpm_model_for_omni(base)
|
||||
|
||||
|
||||
def _load_native_voxcpm_model(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: str | None,
|
||||
):
|
||||
VoxCPMModel = _import_voxcpm_model_class()
|
||||
model_dir = resolve_voxcpm_model_dir(model_path)
|
||||
runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype=dtype)
|
||||
|
||||
if device.type == "npu" and hasattr(torch, "npu"):
|
||||
torch.npu.set_device(device)
|
||||
|
||||
with _force_cuda_available_for_npu(device):
|
||||
return VoxCPMModel.from_local(
|
||||
runtime_model_path,
|
||||
optimize=device.type == "cuda",
|
||||
)
|
||||
|
||||
|
||||
def _load_native_voxcpm_latent_generator(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: str | None,
|
||||
) -> _DirectVoxCPMLatentGenerator:
|
||||
return _DirectVoxCPMLatentGenerator(_load_native_voxcpm_model(model_path, device=device, dtype=dtype))
|
||||
|
||||
|
||||
def _load_native_voxcpm_audio_vae(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
) -> _DirectVoxCPMAudioVAE:
|
||||
AudioVAE, AudioVAEConfig = _import_voxcpm_audio_vae_classes()
|
||||
model_dir = resolve_voxcpm_model_dir(model_path)
|
||||
runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype="float32")
|
||||
config_dict = json.loads((Path(runtime_model_path) / "config.json").read_text())
|
||||
audio_vae_config = config_dict.get("audio_vae_config")
|
||||
audio_vae = AudioVAE(config=AudioVAEConfig(**audio_vae_config)) if audio_vae_config is not None else AudioVAE()
|
||||
|
||||
state_dict = torch.load(
|
||||
Path(runtime_model_path) / "audiovae.pth",
|
||||
map_location="cpu",
|
||||
weights_only=True,
|
||||
)["state_dict"]
|
||||
audio_vae.load_state_dict(state_dict, strict=True)
|
||||
audio_vae = audio_vae.to(device=device, dtype=torch.float32).eval()
|
||||
if device.type == "npu" and hasattr(torch, "npu"):
|
||||
torch.npu.set_device(device)
|
||||
patch_size = int(config_dict.get("patch_size", 2))
|
||||
return _DirectVoxCPMAudioVAE(audio_vae, patch_size=patch_size)
|
||||
|
||||
|
||||
class VoxCPMForConditionalGeneration(nn.Module):
|
||||
input_modalities = "audio"
|
||||
_LATENT_STAGES = {"latent_generator", "latent", "ar_dit"}
|
||||
_VAE_STAGES = {"vae", "audio_vae"}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
del prefix
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
self.model_stage = getattr(vllm_config.model_config, "model_stage", "latent_generator")
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
self.inject_omni_request_id_into_runtime_info = True
|
||||
self._pipeline = None
|
||||
self._latent_stream_gens: dict[str, Any] = {}
|
||||
self._latent_stream_terminal_pending: dict[str, int] = {}
|
||||
self._latent_stream_completed: set[str] = set()
|
||||
self._next_local_stream_key = 0
|
||||
self._ar_emit_stop_token = True
|
||||
|
||||
def _runner_hidden_device_dtype(self) -> tuple[torch.device, torch.dtype]:
|
||||
device = _resolve_runtime_device(self.vllm_config)
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
dtype = getattr(model_config, "dtype", torch.float32) if model_config is not None else torch.float32
|
||||
return device, dtype
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
if self._pipeline is not None:
|
||||
return
|
||||
|
||||
target_device = _resolve_runtime_device(self.vllm_config)
|
||||
model_dtype = getattr(self.vllm_config.model_config, "dtype", None)
|
||||
normalized_dtype = _normalize_dtype_name(model_dtype)
|
||||
if self.model_stage in self._LATENT_STAGES:
|
||||
self._pipeline = _load_native_voxcpm_latent_generator(
|
||||
self.model_path,
|
||||
device=target_device,
|
||||
dtype=normalized_dtype,
|
||||
)
|
||||
elif self.model_stage in self._VAE_STAGES:
|
||||
self._pipeline = _load_native_voxcpm_audio_vae(
|
||||
self.model_path,
|
||||
device=target_device,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported VoxCPM model_stage: {self.model_stage}. "
|
||||
"pure_voxcpm only supports split-stage latent_generator/vae inference."
|
||||
)
|
||||
|
||||
logger.info("Loaded VoxCPM stage '%s' on %s", self.model_stage, _device_to_string(target_device))
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
del weights
|
||||
self._ensure_model_loaded()
|
||||
return set()
|
||||
|
||||
@staticmethod
|
||||
def _extract_val(info: dict[str, Any], key: str, default: Any) -> Any:
|
||||
value = info.get(key, default)
|
||||
if isinstance(value, list):
|
||||
return value[0] if value else default
|
||||
return value
|
||||
|
||||
def _resolve_stream_request_key(self, info: dict[str, Any]) -> str:
|
||||
request_key = info.get("__voxcpm_stream_key")
|
||||
if request_key is not None:
|
||||
return str(request_key)
|
||||
|
||||
request_key = info.get("_omni_req_id")
|
||||
if request_key is not None:
|
||||
request_key = str(request_key)
|
||||
info["__voxcpm_stream_key"] = request_key
|
||||
return request_key
|
||||
|
||||
request_key = f"voxcpm-local-{self._next_local_stream_key}"
|
||||
self._next_local_stream_key += 1
|
||||
info["__voxcpm_stream_key"] = request_key
|
||||
return str(request_key)
|
||||
|
||||
def _recover_latent_from_input_ids(self, input_ids: torch.Tensor | None) -> torch.Tensor | None:
|
||||
if input_ids is None or input_ids.numel() == 0:
|
||||
return None
|
||||
flat_ids = input_ids.detach().reshape(-1).to("cpu")
|
||||
if flat_ids.numel() < 4 or int(flat_ids[0].item()) != _VOXCPM_LATENT_MAGIC:
|
||||
return None
|
||||
latent_dim = int(flat_ids[1].item())
|
||||
time_dim = int(flat_ids[2].item())
|
||||
payload = flat_ids[3:]
|
||||
expected = latent_dim * time_dim
|
||||
if latent_dim <= 0 or time_dim <= 0:
|
||||
raise ValueError(f"Invalid VoxCPM latent header: latent_dim={latent_dim}, time_dim={time_dim}")
|
||||
if int(payload.numel()) != expected:
|
||||
raise ValueError(
|
||||
"Invalid VoxCPM latent payload size: "
|
||||
f"expected={expected}, actual={int(payload.numel())}, "
|
||||
f"latent_dim={latent_dim}, time_dim={time_dim}"
|
||||
)
|
||||
packed = payload.to(dtype=torch.int32).to(torch.uint16)
|
||||
return packed.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim)
|
||||
|
||||
def _maybe_recover_vae_infos(
|
||||
self,
|
||||
infos: list[dict[str, Any]],
|
||||
input_ids: torch.Tensor | None,
|
||||
*,
|
||||
async_chunk: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if not async_chunk:
|
||||
return infos
|
||||
if any(self._extract_val(info, "latent_audio_feat", None) is not None for info in infos):
|
||||
return infos
|
||||
recovered = self._recover_latent_from_input_ids(input_ids)
|
||||
if recovered is None:
|
||||
return infos
|
||||
return [{"latent_audio_feat": recovered}]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_audio_samples(samples: Any) -> np.ndarray:
|
||||
if isinstance(samples, torch.Tensor):
|
||||
return samples.detach().cpu().float().reshape(-1).numpy()
|
||||
return np.asarray(samples, dtype=np.float32).reshape(-1)
|
||||
|
||||
@classmethod
|
||||
def _normalize_ref_audio(cls, ref_audio: Any) -> tuple[np.ndarray, int]:
|
||||
if isinstance(ref_audio, str):
|
||||
raise TypeError("String ref_audio should be handled as a path before waveform normalization.")
|
||||
|
||||
if isinstance(ref_audio, dict):
|
||||
sample_rate = ref_audio.get("sample_rate") or ref_audio.get("sampling_rate") or ref_audio.get("sr")
|
||||
samples = None
|
||||
for key in ("audio", "wav", "samples", "array", "waveform"):
|
||||
if key in ref_audio and ref_audio[key] is not None:
|
||||
samples = ref_audio[key]
|
||||
break
|
||||
if sample_rate is None or samples is None:
|
||||
raise ValueError("ref_audio dict must contain waveform data and sample rate.")
|
||||
return cls._normalize_audio_samples(samples), int(sample_rate)
|
||||
|
||||
if isinstance(ref_audio, (list, tuple)):
|
||||
if len(ref_audio) == 1:
|
||||
return cls._normalize_ref_audio(ref_audio[0])
|
||||
if len(ref_audio) == 2 and np.isscalar(ref_audio[1]):
|
||||
return cls._normalize_audio_samples(ref_audio[0]), int(ref_audio[1])
|
||||
|
||||
raise TypeError(f"Unsupported ref_audio format: {type(ref_audio)!r}")
|
||||
|
||||
@staticmethod
|
||||
def _write_temp_prompt_wav(waveform: np.ndarray, sample_rate: int) -> str:
|
||||
prompt_file = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
|
||||
prompt_file.close()
|
||||
|
||||
wav = np.asarray(waveform, dtype=np.float32).reshape(-1)
|
||||
wav = np.clip(wav, -1.0, 1.0)
|
||||
pcm16 = (wav * 32767.0).astype(np.int16)
|
||||
with wave.open(prompt_file.name, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(int(sample_rate))
|
||||
wav_file.writeframes(pcm16.tobytes())
|
||||
|
||||
return prompt_file.name
|
||||
|
||||
@classmethod
|
||||
def _resolve_prompt_inputs(cls, info: dict[str, Any]) -> tuple[str | None, str | None, str | None]:
|
||||
prompt_text = cls._extract_val(info, "prompt_text", None)
|
||||
prompt_wav_path = cls._extract_val(info, "prompt_wav_path", None)
|
||||
if prompt_wav_path:
|
||||
if prompt_text is None:
|
||||
prompt_text = cls._extract_val(info, "ref_text", None)
|
||||
return prompt_wav_path, prompt_text, None
|
||||
|
||||
ref_audio = cls._extract_val(info, "ref_audio", None)
|
||||
ref_text = cls._extract_val(info, "ref_text", None)
|
||||
if ref_audio is None or ref_text is None:
|
||||
return None, None, None
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio, ref_text, None
|
||||
|
||||
waveform, sample_rate = cls._normalize_ref_audio(ref_audio)
|
||||
temp_prompt_wav = cls._write_temp_prompt_wav(waveform, sample_rate)
|
||||
return temp_prompt_wav, ref_text, temp_prompt_wav
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def _get_vocab_size(self) -> int:
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
if model_config is not None:
|
||||
getter = getattr(model_config, "get_vocab_size", None)
|
||||
if callable(getter):
|
||||
try:
|
||||
return int(getter())
|
||||
except Exception:
|
||||
pass
|
||||
hf_config = getattr(model_config, "hf_text_config", None)
|
||||
if hf_config is not None and hasattr(hf_config, "vocab_size"):
|
||||
return int(hf_config.vocab_size)
|
||||
return 32000
|
||||
|
||||
def _make_empty_output(
|
||||
self,
|
||||
*,
|
||||
output_key: str,
|
||||
payload_factory: Callable[[], torch.Tensor],
|
||||
infos: list[dict[str, Any]],
|
||||
sample_rate: int,
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
hidden_rows: int | None = None,
|
||||
) -> OmniOutput:
|
||||
if hidden_rows is None:
|
||||
hidden_rows = len(infos)
|
||||
return OmniOutput(
|
||||
text_hidden_states=torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype),
|
||||
multimodal_outputs={
|
||||
output_key: [payload_factory() for _ in infos],
|
||||
"sr": [torch.tensor(sample_rate, dtype=torch.int32) for _ in infos],
|
||||
},
|
||||
)
|
||||
|
||||
def _finalize_stage_output(
|
||||
self,
|
||||
*,
|
||||
output_key: str,
|
||||
outputs: list[torch.Tensor],
|
||||
sample_rates: list[torch.Tensor],
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
hidden_rows: int | None = None,
|
||||
) -> OmniOutput:
|
||||
multimodal_outputs: dict[str, Any] = {output_key: outputs, "sr": sample_rates}
|
||||
if hidden_rows is not None:
|
||||
text_hidden_states = torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype)
|
||||
elif outputs:
|
||||
outputs_tensor = torch.stack(outputs)
|
||||
text_hidden_states = (
|
||||
outputs_tensor.unsqueeze(-1)
|
||||
if outputs_tensor.ndim == 1
|
||||
else outputs_tensor.reshape(-1, outputs_tensor.shape[-1])
|
||||
)
|
||||
else:
|
||||
text_hidden_states = torch.zeros((0, 1), device=out_device, dtype=out_dtype)
|
||||
text_hidden_states = text_hidden_states.to(device=out_device, dtype=out_dtype)
|
||||
return OmniOutput(
|
||||
text_hidden_states=text_hidden_states,
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
)
|
||||
|
||||
def _forward_vae_stage(
|
||||
self,
|
||||
infos: list[dict[str, Any]],
|
||||
*,
|
||||
sample_rate: int,
|
||||
async_chunk: bool,
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
) -> OmniOutput:
|
||||
if all(self._extract_val(info, "latent_audio_feat", None) is None for info in infos):
|
||||
self._ar_emit_stop_token = True
|
||||
return self._make_empty_output(
|
||||
output_key="model_outputs",
|
||||
payload_factory=lambda: torch.zeros((0,), dtype=torch.float32),
|
||||
infos=infos,
|
||||
sample_rate=sample_rate,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
outputs: list[torch.Tensor] = []
|
||||
sample_rates: list[torch.Tensor] = []
|
||||
for info in infos:
|
||||
latent_audio_feat = self._extract_val(info, "latent_audio_feat", None)
|
||||
audio_tensor = self._pipeline.decode(latent_audio_feat, trim_streaming_patch=async_chunk)
|
||||
outputs.append(audio_tensor.float().cpu())
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
|
||||
self._ar_emit_stop_token = True
|
||||
return self._finalize_stage_output(
|
||||
output_key="model_outputs",
|
||||
outputs=outputs,
|
||||
sample_rates=sample_rates,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
def _forward_latent_stage(
|
||||
self,
|
||||
infos: list[dict[str, Any]],
|
||||
*,
|
||||
sample_rate: int,
|
||||
async_chunk: bool,
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
hidden_rows: int,
|
||||
) -> OmniOutput:
|
||||
texts = [self._extract_val(info, "text", "") for info in infos]
|
||||
if all(not text for text in texts):
|
||||
self._ar_emit_stop_token = True
|
||||
return self._make_empty_output(
|
||||
output_key="latent_audio_feat",
|
||||
payload_factory=lambda: torch.zeros((0,), dtype=torch.float32),
|
||||
infos=infos,
|
||||
sample_rate=sample_rate,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
hidden_rows=hidden_rows,
|
||||
)
|
||||
|
||||
outputs: list[torch.Tensor] = []
|
||||
sample_rates: list[torch.Tensor] = []
|
||||
last_chunk_flags: list[bool] | None = [] if async_chunk else None
|
||||
payload_finished_flags: list[bool] | None = [] if async_chunk else None
|
||||
for info in infos:
|
||||
text = self._extract_val(info, "text", "")
|
||||
cfg_value = float(self._extract_val(info, "cfg_value", 2.0))
|
||||
inference_timesteps = int(self._extract_val(info, "inference_timesteps", 10))
|
||||
min_len = int(self._extract_val(info, "min_len", 2))
|
||||
max_len = int(self._extract_val(info, "max_len", self._extract_val(info, "max_new_tokens", 4096)))
|
||||
retry_badcase = bool(self._extract_val(info, "retry_badcase", True))
|
||||
retry_badcase_max_times = int(self._extract_val(info, "retry_badcase_max_times", 3))
|
||||
retry_badcase_ratio_threshold = float(self._extract_val(info, "retry_badcase_ratio_threshold", 6.0))
|
||||
streaming_prefix_len = int(self._extract_val(info, "streaming_prefix_len", 3))
|
||||
|
||||
request_key = self._resolve_stream_request_key(info)
|
||||
created_temp: str | None = None
|
||||
|
||||
if async_chunk:
|
||||
terminal_pending = self._latent_stream_terminal_pending.get(request_key, 0)
|
||||
if terminal_pending > 0:
|
||||
outputs.append(torch.zeros((0,), dtype=torch.float32))
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(True)
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(terminal_pending == 1)
|
||||
if terminal_pending == 1:
|
||||
self._latent_stream_terminal_pending.pop(request_key, None)
|
||||
else:
|
||||
self._latent_stream_terminal_pending[request_key] = terminal_pending - 1
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
continue
|
||||
|
||||
if request_key in self._latent_stream_completed:
|
||||
outputs.append(torch.zeros((0,), dtype=torch.float32))
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(True)
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(False)
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
continue
|
||||
|
||||
if request_key not in self._latent_stream_gens:
|
||||
prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info)
|
||||
created_temp = temp_prompt_wav
|
||||
self._latent_stream_gens[request_key] = self._pipeline.iter_latent_chunks_streaming(
|
||||
text=text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
prompt_text=prompt_text,
|
||||
cfg_value=cfg_value,
|
||||
inference_timesteps=inference_timesteps,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
retry_badcase=False,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
)
|
||||
generator = self._latent_stream_gens[request_key]
|
||||
try:
|
||||
chunk_latent, is_last = next(generator)
|
||||
except StopIteration:
|
||||
self._latent_stream_gens.pop(request_key, None)
|
||||
self._latent_stream_terminal_pending[request_key] = 1
|
||||
self._latent_stream_completed.add(request_key)
|
||||
outputs.append(torch.zeros((0,), dtype=torch.float32))
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(True)
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(True)
|
||||
else:
|
||||
if is_last:
|
||||
self._latent_stream_gens.pop(request_key, None)
|
||||
self._latent_stream_terminal_pending[request_key] = 1
|
||||
self._latent_stream_completed.add(request_key)
|
||||
outputs.append(chunk_latent.detach().float().cpu())
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(bool(is_last))
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(False)
|
||||
finally:
|
||||
if created_temp is not None and os.path.exists(created_temp):
|
||||
os.unlink(created_temp)
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
continue
|
||||
|
||||
prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info)
|
||||
try:
|
||||
latent_audio_feat = self._pipeline.generate_latents(
|
||||
text=text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
prompt_text=prompt_text,
|
||||
cfg_value=cfg_value,
|
||||
inference_timesteps=inference_timesteps,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
)
|
||||
outputs.append(latent_audio_feat.float().cpu())
|
||||
finally:
|
||||
if temp_prompt_wav is not None and os.path.exists(temp_prompt_wav):
|
||||
os.unlink(temp_prompt_wav)
|
||||
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
|
||||
self._ar_emit_stop_token = all(last_chunk_flags) if async_chunk and last_chunk_flags else True
|
||||
output = self._finalize_stage_output(
|
||||
output_key="latent_audio_feat",
|
||||
outputs=outputs,
|
||||
sample_rates=sample_rates,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
hidden_rows=hidden_rows,
|
||||
)
|
||||
if async_chunk and payload_finished_flags is not None:
|
||||
output.multimodal_outputs["finished"] = [
|
||||
torch.tensor(flag, dtype=torch.bool) for flag in payload_finished_flags
|
||||
]
|
||||
return output
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> torch.Tensor:
|
||||
del sampling_metadata
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if hidden_states is None:
|
||||
device, dtype = self._runner_hidden_device_dtype()
|
||||
hidden_states = torch.zeros((0, 1), device=device, dtype=dtype)
|
||||
if hidden_states.ndim == 1:
|
||||
hidden_states = hidden_states.unsqueeze(-1)
|
||||
elif hidden_states.ndim > 2:
|
||||
hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
|
||||
vocab_size = self._get_vocab_size()
|
||||
num_rows = int(hidden_states.shape[0])
|
||||
logits = torch.zeros((num_rows, vocab_size), dtype=torch.float32, device=hidden_states.device)
|
||||
eos_id = 2 if vocab_size > 2 else 0
|
||||
safe_id = 1 if vocab_size > 1 and 1 != eos_id else 0
|
||||
emit_stop = getattr(self, "_ar_emit_stop_token", True)
|
||||
if num_rows > 0:
|
||||
if emit_stop:
|
||||
logits[:, eos_id] = 1.0e6
|
||||
else:
|
||||
logits[:, eos_id] = -1.0e9
|
||||
logits[:, safe_id] = 1.0e6
|
||||
return logits
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
model_intermediate_buffer: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
del positions, intermediate_tensors, inputs_embeds, kwargs
|
||||
self._ensure_model_loaded()
|
||||
out_device, out_dtype = self._runner_hidden_device_dtype()
|
||||
if input_ids is not None and input_ids.device.type == out_device.type:
|
||||
out_device = input_ids.device
|
||||
|
||||
infos = model_intermediate_buffer or runtime_additional_information or [{}]
|
||||
hidden_rows = len(infos)
|
||||
if input_ids is not None and len(input_ids.shape) > 0:
|
||||
hidden_rows = max(hidden_rows, int(input_ids.shape[0]))
|
||||
sample_rate = int(getattr(self._pipeline, "sample_rate", 24000))
|
||||
async_chunk = bool(getattr(self.vllm_config.model_config, "async_chunk", False))
|
||||
if self.model_stage in self._VAE_STAGES:
|
||||
infos = self._maybe_recover_vae_infos(infos, input_ids, async_chunk=async_chunk)
|
||||
return self._forward_vae_stage(
|
||||
infos,
|
||||
sample_rate=sample_rate,
|
||||
async_chunk=async_chunk,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
if self.model_stage in self._LATENT_STAGES:
|
||||
return self._forward_latent_stage(
|
||||
infos,
|
||||
sample_rate=sample_rate,
|
||||
async_chunk=async_chunk,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
hidden_rows=hidden_rows,
|
||||
)
|
||||
raise ValueError(f"Unsupported VoxCPM model_stage at runtime: {self.model_stage}")
|
||||
|
||||
def make_empty_intermediate_tensors(
|
||||
self, batch_size: int, dtype: torch.dtype, device: torch.device
|
||||
) -> IntermediateTensors:
|
||||
del batch_size, dtype, device
|
||||
return {}
|
||||
|
||||
|
||||
__all__ = ["VoxCPMForConditionalGeneration"]
|
||||
@@ -0,0 +1,247 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _iter_voxcpm_src_candidates() -> list[Path]:
|
||||
candidates: list[Path] = []
|
||||
env_path = os.environ.get("VLLM_OMNI_VOXCPM_CODE_PATH")
|
||||
if env_path:
|
||||
candidates.append(Path(env_path).expanduser())
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
candidates.append(repo_root.parent / "VoxCPM" / "src")
|
||||
|
||||
unique_candidates: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
candidate_key = str(candidate)
|
||||
if candidate_key in seen:
|
||||
continue
|
||||
seen.add(candidate_key)
|
||||
unique_candidates.append(candidate)
|
||||
return unique_candidates
|
||||
|
||||
|
||||
def _prepend_voxcpm_src(candidate: Path) -> None:
|
||||
candidate_str = str(candidate)
|
||||
if candidate_str not in sys.path:
|
||||
sys.path.insert(0, candidate_str)
|
||||
|
||||
|
||||
def _import_voxcpm_attrs(module_name: str, *attr_names: str) -> tuple[Any, ...]:
|
||||
last_exc: ImportError | None = None
|
||||
for candidate in _iter_voxcpm_src_candidates():
|
||||
if not candidate.exists():
|
||||
continue
|
||||
_prepend_voxcpm_src(candidate)
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
return tuple(getattr(module, attr_name) for attr_name in attr_names)
|
||||
except ImportError as exc:
|
||||
last_exc = exc
|
||||
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
return tuple(getattr(module, attr_name) for attr_name in attr_names)
|
||||
except ImportError as exc:
|
||||
last_exc = exc
|
||||
|
||||
raise ImportError(f"Failed to import {module_name}.") from last_exc
|
||||
|
||||
|
||||
def _import_voxcpm_base_model_class():
|
||||
"""Import upstream ``VoxCPMModel`` from ``VoxCPM/src/voxcpm`` (env, sibling tree, or pip)."""
|
||||
try:
|
||||
(VoxCPMModel,) = _import_voxcpm_attrs("voxcpm.model.voxcpm", "VoxCPMModel")
|
||||
return VoxCPMModel
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Failed to import VoxCPMModel. Install the `voxcpm` package or set "
|
||||
"`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory "
|
||||
"(the parent of the `voxcpm` package that contains `model/` and `modules/`)."
|
||||
) from exc
|
||||
|
||||
|
||||
def _import_voxcpm_audio_vae_classes():
|
||||
try:
|
||||
return _import_voxcpm_attrs("voxcpm.modules.audiovae", "AudioVAE", "AudioVAEConfig")
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Failed to import VoxCPM AudioVAE. Install the `voxcpm` package or set "
|
||||
"`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory."
|
||||
) from exc
|
||||
|
||||
|
||||
def _device_to_string(device: torch.device) -> str:
|
||||
if device.index is None:
|
||||
return device.type
|
||||
return f"{device.type}:{device.index}"
|
||||
|
||||
|
||||
def _normalize_dtype_name(dtype: Any) -> str | None:
|
||||
if dtype is None:
|
||||
return None
|
||||
if isinstance(dtype, torch.dtype):
|
||||
mapping = {
|
||||
torch.bfloat16: "bfloat16",
|
||||
torch.float16: "float16",
|
||||
torch.float32: "float32",
|
||||
}
|
||||
return mapping.get(dtype, str(dtype).removeprefix("torch."))
|
||||
dtype_str = str(dtype)
|
||||
return dtype_str.removeprefix("torch.")
|
||||
|
||||
|
||||
def _resolve_runtime_device(vllm_config: VllmConfig) -> torch.device:
|
||||
try:
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
return current_omni_platform.get_torch_device()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
device = getattr(getattr(vllm_config, "device_config", None), "device", None)
|
||||
if isinstance(device, torch.device):
|
||||
return device
|
||||
if device:
|
||||
return torch.device(device)
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def _prepare_runtime_model_dir(
|
||||
model_path: str | Path,
|
||||
*,
|
||||
target_device: torch.device,
|
||||
target_dtype: str | None,
|
||||
) -> str:
|
||||
source_dir = Path(model_path)
|
||||
config_path = source_dir / "config.json"
|
||||
if not config_path.exists():
|
||||
return str(source_dir)
|
||||
|
||||
config_text = config_path.read_text()
|
||||
config_dict = json.loads(config_text)
|
||||
desired_device = target_device.type
|
||||
desired_dtype = target_dtype or config_dict.get("dtype")
|
||||
|
||||
if config_dict.get("device") == desired_device and config_dict.get("dtype") == desired_dtype:
|
||||
return str(source_dir)
|
||||
|
||||
digest = sha256(f"{source_dir.resolve()}:{config_text}:{desired_device}:{desired_dtype}".encode()).hexdigest()[:16]
|
||||
runtime_dir = Path(tempfile.gettempdir()) / "vllm_omni_voxcpm_runtime" / digest
|
||||
runtime_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for entry in source_dir.iterdir():
|
||||
target = runtime_dir / entry.name
|
||||
if entry.name == "config.json" or target.exists():
|
||||
continue
|
||||
try:
|
||||
target.symlink_to(entry, target_is_directory=entry.is_dir())
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Falling back to copying VoxCPM runtime artifact %s into %s because symlink creation failed: %s",
|
||||
entry,
|
||||
runtime_dir,
|
||||
exc,
|
||||
)
|
||||
if entry.is_dir():
|
||||
shutil.copytree(entry, target, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(entry, target)
|
||||
|
||||
patched_config = dict(config_dict)
|
||||
patched_config["device"] = desired_device
|
||||
if desired_dtype is not None:
|
||||
patched_config["dtype"] = desired_dtype
|
||||
(runtime_dir / "config.json").write_text(json.dumps(patched_config, indent=2, sort_keys=True))
|
||||
return str(runtime_dir)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _force_cuda_available_for_npu(device: torch.device):
|
||||
if device.type != "npu":
|
||||
yield
|
||||
return
|
||||
|
||||
with patch("torch.cuda.is_available", return_value=True):
|
||||
yield
|
||||
|
||||
|
||||
def _is_torchcodec_load_error(exc: BaseException) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "torchcodec" in message or "load_with_torchcodec" in message
|
||||
|
||||
|
||||
def _load_audio_with_soundfile(
|
||||
prompt_wav_path: str,
|
||||
*,
|
||||
sample_rate: int,
|
||||
) -> torch.Tensor:
|
||||
try:
|
||||
import soundfile as sf
|
||||
except ImportError:
|
||||
raise
|
||||
|
||||
audio_np, source_sr = sf.read(prompt_wav_path, dtype="float32", always_2d=True)
|
||||
audio = torch.from_numpy(np.ascontiguousarray(audio_np.T))
|
||||
|
||||
if audio.size(0) > 1:
|
||||
audio = audio.mean(dim=0, keepdim=True)
|
||||
|
||||
if int(source_sr) != int(sample_rate):
|
||||
try:
|
||||
import torchaudio
|
||||
except ImportError as exc:
|
||||
raise ImportError("torchaudio is required for resampling prompt audio.") from exc
|
||||
audio = torchaudio.functional.resample(audio, int(source_sr), int(sample_rate))
|
||||
|
||||
return audio
|
||||
|
||||
|
||||
def _build_prompt_cache_with_soundfile(model: Any, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
if args:
|
||||
prompt_text = args[0]
|
||||
prompt_wav_path = args[1] if len(args) > 1 else kwargs.get("prompt_wav_path")
|
||||
else:
|
||||
prompt_text = kwargs.get("prompt_text")
|
||||
prompt_wav_path = kwargs.get("prompt_wav_path")
|
||||
|
||||
if not prompt_text or not prompt_wav_path:
|
||||
raise ValueError("prompt_text and prompt_wav_path are required")
|
||||
|
||||
audio = _load_audio_with_soundfile(prompt_wav_path, sample_rate=int(model.sample_rate))
|
||||
|
||||
patch_len = model.patch_size * model.chunk_size
|
||||
if audio.size(1) % patch_len != 0:
|
||||
padding_size = patch_len - audio.size(1) % patch_len
|
||||
audio = torch.nn.functional.pad(audio, (padding_size, 0))
|
||||
|
||||
audio_feat = model.audio_vae.encode(audio.to(model.device), model.sample_rate).cpu()
|
||||
audio_feat = audio_feat.view(
|
||||
model.audio_vae.latent_dim,
|
||||
-1,
|
||||
model.patch_size,
|
||||
).permute(1, 2, 0)
|
||||
|
||||
return {
|
||||
"prompt_text": prompt_text,
|
||||
"audio_feat": audio_feat,
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def resolve_voxcpm_model_dir(model: str) -> Path:
|
||||
model_path = Path(model).expanduser()
|
||||
if model_path.exists():
|
||||
return model_path
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
return Path(snapshot_download(repo_id=model))
|
||||
|
||||
|
||||
def prepare_voxcpm_hf_config_dir(model_dir: str | Path, hf_config_dir: str | Path) -> Path:
|
||||
model_dir = Path(model_dir).expanduser()
|
||||
hf_config_dir = Path(hf_config_dir).expanduser()
|
||||
hf_config_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
source_config_path = model_dir / "config.json"
|
||||
if not source_config_path.exists():
|
||||
raise FileNotFoundError(f"VoxCPM config.json not found under {model_dir}")
|
||||
|
||||
config_path = hf_config_dir / "config.json"
|
||||
shutil.copy2(source_config_path, config_path)
|
||||
|
||||
source_generation_config_path = model_dir / "generation_config.json"
|
||||
if source_generation_config_path.exists():
|
||||
shutil.copy2(source_generation_config_path, hf_config_dir / "generation_config.json")
|
||||
|
||||
config_dict = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
config_dict["model_type"] = "voxcpm"
|
||||
config_dict.setdefault("architectures", ["VoxCPMForConditionalGeneration"])
|
||||
config_path.write_text(json.dumps(config_dict, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
return hf_config_dir
|
||||
|
||||
|
||||
__all__ = [
|
||||
"prepare_voxcpm_hf_config_dir",
|
||||
"resolve_voxcpm_model_dir",
|
||||
]
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
|
||||
|
||||
class _DirectVoxCPMLatentGenerator:
|
||||
def __init__(self, tts_model: Any):
|
||||
self.tts_model = tts_model
|
||||
self.sample_rate = int(getattr(tts_model, "sample_rate", 24000))
|
||||
|
||||
def generate_latents(
|
||||
self,
|
||||
*,
|
||||
text: str,
|
||||
prompt_wav_path: str | None = None,
|
||||
prompt_text: str | None = None,
|
||||
cfg_value: float = 2.0,
|
||||
inference_timesteps: int = 10,
|
||||
min_len: int = 2,
|
||||
max_len: int = 4096,
|
||||
retry_badcase: bool = True,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
) -> torch.Tensor:
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("target text must be a non-empty string")
|
||||
if (prompt_wav_path is None) != (prompt_text is None):
|
||||
raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None")
|
||||
if prompt_wav_path is not None and not os.path.exists(prompt_wav_path):
|
||||
raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}")
|
||||
|
||||
prompt_cache = None
|
||||
if prompt_wav_path is not None and prompt_text is not None:
|
||||
prompt_cache = self.tts_model.build_prompt_cache(
|
||||
prompt_text=prompt_text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
)
|
||||
|
||||
gen_kw = dict(
|
||||
target_text=" ".join(text.split()),
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
)
|
||||
latent_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache", None)
|
||||
if latent_entry is not None:
|
||||
_, _, pred_audio_feat = latent_entry(**gen_kw)
|
||||
else:
|
||||
try:
|
||||
_, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(
|
||||
**gen_kw,
|
||||
latents_only=True,
|
||||
)
|
||||
except TypeError:
|
||||
_, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(**gen_kw)
|
||||
return pred_audio_feat.detach().cpu().to(torch.float32)
|
||||
|
||||
def iter_latent_chunks_streaming(
|
||||
self,
|
||||
*,
|
||||
text: str,
|
||||
prompt_wav_path: str | None = None,
|
||||
prompt_text: str | None = None,
|
||||
cfg_value: float = 2.0,
|
||||
inference_timesteps: int = 10,
|
||||
min_len: int = 2,
|
||||
max_len: int = 4096,
|
||||
streaming_prefix_len: int = 3,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
) -> Generator[tuple[torch.Tensor, bool], None, None]:
|
||||
"""Yield ``(latent_window, is_last_chunk)`` for Omni async_chunk latent to VAE."""
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("target text must be a non-empty string")
|
||||
if (prompt_wav_path is None) != (prompt_text is None):
|
||||
raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None")
|
||||
if prompt_wav_path is not None and not os.path.exists(prompt_wav_path):
|
||||
raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}")
|
||||
|
||||
prompt_cache = None
|
||||
if prompt_wav_path is not None and prompt_text is not None:
|
||||
prompt_cache = self.tts_model.build_prompt_cache(
|
||||
prompt_text=prompt_text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
)
|
||||
|
||||
gen_kw = dict(
|
||||
target_text=" ".join(text.split()),
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
)
|
||||
stream_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache_streaming", None)
|
||||
if stream_entry is not None:
|
||||
gen = stream_entry(**gen_kw)
|
||||
else:
|
||||
fallback_stream_entry = getattr(self.tts_model, "generate_with_prompt_cache_streaming", None)
|
||||
if fallback_stream_entry is not None:
|
||||
gen = fallback_stream_entry(**gen_kw, latents_only=True)
|
||||
else:
|
||||
gen = self.tts_model._generate_with_prompt_cache(streaming=True, latents_only=True, **gen_kw)
|
||||
|
||||
iterator = iter(gen)
|
||||
previous = next(iterator, None)
|
||||
while previous is not None:
|
||||
current = next(iterator, None)
|
||||
_, _target_tok, chunk_latent = previous
|
||||
if not isinstance(chunk_latent, torch.Tensor):
|
||||
chunk_latent = torch.as_tensor(chunk_latent)
|
||||
yield chunk_latent, current is None
|
||||
previous = current
|
||||
|
||||
|
||||
class _DirectVoxCPMAudioVAE:
|
||||
def __init__(self, audio_vae: nn.Module, *, patch_size: int = 2):
|
||||
self.audio_vae = audio_vae
|
||||
self.sample_rate = int(getattr(audio_vae, "sample_rate", 24000))
|
||||
self.latent_dim = int(getattr(audio_vae, "latent_dim", 64))
|
||||
self.patch_size = int(patch_size)
|
||||
self._chunk_size = int(getattr(audio_vae, "chunk_size", 1))
|
||||
self._stream_audio_patch_samples = max(1, self.patch_size * self._chunk_size)
|
||||
|
||||
def _prepare_latents_for_decode(self, latent_audio_feat: Any) -> torch.Tensor:
|
||||
latents = latent_audio_feat
|
||||
if not isinstance(latents, torch.Tensor):
|
||||
latents = torch.tensor(latents, dtype=torch.float32)
|
||||
latents = latents.detach().to(torch.float32)
|
||||
|
||||
if latents.ndim == 3:
|
||||
if latents.shape[-1] == self.latent_dim:
|
||||
latents = rearrange(latents, "t p d -> 1 d (t p)")
|
||||
elif latents.shape[1] == self.latent_dim:
|
||||
latents = latents.contiguous()
|
||||
else:
|
||||
raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}")
|
||||
elif latents.ndim == 2:
|
||||
if latents.shape[0] == self.latent_dim:
|
||||
latents = latents.unsqueeze(0)
|
||||
elif latents.shape[1] == self.latent_dim:
|
||||
latents = rearrange(latents, "t d -> 1 d t")
|
||||
else:
|
||||
raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}")
|
||||
else:
|
||||
raise ValueError(f"Unsupported latent_audio_feat ndim: {latents.ndim}")
|
||||
|
||||
return latents
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latent_audio_feat: Any, *, trim_streaming_patch: bool = False) -> torch.Tensor:
|
||||
latents = self._prepare_latents_for_decode(latent_audio_feat)
|
||||
device = next(self.audio_vae.parameters()).device
|
||||
raw = self.audio_vae.decode(latents.to(device=device, dtype=torch.float32))
|
||||
if isinstance(raw, dict):
|
||||
audio = raw.get("audio")
|
||||
if audio is None:
|
||||
audio = next(v for v in raw.values() if isinstance(v, torch.Tensor))
|
||||
else:
|
||||
audio = raw
|
||||
if audio.dim() == 3:
|
||||
stream = audio.squeeze(1)
|
||||
elif audio.dim() == 2:
|
||||
stream = audio
|
||||
else:
|
||||
stream = audio.reshape(audio.shape[0], -1)
|
||||
if trim_streaming_patch:
|
||||
stream = stream[..., -self._stream_audio_patch_samples :]
|
||||
return stream.reshape(-1).detach().cpu().to(torch.float32)
|
||||
@@ -307,32 +307,39 @@ class MiniCPM4PagedForVoxCPM2(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def precompute_fused_qkv(self) -> None:
|
||||
"""Materialize fused QKV weights before CUDA Graph capture."""
|
||||
for layer in self.layers:
|
||||
attn = layer.self_attn
|
||||
if attn._fused_qkv_weight is None:
|
||||
attn._fused_qkv_weight = torch.cat(
|
||||
[attn.q_proj.weight, attn.k_proj.weight, attn.v_proj.weight],
|
||||
dim=0,
|
||||
).detach()
|
||||
|
||||
def compile_selective(self) -> list[str]:
|
||||
"""Compile MLP + o_proj; keep RMSNorm/RoPE eager for precision."""
|
||||
compiled: list[str] = []
|
||||
for i, layer in enumerate(self.layers):
|
||||
if i in self._compiled_layers:
|
||||
continue
|
||||
try:
|
||||
layer.mlp = torch.compile(
|
||||
layer.mlp,
|
||||
mode="default",
|
||||
fullgraph=True,
|
||||
)
|
||||
layer.self_attn.o_proj = torch.compile(
|
||||
layer.self_attn.o_proj,
|
||||
mode="default",
|
||||
fullgraph=True,
|
||||
)
|
||||
layer.self_attn._fused_qkv_weight = None
|
||||
self._compiled_layers.add(i)
|
||||
if i == 0:
|
||||
compiled.append(f"layers.*.mlp (×{len(self.layers)})")
|
||||
compiled.append(f"layers.*.self_attn.o_proj (×{len(self.layers)})")
|
||||
except Exception as e:
|
||||
logger.warning("compile_selective: layer %d failed: %s", i, e)
|
||||
break
|
||||
return compiled
|
||||
"""Compile the full model forward as one graph.
|
||||
|
||||
Earlier versions compiled ``layer.mlp`` + ``layer.self_attn.o_proj``
|
||||
(PR #2690) and then the whole ``layer`` (perf/voxcpm2-streaming-vae).
|
||||
Both still paid one Dynamo dispatch per layer per decode step.
|
||||
V3 profiling showed 1,332 per-layer dispatches (~28 layers × ~47
|
||||
decode steps) costing ~726 ms of CPU self-time for a long prompt.
|
||||
|
||||
Compiling ``forward`` at the model level lets Dynamo unroll the
|
||||
28-layer Python loop inside the graph. Graph breaks at
|
||||
PagedAttention produce sub-graphs but Dynamo memoises the whole
|
||||
trace once, so the per-step dispatch drops from 28 to just a few.
|
||||
"""
|
||||
if self._compiled_layers:
|
||||
return []
|
||||
# Null the fused-qkv caches so the compile sees the real weight layout.
|
||||
for layer in self.layers:
|
||||
layer.self_attn._fused_qkv_weight = None
|
||||
self.forward = torch.compile(self.forward, mode="default", fullgraph=False)
|
||||
# Mark every layer as compiled so idempotent callers don't double-wrap.
|
||||
self._compiled_layers.update(range(len(self.layers)))
|
||||
return ["forward (whole model)"]
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights from native checkpoint (base_lm. prefix pre-stripped)."""
|
||||
@@ -414,23 +421,25 @@ class MiniCPM4PagedResidualLM(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def precompute_fused_qkv(self) -> None:
|
||||
"""Materialize fused QKV weights before CUDA Graph capture."""
|
||||
for layer in self.layers:
|
||||
attn = layer.self_attn
|
||||
if attn._fused_qkv_weight is None:
|
||||
attn._fused_qkv_weight = torch.cat(
|
||||
[attn.q_proj.weight, attn.k_proj.weight, attn.v_proj.weight],
|
||||
dim=0,
|
||||
).detach()
|
||||
|
||||
def compile_selective(self) -> list[str]:
|
||||
"""Compile MLP + o_proj (same as base_lm)."""
|
||||
compiled: list[str] = []
|
||||
for i, layer in enumerate(self.layers):
|
||||
if i in self._compiled_layers:
|
||||
continue
|
||||
try:
|
||||
layer.mlp = torch.compile(layer.mlp, mode="default", fullgraph=True)
|
||||
layer.self_attn.o_proj = torch.compile(layer.self_attn.o_proj, mode="default", fullgraph=True)
|
||||
layer.self_attn._fused_qkv_weight = None
|
||||
self._compiled_layers.add(i)
|
||||
if i == 0:
|
||||
compiled.append(f"layers.*.mlp (×{len(self.layers)})")
|
||||
compiled.append(f"layers.*.self_attn.o_proj (×{len(self.layers)})")
|
||||
except Exception as e:
|
||||
logger.warning("compile_selective: residual layer %d failed: %s", i, e)
|
||||
return compiled
|
||||
"""Compile the full residual model forward as one graph (same strategy as base_lm)."""
|
||||
if self._compiled_layers:
|
||||
return []
|
||||
for layer in self.layers:
|
||||
layer.self_attn._fused_qkv_weight = None
|
||||
self.forward = torch.compile(self.forward, mode="default", fullgraph=False)
|
||||
self._compiled_layers.update(range(len(self.layers)))
|
||||
return ["forward (whole residual)"]
|
||||
|
||||
def load_weights_from_native(self, native_residual_lm: nn.Module) -> int:
|
||||
"""Load weights from native residual_lm. Returns param count."""
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""VoxCPM2 pipeline topology (frozen).
|
||||
|
||||
Single-stage AR TTS: text → speech waveform in one pass.
|
||||
Uses the native MiniCPM4 base_lm with a per-request StaticKVCache that the
|
||||
talker restores into the paged attention layer at step boundaries.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
VOXCPM2_PIPELINE = PipelineConfig(
|
||||
model_type="voxcpm2",
|
||||
model_arch="VoxCPM2TalkerForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="latent_generator",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [1],
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -10,23 +10,26 @@ Architecture:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import dataclasses
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import librosa
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, override_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
@@ -38,6 +41,88 @@ logger = init_logger(__name__)
|
||||
|
||||
_ENABLE_PROFILING = os.environ.get("VOXCPM2_PROFILE", "0") == "1"
|
||||
|
||||
# Lower bound for the _active_states leak-warn threshold. The effective
|
||||
# threshold is max(_ACTIVE_STATE_LEAK_WARN_MIN, 4 * max_batch_size) so small
|
||||
# deployments still get a usable floor instead of a tiny noisy one.
|
||||
_ACTIVE_STATE_LEAK_WARN_MIN = 512
|
||||
|
||||
|
||||
def is_cjk_char(c: str) -> bool:
|
||||
"""Check if a character is a CJK ideograph."""
|
||||
cp = ord(c)
|
||||
return (
|
||||
0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs
|
||||
or 0x3400 <= cp <= 0x4DBF # Extension A
|
||||
or 0xF900 <= cp <= 0xFAFF # Compatibility Ideographs
|
||||
or 0x20000 <= cp <= 0x2A6DF # Extension B
|
||||
or 0x2A700 <= cp <= 0x2B73F # Extension C
|
||||
or 0x2B740 <= cp <= 0x2B81F # Extension D
|
||||
or 0x2F800 <= cp <= 0x2FA1F # Compatibility Supplement
|
||||
)
|
||||
|
||||
|
||||
def build_cjk_split_map(tokenizer: Any) -> dict[int, list[int]]:
|
||||
"""Build {multichar_cjk_token_id: [single_char_ids]} from tokenizer vocab."""
|
||||
vocab = tokenizer.get_vocab()
|
||||
split_map: dict[int, list[int]] = {}
|
||||
for token, token_id in vocab.items():
|
||||
clean = token.replace("\u2581", "")
|
||||
if len(clean) >= 2 and all(is_cjk_char(c) for c in clean):
|
||||
char_ids = tokenizer.convert_tokens_to_ids(list(clean))
|
||||
if all(cid != tokenizer.unk_token_id for cid in char_ids):
|
||||
split_map[token_id] = char_ids
|
||||
return split_map
|
||||
|
||||
|
||||
def split_multichar_chinese(token_ids: list[int], split_map: dict[int, list[int]]) -> list[int]:
|
||||
"""Replace multichar Chinese token IDs with single-char IDs (idempotent)."""
|
||||
result: list[int] = []
|
||||
for tid in token_ids:
|
||||
expansion = split_map.get(tid)
|
||||
if expansion is not None:
|
||||
result.extend(expansion)
|
||||
else:
|
||||
result.append(tid)
|
||||
return result
|
||||
|
||||
|
||||
def build_voxcpm2_prompt(
|
||||
hf_config: Any,
|
||||
tokenizer: Any,
|
||||
split_map: dict[int, list[int]],
|
||||
text: str,
|
||||
ref_audio: Any | None = None,
|
||||
ref_sr: int | None = None,
|
||||
ref_text: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a VoxCPM2 prefill prompt whose ``prompt_token_ids`` length matches
|
||||
the talker-side prefill length.
|
||||
|
||||
Used by both online serving (``serving_speech._build_voxcpm2_prompt``) and
|
||||
the offline example, so the talker-side length assertion never fires.
|
||||
"""
|
||||
ids = split_multichar_chinese(tokenizer.encode(text, add_special_tokens=True), split_map)
|
||||
bos = tokenizer.bos_token_id
|
||||
if ids and ids[0] == bos:
|
||||
ids = ids[1:]
|
||||
prefill_len = len(ids) + 1 # + audio_start
|
||||
additional: dict[str, Any] = {"text_token_ids": [ids]}
|
||||
if ref_audio is not None:
|
||||
vae = hf_config.audio_vae_config
|
||||
patch_samples = hf_config.patch_size * math.prod(vae["encoder_rates"])
|
||||
ref_len = math.ceil(math.ceil(len(ref_audio) * vae["sample_rate"] / ref_sr) / patch_samples)
|
||||
if ref_text is not None:
|
||||
additional["prompt_audio"] = [[ref_audio, ref_sr]]
|
||||
additional["prompt_text"] = [ref_text]
|
||||
ref_ids = split_multichar_chinese(tokenizer.encode(ref_text, add_special_tokens=True), split_map)
|
||||
if ref_ids and ref_ids[0] == bos:
|
||||
ref_ids = ref_ids[1:]
|
||||
prefill_len += ref_len + len(ref_ids)
|
||||
else:
|
||||
additional["reference_audio"] = [[ref_audio, ref_sr]]
|
||||
prefill_len += ref_len + 2 # ref_start / ref_end
|
||||
return {"prompt_token_ids": [1] * prefill_len, "additional_information": additional}
|
||||
|
||||
|
||||
def _encode_raw_audio(
|
||||
tts: nn.Module,
|
||||
@@ -60,7 +145,8 @@ def _encode_raw_audio(
|
||||
encode_sr = tts._encode_sample_rate
|
||||
if sr != encode_sr:
|
||||
audio_np = audio.squeeze(0).numpy()
|
||||
audio_np = librosa.resample(audio_np, orig_sr=sr, target_sr=encode_sr)
|
||||
resampler = AudioResampler(target_sr=encode_sr)
|
||||
audio_np = resampler.resample(audio_np, orig_sr=sr)
|
||||
audio = torch.from_numpy(audio_np).unsqueeze(0)
|
||||
|
||||
patch_len = tts.patch_size * tts.chunk_size
|
||||
@@ -86,7 +172,11 @@ class _RequestState:
|
||||
curr_prefix_feat_cond: torch.Tensor | None = None
|
||||
last_audio_patch_gpu: torch.Tensor | None = None
|
||||
precomputed_stop_logits: torch.Tensor | None = None
|
||||
accumulated_patches: list[torch.Tensor] = dataclasses.field(default_factory=list)
|
||||
# Rolling tail of previously-decoded latents used as VAE receptive-field context.
|
||||
# Shape (n_pad_frames, feat_dim) on GPU. None before first decode.
|
||||
decode_pad: torch.Tensor | None = None
|
||||
# Audio chunks already emitted (CPU float32), concatenated for cumulative output.
|
||||
audio_chunks: list[torch.Tensor] = dataclasses.field(default_factory=list)
|
||||
decode_step_count: int = 0
|
||||
request_start_time: float = 0.0
|
||||
prefill_completed: bool = False
|
||||
@@ -97,6 +187,14 @@ class _RequestState:
|
||||
last_decoded_audio: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _CapturedGraph:
|
||||
graph: torch.cuda.CUDAGraph
|
||||
input_embeds: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
output: torch.Tensor
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Profiling timer
|
||||
# ===================================================================
|
||||
@@ -229,11 +327,11 @@ def _optimized_solve_euler(
|
||||
buffers.x_in[b : 2 * b].copy_(x)
|
||||
buffers.mu_in[:b].copy_(mu)
|
||||
buffers.mu_in[b : 2 * b].zero_()
|
||||
buffers.t_in[:b].fill_(t.item())
|
||||
buffers.t_in[b : 2 * b].fill_(t.item())
|
||||
# Broadcast the 0-dim GPU scalar directly instead of
|
||||
# ``.fill_(t.item())`` — ``.item()`` forces a GPU->CPU sync.
|
||||
buffers.t_in[: 2 * b].copy_(t)
|
||||
if mean_mode:
|
||||
buffers.dt_in[:b].fill_(dt.item())
|
||||
buffers.dt_in[b : 2 * b].fill_(dt.item())
|
||||
buffers.dt_in[: 2 * b].copy_(dt)
|
||||
else:
|
||||
buffers.dt_in.zero_()
|
||||
buffers.cond_in[:b].copy_(cond[:b])
|
||||
@@ -263,9 +361,10 @@ def _optimized_solve_euler(
|
||||
else:
|
||||
buffers.x_in[:b].copy_(x)
|
||||
buffers.mu_in[:b].copy_(mu)
|
||||
buffers.t_in[:b].fill_(t.item())
|
||||
# Broadcast the 0-dim GPU scalar; ``.fill_(t.item())`` would sync.
|
||||
buffers.t_in[:b].copy_(t)
|
||||
if mean_mode:
|
||||
buffers.dt_in[:b].fill_(dt.item())
|
||||
buffers.dt_in[:b].copy_(dt)
|
||||
else:
|
||||
buffers.dt_in[:b].zero_()
|
||||
buffers.cond_in[:b].copy_(cond[:b])
|
||||
@@ -320,7 +419,10 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
self._inference_timesteps = 10
|
||||
self._cfg_value = 2.0
|
||||
self._cfg_cutoff_ratio = 1.0
|
||||
self._vae_decode_interval = 5
|
||||
# Number of trailing latent frames to keep as VAE receptive-field context
|
||||
# for sliding-window streaming decode. 12 matches the nanovllm reference
|
||||
# implementation and covers the longest VAE decoder receptive field.
|
||||
self._n_decode_pad_frames = 12
|
||||
self._enable_torch_compile = True
|
||||
self._compile_vae = True
|
||||
self._max_decode_steps = 2000
|
||||
@@ -328,6 +430,15 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
self._perf = _PerfTimer(enabled=_ENABLE_PROFILING)
|
||||
self._cfm_buffers: _CFMBufferManager | None = None
|
||||
self._enable_cuda_graph = True
|
||||
self._scaffold_graphs: dict[int, _CapturedGraph] = {}
|
||||
self._residual_graphs: dict[int, _CapturedGraph] = {}
|
||||
self._max_cached_graphs = self._max_batch_size
|
||||
self._cuda_graph_pool: tuple | None = None
|
||||
self._cuda_graph_warmup_steps = 0
|
||||
self._cuda_graph_warmup_threshold = 3
|
||||
|
||||
self._multichar_zh_split: dict[int, list[int]] | None = None
|
||||
|
||||
self._active_states: dict[str, _RequestState] = {}
|
||||
self._current_request_id: str | None = None
|
||||
@@ -335,6 +446,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
self._results_queue: list[tuple[str, torch.Tensor | None]] = []
|
||||
self._audio_queue: list[tuple[str, Any]] = []
|
||||
self._deferred_cleanup_ids: set[str] = set()
|
||||
self._active_state_warn_threshold = max(_ACTIVE_STATE_LEAK_WARN_MIN, 4 * self._max_batch_size)
|
||||
# one-shot by design: fires at most once per process to avoid log spam.
|
||||
self._active_state_warned = False
|
||||
|
||||
@property
|
||||
def tts(self) -> nn.Module:
|
||||
@@ -344,9 +458,20 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
# -------------------- request state management --------------------
|
||||
|
||||
def _get_or_create_state(self, request_id: str) -> _RequestState:
|
||||
if request_id not in self._active_states:
|
||||
self._active_states[request_id] = _RequestState(request_id=request_id)
|
||||
return self._active_states[request_id]
|
||||
state = self._active_states.get(request_id)
|
||||
if state is None:
|
||||
state = _RequestState(request_id=request_id)
|
||||
self._active_states[request_id] = state
|
||||
if len(self._active_states) > self._active_state_warn_threshold and not self._active_state_warned:
|
||||
logger.warning(
|
||||
"VoxCPM2: _active_states size=%d exceeds threshold %d "
|
||||
"(max_batch_size=%d); possible cleanup path leak",
|
||||
len(self._active_states),
|
||||
self._active_state_warn_threshold,
|
||||
self._max_batch_size,
|
||||
)
|
||||
self._active_state_warned = True
|
||||
return state
|
||||
|
||||
def _switch_to_request(self, request_id: str) -> _RequestState:
|
||||
if request_id != self._current_request_id:
|
||||
@@ -475,19 +600,24 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
except Exception as e:
|
||||
logger.warning("torch.compile AudioVAE failed: %s", e)
|
||||
|
||||
if not getattr(self.model, "_selective_compiled", False):
|
||||
try:
|
||||
targets.extend(f"scaffold.{t}" for t in self.model.compile_selective())
|
||||
self.model._selective_compiled = True
|
||||
except Exception as e:
|
||||
logger.warning("scaffold compile failed: %s", e)
|
||||
if not self._enable_cuda_graph:
|
||||
if not getattr(self.model, "_selective_compiled", False):
|
||||
try:
|
||||
targets.extend(f"scaffold.{t}" for t in self.model.compile_selective())
|
||||
self.model._selective_compiled = True
|
||||
except Exception as e:
|
||||
logger.warning("scaffold compile failed: %s", e)
|
||||
|
||||
if not getattr(self.residual_model, "_selective_compiled", False):
|
||||
try:
|
||||
targets.extend(f"residual.{t}" for t in self.residual_model.compile_selective())
|
||||
self.residual_model._selective_compiled = True
|
||||
except Exception as e:
|
||||
logger.warning("residual compile failed: %s", e)
|
||||
if not getattr(self.residual_model, "_selective_compiled", False):
|
||||
try:
|
||||
targets.extend(f"residual.{t}" for t in self.residual_model.compile_selective())
|
||||
self.residual_model._selective_compiled = True
|
||||
except Exception as e:
|
||||
logger.warning("residual compile failed: %s", e)
|
||||
else:
|
||||
self.model.precompute_fused_qkv()
|
||||
self.residual_model.precompute_fused_qkv()
|
||||
targets.append("scaffold+residual (CUDA Graph, skipping compile)")
|
||||
|
||||
if not getattr(self, "_projections_compiled", False):
|
||||
try:
|
||||
@@ -510,6 +640,90 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
tts = self.tts
|
||||
return tts.stop_head(tts.stop_actn(tts.stop_proj(lm_h)))
|
||||
|
||||
def _get_cuda_graph_pool(self) -> tuple:
|
||||
if self._cuda_graph_pool is None:
|
||||
self._cuda_graph_pool = torch.cuda.graph_pool_handle()
|
||||
return self._cuda_graph_pool
|
||||
|
||||
@staticmethod
|
||||
def _nullify_volatile_metadata(ctx: Any) -> Any:
|
||||
"""Set ``scheduler_metadata`` to None on all attention layers.
|
||||
|
||||
This is the only tensor FA3 reallocates each step (variable shape).
|
||||
All other metadata tensors are persistent model-runner buffers.
|
||||
Setting it to None makes FA3 use default scheduling (~0.1ms cost).
|
||||
"""
|
||||
if not isinstance(ctx.attn_metadata, dict):
|
||||
return ctx
|
||||
|
||||
ctx = copy.copy(ctx)
|
||||
new_meta: dict[str, Any] = {}
|
||||
for layer_name, meta in ctx.attn_metadata.items():
|
||||
if getattr(meta, "scheduler_metadata", None) is not None:
|
||||
meta = copy.copy(meta)
|
||||
meta.scheduler_metadata = None
|
||||
new_meta[layer_name] = meta
|
||||
ctx.attn_metadata = new_meta
|
||||
return ctx
|
||||
|
||||
def _capture_graph(
|
||||
self,
|
||||
model: nn.Module,
|
||||
batch_size: int,
|
||||
label: str,
|
||||
is_residual: bool = False,
|
||||
) -> _CapturedGraph:
|
||||
"""Capture a CUDA Graph for *model* at *batch_size*."""
|
||||
hidden_size = self.config.hidden_size
|
||||
dtype = self._side_dtype
|
||||
dev = torch.device(self._device)
|
||||
pool = self._get_cuda_graph_pool()
|
||||
|
||||
model.precompute_fused_qkv()
|
||||
|
||||
g = _CapturedGraph(
|
||||
graph=torch.cuda.CUDAGraph(),
|
||||
input_embeds=torch.zeros(batch_size, hidden_size, device=dev, dtype=dtype),
|
||||
positions=torch.zeros(batch_size, device=dev, dtype=torch.long),
|
||||
output=torch.zeros(batch_size, hidden_size, device=dev, dtype=dtype),
|
||||
)
|
||||
|
||||
if is_residual:
|
||||
call_kwargs = dict(positions=g.positions, inputs_embeds=g.input_embeds)
|
||||
else:
|
||||
call_kwargs = dict(input_ids=None, positions=g.positions, inputs_embeds=g.input_embeds)
|
||||
|
||||
ctx = get_forward_context()
|
||||
patched_ctx = self._nullify_volatile_metadata(ctx)
|
||||
|
||||
with override_forward_context(patched_ctx):
|
||||
for _ in range(3):
|
||||
_ = model(**call_kwargs)
|
||||
|
||||
with torch.cuda.graph(g.graph, pool=pool):
|
||||
g.output = model(**call_kwargs)
|
||||
|
||||
logger.info("CUDA Graph captured for %s (batch_size=%d)", label, batch_size)
|
||||
return g
|
||||
|
||||
def _replay_graph(
|
||||
self,
|
||||
g: _CapturedGraph,
|
||||
inputs_embeds: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
batch_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Copy fresh inputs into static buffers, then replay.
|
||||
|
||||
No metadata copy needed: persistent buffers (seq_lens, slot_mapping,
|
||||
etc.) are updated in-place by the model runner. scheduler_metadata
|
||||
was nullified at capture time so no kernel references it.
|
||||
"""
|
||||
g.input_embeds[:batch_size].copy_(inputs_embeds[:batch_size])
|
||||
g.positions[:batch_size].copy_(positions[:batch_size])
|
||||
g.graph.replay()
|
||||
return g.output[:batch_size].clone()
|
||||
|
||||
# -------------------- vllm hooks --------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
@@ -526,12 +740,35 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
self._perf.start("forward_total")
|
||||
dev = input_ids.device
|
||||
|
||||
model_output = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
if isinstance(model_output, IntermediateTensors):
|
||||
return model_output
|
||||
scaffold_hidden = model_output
|
||||
if isinstance(scaffold_hidden, tuple):
|
||||
scaffold_hidden = scaffold_hidden[0]
|
||||
num_reqs = len(self._pending_requests)
|
||||
num_decode = sum(1 for _, is_p, _, n in self._pending_requests if not is_p and n == 1)
|
||||
is_all_decode = num_decode == num_reqs and num_reqs > 0
|
||||
|
||||
tts_compiled = getattr(self.tts.feat_decoder.estimator, "_compiled", False) if self._tts is not None else False
|
||||
graph_ready = tts_compiled and self._cuda_graph_warmup_steps >= self._cuda_graph_warmup_threshold
|
||||
if num_decode > 0:
|
||||
self._cuda_graph_warmup_steps += 1
|
||||
|
||||
can_use_graph = (
|
||||
self._enable_cuda_graph and graph_ready and intermediate_tensors is None and inputs_embeds is not None
|
||||
)
|
||||
|
||||
if can_use_graph and is_all_decode and num_reqs <= self._max_cached_graphs:
|
||||
self._perf.start("scaffold_fwd")
|
||||
if num_reqs not in self._scaffold_graphs:
|
||||
self._scaffold_graphs[num_reqs] = self._capture_graph(self.model, num_reqs, "scaffold")
|
||||
scaffold_hidden = self._replay_graph(self._scaffold_graphs[num_reqs], inputs_embeds, positions, num_reqs)
|
||||
self._perf.stop("scaffold_fwd")
|
||||
|
||||
else:
|
||||
self._perf.start("scaffold_fwd")
|
||||
model_output = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
self._perf.stop("scaffold_fwd")
|
||||
if isinstance(model_output, IntermediateTensors):
|
||||
return model_output
|
||||
scaffold_hidden = model_output
|
||||
if isinstance(scaffold_hidden, tuple):
|
||||
scaffold_hidden = scaffold_hidden[0]
|
||||
|
||||
# Phase 1: per-request FSQ + residual input
|
||||
token_offset = 0
|
||||
@@ -563,7 +800,28 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
if residual_inputs:
|
||||
batch_in = torch.cat(residual_inputs, dim=0)
|
||||
batch_pos = torch.cat(residual_positions, dim=0)
|
||||
batch_out = self.residual_model(batch_pos, batch_in)
|
||||
|
||||
residual_batch_size = batch_in.shape[0]
|
||||
use_residual_graph = (
|
||||
self._enable_cuda_graph
|
||||
and is_all_decode
|
||||
and graph_ready
|
||||
and residual_batch_size == num_reqs # 1 token per request
|
||||
and residual_batch_size <= self._max_cached_graphs
|
||||
)
|
||||
|
||||
self._perf.start("residual_fwd")
|
||||
if use_residual_graph:
|
||||
if residual_batch_size not in self._residual_graphs:
|
||||
self._residual_graphs[residual_batch_size] = self._capture_graph(
|
||||
self.residual_model, residual_batch_size, "residual", is_residual=True
|
||||
)
|
||||
batch_out = self._replay_graph(
|
||||
self._residual_graphs[residual_batch_size], batch_in, batch_pos, residual_batch_size
|
||||
)
|
||||
else:
|
||||
batch_out = self.residual_model(batch_pos, batch_in)
|
||||
self._perf.stop("residual_fwd")
|
||||
|
||||
# Phase 3: per-request LocDiT + update
|
||||
offset = 0
|
||||
@@ -594,19 +852,12 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
tts_len = text_mask.shape[1]
|
||||
scaffold_len = base_lm_out.shape[0]
|
||||
|
||||
if scaffold_len < tts_len:
|
||||
# Voice clone / continuation: scaffold only processed vllm tokens.
|
||||
# Pad to match TTS sequence length (extra positions are masked out).
|
||||
pad = torch.zeros(
|
||||
tts_len - scaffold_len,
|
||||
base_lm_out.shape[-1],
|
||||
device=base_lm_out.device,
|
||||
dtype=base_lm_out.dtype,
|
||||
)
|
||||
enc_out = torch.cat([base_lm_out, pad], dim=0).unsqueeze(0)
|
||||
else:
|
||||
enc_out = base_lm_out.unsqueeze(0)
|
||||
assert scaffold_len == tts_len, (
|
||||
f"voxcpm2 prefill length mismatch: scaffold_len={scaffold_len} tts_len={tts_len}; "
|
||||
"caller must pad prompt_token_ids to the full prefill length "
|
||||
"(see serving_speech._build_voxcpm2_prompt or the offline example)."
|
||||
)
|
||||
enc_out = base_lm_out.unsqueeze(0)
|
||||
|
||||
prefix_feat_cond = (
|
||||
feat[:, -1, ...]
|
||||
@@ -686,7 +937,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
state.request_start_time = time.perf_counter()
|
||||
state.prefill_completed = True
|
||||
|
||||
logger.info("PREFILL[%s]: patch norm=%.4f", state.request_id, pred_feat.norm().item())
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
# Only compute the norm (which forces a GPU->CPU sync) if we will log it.
|
||||
logger.debug("PREFILL[%s]: patch norm=%.4f", state.request_id, pred_feat.norm().item())
|
||||
self._perf.reset()
|
||||
|
||||
def _finish_decode(self, state: _RequestState, meta: dict, res_out: torch.Tensor, dev: Any):
|
||||
@@ -720,26 +973,54 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
# -------------------- audio collection --------------------
|
||||
|
||||
def _collect_audio(self, state: _RequestState) -> torch.Tensor | None:
|
||||
"""Per-step sliding-window VAE decode (nanovllm pattern).
|
||||
|
||||
Each decode step feeds ``[decode_pad, new_patch]`` through the VAE
|
||||
and slices out only the audio region corresponding to the new patch.
|
||||
The pad buffer (last ``_n_decode_pad_frames`` latent frames) provides
|
||||
the receptive-field context needed by the VAE's transposed convolutions,
|
||||
eliminating boundary artifacts between chunks.
|
||||
|
||||
Returns the delta audio chunk (not cumulative) so the output processor
|
||||
can stream each chunk to the client independently.
|
||||
"""
|
||||
patch = state.last_audio_patch_gpu
|
||||
if patch is not None:
|
||||
state.last_audio_patch_gpu = None
|
||||
state.accumulated_patches.append(patch.reshape(1, -1).float())
|
||||
|
||||
if not state.accumulated_patches:
|
||||
if patch is None:
|
||||
return None
|
||||
state.last_audio_patch_gpu = None
|
||||
|
||||
n = len(state.accumulated_patches)
|
||||
if n <= 1 or n % self._vae_decode_interval == 0 or state.is_stopping:
|
||||
self._perf.start("vae_decode")
|
||||
all_p = torch.cat(state.accumulated_patches, dim=0)
|
||||
state.accumulated_patches = [all_p]
|
||||
feat = rearrange(all_p.reshape(1, -1, self._feat_dim), "b t d -> b d t")
|
||||
with torch.no_grad():
|
||||
audio = self.tts.audio_vae.decode(feat.to(self._device)).reshape(-1).cpu().float()
|
||||
self._perf.stop("vae_decode")
|
||||
state.last_decoded_audio = audio
|
||||
return audio
|
||||
return state.last_decoded_audio
|
||||
# patch shape: (patch_size, feat_dim) or (1, patch_size, feat_dim)
|
||||
new_latent = patch.reshape(-1, self._feat_dim).to(torch.float32)
|
||||
n_new = new_latent.shape[0] # = patch_size (typically 4)
|
||||
|
||||
self._perf.start("vae_decode")
|
||||
|
||||
# Build VAE input: [pad_frames | new_latent]
|
||||
if state.decode_pad is not None:
|
||||
vae_input = torch.cat([state.decode_pad, new_latent], dim=0)
|
||||
pad_frames = state.decode_pad.shape[0]
|
||||
else:
|
||||
vae_input = new_latent
|
||||
pad_frames = 0
|
||||
|
||||
# VAE decode: (1, feat_dim, T_frames) -> (1, 1, T_samples)
|
||||
feat = vae_input.unsqueeze(0).transpose(1, 2).contiguous()
|
||||
with torch.no_grad():
|
||||
audio = self.tts.audio_vae.decode(feat.to(self._device)).reshape(-1)
|
||||
|
||||
# Slice out only the new audio (after the pad region).
|
||||
# Each latent frame maps to decoder_chunk_size audio samples.
|
||||
dcs = int(getattr(self.tts.audio_vae, "decode_chunk_size", audio.numel() // vae_input.shape[0]))
|
||||
new_audio = audio[pad_frames * dcs : (pad_frames + n_new) * dcs].detach().cpu().float()
|
||||
|
||||
# Roll the pad buffer: keep last N latent frames as context for next step.
|
||||
all_latents = vae_input # [pad + new]
|
||||
state.decode_pad = all_latents[-self._n_decode_pad_frames :].detach()
|
||||
|
||||
state.audio_chunks.append(new_audio)
|
||||
state.last_decoded_audio = new_audio
|
||||
self._perf.stop("vae_decode")
|
||||
return new_audio
|
||||
|
||||
# -------------------- compute_logits --------------------
|
||||
|
||||
@@ -797,6 +1078,17 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=mm)
|
||||
|
||||
# -------------------- Chinese token splitting --------------------
|
||||
|
||||
def _get_multichar_zh_split(self) -> dict[int, list[int]]:
|
||||
"""Lazy-build {multichar_chinese_token_id: [char_id, ...]} map."""
|
||||
if self._multichar_zh_split is not None:
|
||||
return self._multichar_zh_split
|
||||
base_tokenizer = self.tts.text_tokenizer.tokenizer
|
||||
self._multichar_zh_split = build_cjk_split_map(base_tokenizer)
|
||||
logger.info("VoxCPM2: built multichar Chinese split map (%d entries)", len(self._multichar_zh_split))
|
||||
return self._multichar_zh_split
|
||||
|
||||
# -------------------- preprocess / postprocess --------------------
|
||||
|
||||
def preprocess(
|
||||
@@ -815,22 +1107,29 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
|
||||
is_prefill = span_len > 1
|
||||
|
||||
if is_prefill:
|
||||
# Evict stale states
|
||||
pending_ids = {rid for rid, *_ in self._pending_requests}
|
||||
pending_ids.add(req_id)
|
||||
if self._current_request_id:
|
||||
pending_ids.add(self._current_request_id)
|
||||
for rid in [r for r, s in self._active_states.items() if r not in pending_ids and s.prefill_completed]:
|
||||
self._cleanup_request(rid)
|
||||
|
||||
# VoxCPM2Tokenizer does char-level Chinese splitting, so use input_ids directly
|
||||
token_ids = input_ids.tolist()
|
||||
# Do not evict state here: _pending_requests is a per-step prefix,
|
||||
# not the full batch. Cleanup is driven by on_requests_finished ->
|
||||
# _flush_deferred_cleanup (fed by vLLM scheduler._free_request via
|
||||
# gpu_ar_model_runner.py).
|
||||
real = info_dict.get("text_token_ids")
|
||||
token_ids = input_ids.tolist() if real is None else real[0]
|
||||
# Fail-fast: unsplit multichar Chinese IDs in input_ids means the
|
||||
# serving layer didn't pre-split. Silent fixup here would cause
|
||||
# input_ids/embeds length mismatch (scheduler slot count is fixed).
|
||||
split_map = self._get_multichar_zh_split()
|
||||
if split_map and any(tid in split_map for tid in token_ids):
|
||||
raise ValueError(
|
||||
"VoxCPM2 preprocess received unsplit multichar Chinese "
|
||||
"token IDs. The serving layer must send prompt_token_ids "
|
||||
"with single-char CJK IDs (see _voxcpm2_encode)."
|
||||
)
|
||||
if token_ids and token_ids[0] == self.config.bos_token_id:
|
||||
token_ids = token_ids[1:]
|
||||
|
||||
state = self._get_or_create_state(req_id)
|
||||
state.prefill_text = ""
|
||||
state.accumulated_patches = []
|
||||
state.decode_pad = None
|
||||
state.audio_chunks = []
|
||||
state.prefill_completed = False
|
||||
state.decode_step_count = 0
|
||||
state.precomputed_stop_logits = None
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Voxtral TTS pipeline topology (frozen).
|
||||
|
||||
Stage 0: audio_generation — text → acoustic latents (LLM_AR, tokenizer owner).
|
||||
Stage 1: audio_tokenizer — acoustic latents → waveform (LLM_GENERATION).
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts"
|
||||
|
||||
VOXTRAL_TTS_PIPELINE = PipelineConfig(
|
||||
model_type="voxtral_tts",
|
||||
model_arch="VoxtralTTSForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audio_generation",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=False,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.generator2tokenizer_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="audio_tokenizer",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
extras={"tts_args": {"max_instructions_length": 500}},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) 2022 OpenAI
|
||||
#
|
||||
# Shared Whisper encoder primitives used by multiple model implementations.
|
||||
# Originally from the OpenAI Whisper codebase.
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def sinusoids(length, channels, max_timescale=10000):
|
||||
"""Returns sinusoids for positional embedding."""
|
||||
assert channels % 2 == 0
|
||||
log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
|
||||
inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
|
||||
scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
|
||||
return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
|
||||
|
||||
|
||||
class Conv1d(nn.Conv1d):
|
||||
"""Conv1d with automatic dtype casting for mixed precision inference."""
|
||||
|
||||
def _conv_forward(self, x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
||||
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
|
||||
|
||||
|
||||
class ConvTranspose1d(nn.ConvTranspose1d):
|
||||
def _conv_forward(self, x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
||||
return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))
|
||||
|
||||
|
||||
class Linear(nn.Linear):
|
||||
"""Linear layer with automatic dtype casting for mixed precision inference."""
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return F.linear(x, self.weight.to(x.dtype), None if self.bias is None else self.bias.to(x.dtype))
|
||||
Reference in new issue
Block a user