This commit is contained in:
Taichang Zhou committed 2026-01-31 10:59:36 +08:00
commit e033498600
310 files changed
+21792 -8363

No files matched your search

@@ -36,7 +36,7 @@ from vllm_omni.model_executor.model_loader.weight_utils import download_weights_
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
from vllm_omni.model_executor.models.vision import get_llm_pos_ids_for_vision
from vllm_omni.utils.platform_utils import is_npu
from vllm_omni.platforms import current_omni_platform
TALKER_CODEC_EOS_TOKEN_ID = 8294
TALKER_CODEC_BOS_TOKEN_ID = 8293
@@ -247,7 +247,7 @@ class Qwen2_5OmniForConditionalGeneration(
if inputs_embeds is not None and inputs_embeds.device != thinker_dev:
inputs_embeds = inputs_embeds.to(thinker_dev)
if is_npu():
if current_omni_platform.is_npu():
# TODO: remove this hack when NPU supports batched inputs properly
thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
# For MRoPE, positions shape is [3, num_tokens] (T/H/W), don't slice it
@@ -35,7 +35,7 @@ from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import cap_and_align_mel_length, resolve_max_mel_frames
from vllm_omni.utils.platform_utils import is_npu
from vllm_omni.platforms import current_omni_platform
# Provide a no-op auto_docstring decorator to satisfy annotations if missing
@@ -732,10 +732,14 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) ->
beta = 0.0
# TODO: When torch.kaiser_window supports NPU, remove the device="cpu" argument
if is_npu():
if current_omni_platform.is_npu():
kaiser_window = torch.kaiser_window(
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
).to("npu")
elif current_omni_platform.is_xpu():
kaiser_window = torch.kaiser_window(
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
).to("xpu")
else:
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
@@ -796,7 +800,7 @@ class UpSample1d(nn.Module):
def forward(self, hidden_states):
channels = hidden_states.shape[1]
if is_npu():
if current_omni_platform.is_npu():
# TODO: When F.pad supports replicate mode on NPU, remove this branch
input_dtype = hidden_states.dtype
# F.pad in NPU doesn't support BF16 when mode is replicate.
@@ -843,7 +847,7 @@ class DownSample1d(nn.Module):
def forward(self, hidden_states):
channels = hidden_states.shape[1]
if is_npu():
if current_omni_platform.is_npu():
input_dtype = hidden_states.dtype
# F.pad in NPU doesn't support BF16 when mode is replicate.
# To ensure the accuracy, manually pad the input tensor.
@@ -1258,7 +1262,6 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
return output
@torch.no_grad()
def sample(
self,
conditioning_vector,
@@ -1333,7 +1336,6 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
generated_mel_spectrogram = generated_waveform.permute(0, 2, 1)
return generated_mel_spectrogram
@torch.no_grad()
def fast_block_sample(
self,
conditioning_vector: torch.Tensor,
@@ -35,7 +35,7 @@ from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, safe_tensor_reshape
from vllm_omni.utils.platform_utils import is_npu
from vllm_omni.platforms import current_omni_platform
# Special token IDs for Qwen3 Omni MoE
# Reference: https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
@@ -269,7 +269,7 @@ class Qwen3OmniMoeForConditionalGeneration(
# ========== Stage 1: Thinker ==========
if self.model_stage == "thinker":
thinker_dev = self._module_device(self.thinker)
if is_npu():
if current_omni_platform.is_npu():
# Normalize to batched inputs if needed
_added_batch_dim = False
if input_ids is not None and input_ids.ndim == 1:
@@ -308,7 +308,7 @@ class Qwen3OmniMoeForConditionalGeneration(
"capture_layer_indices": [0, int(accept_layer)],
"return_hidden_states": True,
}
if is_npu():
if current_omni_platform.is_npu():
# TODO: remove this hack when NPU supports batched inputs properly
thinker_input_ids = input_ids[0] if input_ids is not None and _added_batch_dim else input_ids
thinker_inputs_embeds = (
@@ -483,12 +483,19 @@ class Qwen3OmniMoeForConditionalGeneration(
talker_codes = talker_codes.unsqueeze(0).unsqueeze(0)
talker_codes = talker_codes.expand(1, 16, -1)
# Use chunked decode for memory efficiency
audio_tensor = self.code2wav.chunked_decode(
talker_codes,
chunk_size=300,
left_context_size=25,
)
if self.vllm_config.model_config.async_chunk:
audio_tensor = self.code2wav.chunked_decode_streaming(
talker_codes,
chunk_size=25,
left_context_size=25,
)
else:
# Use chunked decode for memory efficiency
audio_tensor = self.code2wav.chunked_decode(
talker_codes,
chunk_size=300,
left_context_size=25,
)
return audio_tensor
@@ -595,7 +602,6 @@ class Qwen3OmniMoeForConditionalGeneration(
last_talker_hidden, text_step, update_dict = self.talker_preprocess_decode(
input_ids, input_embeds, **info_dict
)
update_dict["mtp_inputs"] = last_talker_hidden, text_step
return input_ids, input_embeds, update_dict
@@ -624,6 +630,33 @@ class Qwen3OmniMoeForConditionalGeneration(
inputs_embeds = (inputs_embeds + text_step).reshape(-1, self.talker_config.text_config.hidden_size)
return inputs_embeds, code_predictor_codes.squeeze(-1)
def _get_tts_embed(self, thinker_embed, tts_bos_thinker, tts_eos_thinker, tts_pad_thinker):
"""Project thinker-side TTS embeddings into talker text space."""
module_device = self._module_device(self.talker)
def _ensure_1x1(x: torch.Tensor) -> torch.Tensor:
if x.ndim == 3:
return x[0, -1:, :]
if x.ndim == 2:
return x[-1]
return x.view(1, 1, -1)
def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor:
if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0:
xin = _ensure_1x1(x_opt).to(module_device)
else:
xin = torch.zeros(
(1, thinker_embed.shape[-1]),
device=module_device,
dtype=thinker_embed.dtype,
)
return self.talker.text_projection(xin).to(module_device)
self.tts_bos_embed = _proj_from_thinker(tts_bos_thinker)
self.tts_eos_embed = _proj_from_thinker(tts_eos_thinker)
self.tts_pad_embed = _proj_from_thinker(tts_pad_thinker)
return self.tts_bos_embed, self.tts_eos_embed, self.tts_pad_embed
def talker_preprocess_prefill(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict):
# Containers to return per-request updates (e.g., code_predictor_hidden_per_request)
update_dict: dict[str, dict] = {}
@@ -647,7 +680,7 @@ class Qwen3OmniMoeForConditionalGeneration(
if info_dict.get("thinker_input_ids") is None
else torch.as_tensor(info_dict.get("thinker_input_ids"), device=self._module_device(self.talker))
)
# Optional thinker-side TTS token embeddings (to be projected by talker)
tts_bos_thinker = info_dict.get("tts_bos_embed").to(
device=self._module_device(self.talker), dtype=torch.bfloat16
)
@@ -700,7 +733,7 @@ class Qwen3OmniMoeForConditionalGeneration(
tts_pad_thinker=tts_pad_thinker,
)
# Queue tailing_text_hidden for decode (drop first for next steps),
# Queue trailing_text_hidden for decode (drop first for next steps),
try:
if isinstance(trailing_text_hidden, torch.Tensor) and trailing_text_hidden.numel() > 0:
if trailing_text_hidden.ndim == 2:
@@ -716,7 +749,7 @@ class Qwen3OmniMoeForConditionalGeneration(
# compatible with old shape [1,S,D]
rem_tail = trailing_text_hidden.squeeze(0)
if rem_tail.shape[0] > 0:
update_dict["tailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous()
update_dict["trailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous()
# Also persist projected tts_pad for decode fallback if needed
if isinstance(tts_pad_thinker, torch.Tensor):
pad_in = tts_pad_thinker
@@ -762,31 +795,9 @@ class Qwen3OmniMoeForConditionalGeneration(
(thinker_result_ids == self.thinker_config.video_token_id)
).to(input_ids.device) # [t] # fmt: skip
# Project thinker-side TTS embeddings into talker text space (provided by thinker stage)
talker_dev = self._module_device(self.talker)
def _ensure_1x1(x: torch.Tensor) -> torch.Tensor:
if x.ndim == 3:
return x[0, -1:, :]
if x.ndim == 2:
return x[-1]
return x.view(1, 1, -1)
def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor:
if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0:
xin = _ensure_1x1(x_opt).to(talker_dev)
else:
xin = torch.zeros(
(1, thinker_embed.shape[-1]),
device=talker_dev,
dtype=thinker_embed.dtype,
)
return self.talker.text_projection(xin).to(input_ids.device)
tts_bos_embed = _proj_from_thinker(tts_bos_thinker)
tts_eos_embed = _proj_from_thinker(tts_eos_thinker)
tts_pad_embed = _proj_from_thinker(tts_pad_thinker)
self.tts_pad_embed = tts_pad_embed
tts_bos_embed, tts_eos_embed, tts_pad_embed = self._get_tts_embed(
thinker_embed, tts_bos_thinker, tts_eos_thinker, tts_pad_thinker
)
talker_input_embeds = [] # [1 t d]
talker_input_ids = []
@@ -835,24 +846,58 @@ class Qwen3OmniMoeForConditionalGeneration(
return talker_input_id, talker_input_embed, trailing_text_hidden_all
def _thinker_decode_to_talker_decode(
self,
info_dict: dict,
device: torch.device,
update_dict,
):
"""
Project thinker outputs to talker inputs during prefill stage.
Returns:
(input_ids, input_embeds) for talker
"""
thinker_embed = info_dict.get("thinker_embeddings", None)
if thinker_embed is None:
if info_dict.get("finished_flag"):
return self.tts_pad_embed.to(device)
update_dict["finished_flag"] = True
return self.tts_eos_embed.to(device)
thinker_embed = thinker_embed.to(device)
return self.talker.text_projection(thinker_embed).to(device)
def talker_preprocess_decode(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict):
update_dict: dict[str, dict] = {}
last_talker_hidden = None
text_step = None
try:
q_tail = info_dict.get("tailing_text_hidden")
if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0:
use_vec = q_tail[0:1, :]
new_q_tail = (
q_tail[1:, :].detach().to("cpu").contiguous()
if q_tail.shape[0] > 1
else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
)
text_step = use_vec.to(input_embeds.device, dtype=input_embeds.dtype)
update_dict["tailing_text_hidden"] = new_q_tail
if self.vllm_config.model_config.async_chunk:
text_step = self._thinker_decode_to_talker_decode(info_dict, input_ids.device, update_dict)
else:
text_step = self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
q_tail = info_dict.get("trailing_text_hidden", None)
if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0:
use_vec = q_tail[0:1, :]
new_q_tail = (
q_tail[1:, :].detach().to("cpu").contiguous()
if q_tail.shape[0] > 1
else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
)
text_step = use_vec.to(input_embeds.device, dtype=input_embeds.dtype)
update_dict["trailing_text_hidden"] = new_q_tail
else:
text_step = self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
last_talker_hidden = info_dict.get("last_talker_hidden").to(input_embeds.device, dtype=input_embeds.dtype)
last_talker_hidden = last_talker_hidden.reshape(*last_talker_hidden.shape[-2:]) # [1, hidden_size]
last_talker_hidden_tensor = info_dict.get("last_talker_hidden")
if last_talker_hidden_tensor is not None:
last_talker_hidden = last_talker_hidden_tensor.to(input_embeds.device, dtype=input_embeds.dtype)
last_talker_hidden = last_talker_hidden.reshape(*last_talker_hidden.shape[-2:]) # [1, hidden_size]
else:
last_talker_hidden = torch.zeros(
(1, self.talker_config.text_config.hidden_size),
device=input_embeds.device,
dtype=input_embeds.dtype,
)
except Exception as e:
logger.error(f"Error in decode: {e}")
@@ -866,7 +911,6 @@ class Qwen3OmniMoeForConditionalGeneration(
)
user_mm_mask = multimodal_mask[im_start_index:segment_end_index]
# Multimodal data exists
if user_mm_mask.any():
user_thinker_hidden_mm = thinker_hidden[im_start_index:segment_end_index][user_mm_mask]
@@ -883,12 +927,20 @@ class Qwen3OmniMoeForConditionalGeneration(
assistant_hidden = self.talker.text_projection(thinker_embed[im_start_index:segment_end_index]).to(
tts_pad_embed.device
) # [t, d]
# [3 tokens] + [4 pad] + [1 BOS] + [1 first text] = 9 tokens
assistant_text_hidden = torch.cat(
(
assistant_hidden[:3],
tts_pad_embed.expand(4, -1),
tts_bos_embed,
assistant_hidden[3:4], # First text
assistant_hidden[3:4]
if assistant_hidden.shape[0] > 3
else torch.zeros(
(1, assistant_hidden.shape[1]),
device=assistant_hidden.device,
dtype=assistant_hidden.dtype,
), # First text
),
dim=0,
)
@@ -904,6 +956,9 @@ class Qwen3OmniMoeForConditionalGeneration(
device=tts_pad_embed.device,
dtype=torch.long,
)
embed_input_ids = self.talker.embed_input_ids(codec_special_tokens).to(
device=tts_pad_embed.device, dtype=torch.bfloat16
)
assistant_codec_hidden = torch.cat(
(
torch.zeros(
@@ -911,18 +966,21 @@ class Qwen3OmniMoeForConditionalGeneration(
device=tts_pad_embed.device,
dtype=torch.bfloat16,
),
self.talker.embed_input_ids(codec_special_tokens).to(device=tts_pad_embed.device, dtype=torch.bfloat16),
),
dim=0,
)
trailing_text_hidden = torch.cat(
(
assistant_hidden[4:],
tts_eos_embed,
embed_input_ids,
),
dim=0,
)
if assistant_hidden.shape[0] > 4:
trailing_text_hidden = torch.cat(
(assistant_hidden[4:], tts_eos_embed),
dim=0,
)
else:
trailing_text_hidden = torch.zeros(
tts_eos_embed.shape, device=tts_eos_embed.device, dtype=tts_eos_embed.dtype
)
input_embeds = assistant_text_hidden + assistant_codec_hidden
input_ids = torch.full(
(assistant_text_hidden.shape[0],),
@@ -196,6 +196,39 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
return torch.cat(wavs, dim=-1)
def chunked_decode_streaming(
self,
codes: torch.Tensor,
chunk_size: int = 25,
left_context_size: int = 25,
) -> torch.Tensor:
"""
Decode long sequences in chunks to avoid OOM.
Uses overlapping chunks with left context to avoid boundary artifacts.
Args:
codes: [batch, num_quantizers, seq_len] - num_quantizers-layer RVQ codes
chunk_size: Number of codec frames per chunk
left_context_size: Number of overlapping frames for context
Returns:
waveform: [batch, 1, waveform_len] - Complete waveform
"""
wavs = []
end_index = codes.shape[-1]
# TODO: need to optimize algorithms, current only support
# chunk_size = left_context_size = 25
if end_index <= chunk_size:
context_size = 0
else:
context_size = left_context_size
# Decode chunk
wav_chunk = self(codes)
# Remove context from output (context_size * total_upsample samples)
wavs.append(wav_chunk[..., context_size * self.total_upsample :])
return torch.cat(wavs, dim=-1)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights from HuggingFace checkpoint."""
loader = AutoWeightsLoader(
@@ -13,9 +13,15 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import Cache, PretrainedConfig
from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import Qwen3OmniMoeRotaryEmbedding
from transformers.generation.logits_process import (
LogitsProcessorList,
TopKLogitsWarper,
TopPLogitsWarper,
)
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -24,81 +30,12 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
# ============================================================================
# Rotary Embeddings and Helper Functions
# ============================================================================
class Qwen3OmniCodePredictorRotaryEmbedding(nn.Module):
"""Rotary positional embeddings for the code predictor."""
def __init__(self, dim, max_position_embeddings=2048, base=10000):
super().__init__()
self.dim = dim
self.max_position_embeddings = max_position_embeddings
self.base = base
inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim))
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, x: torch.Tensor, position_ids: torch.LongTensor) -> tuple[torch.Tensor, torch.Tensor]:
inv_freq = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
position = position_ids[:, None, :].float()
freqs = (inv_freq @ position).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos().to(x.dtype)
sin = emb.sin().to(x.dtype)
return cos, sin
def rotate_half(x):
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
"""Applies Rotary Position Embedding to the query and key tensors.
Args:
q (`torch.Tensor`): The query tensor.
k (`torch.Tensor`): The key tensor.
cos (`torch.Tensor`): The cosine part of the rotary embedding.
sin (`torch.Tensor`): The sine part of the rotary embedding.
position_ids (`torch.Tensor`, *optional*):
Deprecated and unused.
unsqueeze_dim (`int`, *optional*, defaults to 1):
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
"""
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
q_embed = (q * cos) + (rotate_half(q) * sin)
k_embed = (k * cos) + (rotate_half(k) * sin)
return q_embed, k_embed
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
"""Repeat key/value heads for grouped query attention."""
if n_rep == 1:
return hidden_states
bsz, num_kv_heads, seq_len, head_dim = hidden_states.shape
hidden_states = hidden_states[:, :, None, :, :].expand(bsz, num_kv_heads, n_rep, seq_len, head_dim)
return hidden_states.reshape(bsz, num_kv_heads * n_rep, seq_len, head_dim)
# ============================================================================
# Code Predictor Attention Layer
# ============================================================================
@@ -149,6 +86,12 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
prefix=f"{prefix}.o_proj",
disable_tp=True,
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=config.code_predictor_config.max_position_embeddings,
rope_parameters=None,
dual_chunk_attention_config=None,
)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_key_value_heads * self.head_dim
@@ -159,12 +102,19 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
self.is_causal = True
self.config = config
self.attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"]
cudagraph_mode = get_current_vllm_config().compilation_config.cudagraph_mode
if "flash_attention_2" in ALL_ATTENTION_FUNCTIONS and cudagraph_mode.has_full_cudagraphs():
logger.warning(
f"CUDAGraphMode.{cudagraph_mode.name} is currently not supported "
f"with flash attention for Qwen3-Omni talker MTP."
f"removing flash attention from attention_backends"
)
self.attention_backends.remove("flash_attention_2")
def forward(
self,
hidden_states: torch.Tensor,
causal_mask: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_values: Cache | None = None,
cache_position: torch.LongTensor | None = None,
use_cache: bool = False,
@@ -176,34 +126,39 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
# Reshape for attention
q = q.view(bsz, seq_len, self.num_heads, self.head_dim)
k = k.view(bsz, seq_len, self.num_key_value_heads, self.head_dim)
v = v.view(bsz, seq_len, self.num_key_value_heads, self.head_dim)
q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim)
k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
v = v.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
# Apply normalization
q = self.q_norm(q)
k = self.k_norm(k)
q = self.q_norm(q).contiguous()
k = self.k_norm(k).contiguous()
q = q.reshape(-1, self.q_size)
k = k.reshape(-1, self.kv_size)
# Apply RoPE
q, k = self.rotary_emb(position_ids, q, k)
# Reshape for attention
q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim)
k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
v_heads = v.transpose(1, 2).contiguous()
q_heads = q.transpose(1, 2).contiguous()
k_heads = k.transpose(1, 2).contiguous()
q_heads, k_heads = apply_rotary_pos_emb(q_heads, k_heads, cos, sin)
v_heads = v.transpose(1, 2).contiguous()
if past_key_values is not None:
sin, cos = self.rotary_emb.get_cos_sin(seq_len)
# sin and cos are specific to RoPE models; cache_position needed for the static cache
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
k_heads, v_heads = past_key_values.update(k_heads, v_heads, self.layer_idx, cache_kwargs)
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
# Try attention backends in order of preference, with runtime error handling
# This handles cases where the backend is registered but not actually available
attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"]
attn_output = None
last_error = None
for backend_name in attention_backends:
for backend_name in self.attention_backends:
if backend_name not in ALL_ATTENTION_FUNCTIONS:
continue
@@ -219,15 +174,10 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
scaling=self.head_dim**-0.5,
sliding_window=None,
use_cache=use_cache,
position_ids=position_ids,
position_ids=position_ids[:seq_len].unsqueeze(0),
output_hidden_states=True,
output_attentions=False,
)
# Success - log fallback if not using flash_attention_2
if backend_name != "flash_attention_2":
logger.warning_once(
f"Using {backend_name} attention backend (flash_attention_2 not available or failed)"
)
break
except (ValueError, ImportError, RuntimeError, AttributeError) as e:
# Store error and try next backend
@@ -335,9 +285,6 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
def mtp_block(
self,
hidden_states: torch.Tensor,
causal_mask: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_values: Cache | None = None,
cache_position: torch.LongTensor | None = None,
use_cache: bool = False,
@@ -346,9 +293,7 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
# Self-attention with residual
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(
hidden_states, causal_mask, cos, sin, past_key_values, cache_position, use_cache, position_ids
)
hidden_states = self.self_attn(hidden_states, past_key_values, cache_position, use_cache, position_ids)
hidden_states = residual + hidden_states
# MLP with residual
@@ -359,47 +304,6 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
return hidden_states
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None, "inputs_embeds required for MTP"
# Mask position 0 (not needed for MTP)
inputs_embeds[positions == 0] = 0
hidden_states = torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
# Get position info for RoPE
batch_size, seq_len, _ = hidden_states.shape
device = hidden_states.device
position_ids = torch.arange(seq_len, dtype=torch.long, device=device).unsqueeze(0).expand(batch_size, -1)
# Get RoPE embeddings
head_dim = self.self_attn.head_dim
rotary_emb = Qwen3OmniCodePredictorRotaryEmbedding(
head_dim, max_position_embeddings=self.config.code_predictor_config.max_position_embeddings
)
rotary_emb = Qwen3OmniMoeRotaryEmbedding(self.config)
cos, sin = rotary_emb(hidden_states, position_ids)
# Create causal mask
causal_mask = (
torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, device=device), diagonal=1)
.unsqueeze(0)
.unsqueeze(0)
)
causal_mask = causal_mask.masked_fill(causal_mask, float("-inf"))
# Forward through MTP block
hidden_states = self.mtp_block(hidden_states, causal_mask, cos, sin)
return hidden_states
class Qwen3OmniCodePredictorBaseModel(nn.Module):
"""
@@ -444,9 +348,6 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# RoPE
self.rotary_emb = Qwen3OmniMoeRotaryEmbedding(config=config)
def forward(
self,
inputs_embeds: torch.Tensor,
@@ -462,7 +363,6 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
Args:
inputs_embeds: [batch, seq_len, hidden_size]
attention_mask: Optional attention mask tensor
position_ids: Optional position IDs tensor
past_key_values: Optional cached key-value pairs
use_cache: Whether to use cache
@@ -473,37 +373,11 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
Named tuple with .last_hidden_state and .past_key_values attributes
"""
batch_size, seq_len, _ = inputs_embeds.shape
# Create positions tensor if not provided
# positions must be [num_tokens] or [batch_size, seq_len]
if position_ids is None:
if cache_position is not None:
position_ids = cache_position # [num_tokens]
else:
position_ids = torch.arange(seq_len, device=inputs_embeds.device).unsqueeze(0) # [1, seq_len]
else:
position_ids = position_ids.flatten() # Ensure [num_tokens]
# Extract cos/sin from rotary_emb cache
# The cos_sin_cache is [max_pos, rotary_dim * 2]
cos, sin = self.rotary_emb(inputs_embeds, position_ids)
# Create causal mask
device = inputs_embeds.device
causal_mask = (
torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, device=device), diagonal=1)
.unsqueeze(0)
.unsqueeze(0)
)
causal_mask = causal_mask.masked_fill(causal_mask, float("-inf"))
# Forward through decoder layers
hidden_states = inputs_embeds
for layer in self.layers:
hidden_states = layer.mtp_block(
hidden_states, causal_mask, cos, sin, past_key_values, cache_position, use_cache, position_ids
)
hidden_states = layer.mtp_block(hidden_states, past_key_values, cache_position, use_cache, position_ids)
# Final norm
hidden_states = self.norm(hidden_states)
@@ -517,6 +391,32 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
return self.codec_embedding
def code_predictor_sample(
logits: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
forward_context = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
logits = self.logits_processors(None, logits[:, -1])
probs = F.softmax(logits, dim=-1)
code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1]
return code
def code_predictor_sample_fake(
logits: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
return torch.empty((logits.shape[0], 1), dtype=torch.int64, device=logits.device)
direct_register_custom_op(
op_name="qwen3_omni_code_predictor_sample",
op_func=code_predictor_sample,
fake_impl=code_predictor_sample_fake,
)
@support_torch_compile
class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
"""
@@ -552,30 +452,83 @@ class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
for _ in range(self.num_code_groups - 1)
]
)
self.logits_processors = LogitsProcessorList(
[
TopKLogitsWarper(top_k=50),
TopPLogitsWarper(top_p=0.8),
]
)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
self.layer_name = prefix
def forward(
self,
inputs_embeds: torch.Tensor, # [batch, seq_len, hidden_size]
layer_idx: int, # Which layer to predict (0-num_layers-2 for layers 1-num_layers-1)
layer0_code: torch.Tensor,
layer0_embed: torch.Tensor,
last_talker_hidden: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Forward pass for code predictor.
Args:
inputs_embeds: Input embeddings [batch_size, seq_len, hidden_size]
layer_idx: Which residual layer to predict (0-num_layers-2 for layers 1-num_layers-1)
layer0_code:
Code index for code-group (layer) 0.
Shape: [batch_size, 1], dtype typically int64.
last_talker_hidden:
Shape: [batch_size, hidden_size].
Returns:
logits: Predicted logits [batch_size, seq_len, vocab_size]
hidden_states: Output hidden states [batch_size, seq_len, hidden_size]
pos_all_layers:
Predicted codes for all code groups, including `layer0_code`.
Shape: [batch_size, num_code_groups, 1].
current_input:
The final input embedding sequence after appending embeddings of all
predicted codes (one token per predicted layer).
Shape: [batch_size, num_code_groups + 2, hidden_size].
"""
# Pass through base model
hidden_states = self.model(inputs_embeds)
pos_codes = [layer0_code] # Start with layer 0: [batch, 1]
try:
current_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [batch, 2, hidden_size]
except Exception as e:
print(f"Error in current_input: {e}")
print(f"last_talker_hidden shape: {last_talker_hidden.shape}")
print(f"prev_embed shape: {layer0_embed.shape}")
raise e
batch_size = current_input.shape[0]
# Get logits from corresponding head
logits = self.lm_head[layer_idx](hidden_states)
# Predict all residual layers (layers 1 to num_code_groups-1) autoregressively
for layer_idx in range(self.num_code_groups - 1):
seq_len = layer_idx + 2
# Compute position_ids dynamically to avoid torch.compile specializing batch_size
position_ids = torch.arange(seq_len, device=current_input.device, dtype=torch.int64).repeat(batch_size)
# Forward through code_predictor model
outputs = self.model(
inputs_embeds=current_input,
attention_mask=None,
position_ids=position_ids,
past_key_values=None,
use_cache=False,
cache_position=None,
)
hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size]
return logits, hidden_states
# Use the corresponding lm_head for this layer
logits = self.lm_head[layer_idx](hidden_state[:, -1:, :])
code = torch.ops.vllm.qwen3_omni_code_predictor_sample(logits, self.layer_name)
pos_codes.append(code)
# Update prev_embed for next layer (if not last layer)
# layer_idx=0 predicts layer 1, embed with codec_embedding[1]
new_embed = self.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size]
current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size]
pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1]
return pos_all_layers, current_input
def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with mapping for fused QKV and gate_up projections.
@@ -4,11 +4,6 @@ from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers.generation.logits_process import (
LogitsProcessorList,
TopKLogitsWarper,
TopPLogitsWarper,
)
from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import (
Qwen3OmniMoeTalkerConfig,
)
@@ -140,7 +135,13 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
self.code_predictor = Qwen3OmniMoeTalkerCodePredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "code_predictor")
)
self.empty_code = torch.empty((1, 0), dtype=torch.long)
max_batch_size = max(
vllm_config.scheduler_config.max_num_seqs, vllm_config.compilation_config.max_cudagraph_capture_size
)
self.layer0_embed_buffer = torch.zeros(
(max_batch_size, 1, self.config.text_config.hidden_size),
dtype=vllm_config.model_config.dtype,
)
def code_predictor_forward(
self,
@@ -192,62 +193,18 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
all_codes_per_position = []
middle_hidden_states = [] # Collect hidden states for each position
logits_processors = LogitsProcessorList(
[
TopKLogitsWarper(top_k=top_k),
TopPLogitsWarper(top_p=top_p),
]
)
# Generate residual layers for each position
for pos in range(seq_len):
layer0_code = input_ids[:, pos : pos + 1] # [batch, 1]
# Predict all residual layers (layers 1 to num_code_groups-1) autoregressively
pos_codes = [layer0_code] # Start with layer 0: [batch, 1]
# Initial input: [last_talker_hidden, layer0_embed]
layer0_embed = self.embed_input_ids(layer0_code)
prev_embed = layer0_embed # Track previous layer embedding
try:
current_input = torch.cat([last_talker_hidden, prev_embed], dim=1) # [batch, 2, hidden_size]
except Exception as e:
print(f"Error in current_input: {e}")
print(f"last_talker_hidden shape: {last_talker_hidden.shape}")
print(f"prev_embed shape: {prev_embed.shape}")
raise e
for layer_idx in range(self.num_code_groups - 1):
# Input for this layer: [last_talker_hidden, prev_layer_embed]
# Forward through code_predictor model
outputs = self.code_predictor.model(
inputs_embeds=current_input,
attention_mask=None,
position_ids=None,
past_key_values=None,
use_cache=False,
cache_position=None,
)
hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size]
# Use the corresponding lm_head for this layer
logits = self.code_predictor.lm_head[layer_idx](hidden_state[:, -1:, :]) # [batch, 1, vocab_size]
logits = logits_processors(None, logits[:, -1])
# Sample from the filtered distribution
probs = F.softmax(logits, dim=-1)
code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1]
pos_codes.append(code)
# Update prev_embed for next layer (if not last layer)
# layer_idx=0 predicts layer 1, embed with codec_embedding[1]
new_embed = self.code_predictor.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size]
current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size]
self.layer0_embed_buffer[:batch_size].copy_(layer0_embed)
pos_all_layers, current_input = self.code_predictor(
layer0_code, self.layer0_embed_buffer[:batch_size], last_talker_hidden
)
# Stack all layers for this position: [batch, num_code_groups, 1]
pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1]
all_codes_per_position.append(pos_all_layers)
middle_hidden_states.append(current_input[:, 2:-1, :])
@@ -563,7 +520,10 @@ class Qwen3OmniMoeTalkerSharedExpertWrapper(nn.Module):
- mlp.shared_expert.{gate_proj, up_proj, down_proj}.weight
- mlp.shared_expert_gate.weight (sibling, not child)
The wrapper applies: sigmoid(shared_expert_gate(x)) * shared_expert(x)
The wrapper applies: sigmoid(shared_expert_gate(x)) * shared_expert(x).
It also exposes the underlying shared_expert interface to keep
compatibility with backends that split shared-expert computation.
"""
def __init__(
@@ -575,9 +535,30 @@ class Qwen3OmniMoeTalkerSharedExpertWrapper(nn.Module):
self._shared_expert = shared_expert
self._shared_expert_gate = shared_expert_gate
@property
def gate_up_proj(self):
return self._shared_expert.gate_up_proj
@property
def down_proj(self):
return self._shared_expert.down_proj
@property
def act_fn(self):
return self._shared_expert.act_fn
def expert_gate(self, x: torch.Tensor):
gate_out = self._shared_expert_gate(x)
if isinstance(gate_out, tuple):
return gate_out
return gate_out, None
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = self._shared_expert(x)
gate_values = F.sigmoid(self._shared_expert_gate(x)) # [batch, 1]
gate_out = self._shared_expert_gate(x)
if isinstance(gate_out, tuple):
gate_out = gate_out[0]
gate_values = F.sigmoid(gate_out) # [batch, 1]
return gate_values * out # Broadcasting: [batch, 1] * [batch, hidden]
@@ -43,6 +43,8 @@ from transformers.processing_utils import Unpack
from transformers.utils import can_return_tuple, logging
from transformers.utils.hub import cached_file
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from .configuration_qwen3_tts import (
Qwen3TTSConfig,
Qwen3TTSSpeakerEncoderConfig,
@@ -1860,6 +1862,15 @@ class Qwen3TTSForConditionalGeneration(Qwen3TTSPreTrainedModel, GenerationMixin)
weights_only=weights_only,
**kwargs,
)
if not local_files_only and not os.path.isdir(pretrained_model_name_or_path):
download_cache_dir = kwargs.get("cache_dir", cache_dir)
download_revision = kwargs.get("revision", revision)
download_weights_from_hf_specific(
pretrained_model_name_or_path,
cache_dir=download_cache_dir,
allow_patterns=["speech_tokenizer/*"],
revision=download_revision,
)
speech_tokenizer_path = cached_file(
pretrained_model_name_or_path,
"speech_tokenizer/config.json",
@@ -125,6 +125,14 @@ class Qwen3TTSModelForGeneration(nn.Module):
if isinstance(value, list) and len(value) > 0:
runtime_additional_information[key] = value[0]
# During profile/warmup runs, text is empty and no real inputs exist.
# Cap generation steps so the full pipeline executes (preserving
# KV-cache profiling behaviour) but exits quickly even if the model
# cannot converge from degenerate dummy inputs.
if not text:
logger.info("Profile run detected (empty text). Capping max_new_tokens to 2.")
runtime_additional_information["max_new_tokens"] = 2
# Call the appropriate generation method based on task_type
if task_type == "CustomVoice":
result = self.model.generate_custom_voice(
@@ -39,7 +39,6 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
from transformers.processing_utils import Unpack
from transformers.utils import ModelOutput, auto_docstring, logging
from transformers.utils.deprecation import deprecate_kwarg
from transformers.utils.generic import check_model_inputs
from .configuration_qwen3_tts_tokenizer_v2 import (
Qwen3TTSTokenizerV2Config,
@@ -495,7 +494,8 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr
# Initialize weights and apply final processing
self.post_init()
@check_model_inputs()
# Note: @check_model_inputs decorator removed for vLLM compatibility
# The decorator causes "unexpected keyword argument 'inputs_embeds'" error
@auto_docstring
def forward(
self,