Merge branch 'main' of https://github.com/vllm-project/vllm-omni
This commit is contained in:
commit
e033498600
310 files changed
+21792
-8363
No files matched your search
@@ -36,7 +36,7 @@ from vllm_omni.model_executor.model_loader.weight_utils import download_weights_
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
|
||||
from vllm_omni.model_executor.models.vision import get_llm_pos_ids_for_vision
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
TALKER_CODEC_EOS_TOKEN_ID = 8294
|
||||
TALKER_CODEC_BOS_TOKEN_ID = 8293
|
||||
@@ -247,7 +247,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
if inputs_embeds is not None and inputs_embeds.device != thinker_dev:
|
||||
inputs_embeds = inputs_embeds.to(thinker_dev)
|
||||
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# TODO: remove this hack when NPU supports batched inputs properly
|
||||
thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
|
||||
# For MRoPE, positions shape is [3, num_tokens] (T/H/W), don't slice it
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import cap_and_align_mel_length, resolve_max_mel_frames
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
# Provide a no-op auto_docstring decorator to satisfy annotations if missing
|
||||
@@ -732,10 +732,14 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) ->
|
||||
beta = 0.0
|
||||
|
||||
# TODO: When torch.kaiser_window supports NPU, remove the device="cpu" argument
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
kaiser_window = torch.kaiser_window(
|
||||
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
|
||||
).to("npu")
|
||||
elif current_omni_platform.is_xpu():
|
||||
kaiser_window = torch.kaiser_window(
|
||||
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
|
||||
).to("xpu")
|
||||
else:
|
||||
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
|
||||
|
||||
@@ -796,7 +800,7 @@ class UpSample1d(nn.Module):
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# TODO: When F.pad supports replicate mode on NPU, remove this branch
|
||||
input_dtype = hidden_states.dtype
|
||||
# F.pad in NPU doesn't support BF16 when mode is replicate.
|
||||
@@ -843,7 +847,7 @@ class DownSample1d(nn.Module):
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
input_dtype = hidden_states.dtype
|
||||
# F.pad in NPU doesn't support BF16 when mode is replicate.
|
||||
# To ensure the accuracy, manually pad the input tensor.
|
||||
@@ -1258,7 +1262,6 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
|
||||
return output
|
||||
|
||||
@torch.no_grad()
|
||||
def sample(
|
||||
self,
|
||||
conditioning_vector,
|
||||
@@ -1333,7 +1336,6 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
generated_mel_spectrogram = generated_waveform.permute(0, 2, 1)
|
||||
return generated_mel_spectrogram
|
||||
|
||||
@torch.no_grad()
|
||||
def fast_block_sample(
|
||||
self,
|
||||
conditioning_vector: torch.Tensor,
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.sample.sampler import Sampler
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, safe_tensor_reshape
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Special token IDs for Qwen3 Omni MoE
|
||||
# Reference: https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
|
||||
@@ -269,7 +269,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
# ========== Stage 1: Thinker ==========
|
||||
if self.model_stage == "thinker":
|
||||
thinker_dev = self._module_device(self.thinker)
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# Normalize to batched inputs if needed
|
||||
_added_batch_dim = False
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
@@ -308,7 +308,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
"capture_layer_indices": [0, int(accept_layer)],
|
||||
"return_hidden_states": True,
|
||||
}
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# TODO: remove this hack when NPU supports batched inputs properly
|
||||
thinker_input_ids = input_ids[0] if input_ids is not None and _added_batch_dim else input_ids
|
||||
thinker_inputs_embeds = (
|
||||
@@ -483,12 +483,19 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
talker_codes = talker_codes.unsqueeze(0).unsqueeze(0)
|
||||
talker_codes = talker_codes.expand(1, 16, -1)
|
||||
|
||||
# Use chunked decode for memory efficiency
|
||||
audio_tensor = self.code2wav.chunked_decode(
|
||||
talker_codes,
|
||||
chunk_size=300,
|
||||
left_context_size=25,
|
||||
)
|
||||
if self.vllm_config.model_config.async_chunk:
|
||||
audio_tensor = self.code2wav.chunked_decode_streaming(
|
||||
talker_codes,
|
||||
chunk_size=25,
|
||||
left_context_size=25,
|
||||
)
|
||||
else:
|
||||
# Use chunked decode for memory efficiency
|
||||
audio_tensor = self.code2wav.chunked_decode(
|
||||
talker_codes,
|
||||
chunk_size=300,
|
||||
left_context_size=25,
|
||||
)
|
||||
|
||||
return audio_tensor
|
||||
|
||||
@@ -595,7 +602,6 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
last_talker_hidden, text_step, update_dict = self.talker_preprocess_decode(
|
||||
input_ids, input_embeds, **info_dict
|
||||
)
|
||||
|
||||
update_dict["mtp_inputs"] = last_talker_hidden, text_step
|
||||
|
||||
return input_ids, input_embeds, update_dict
|
||||
@@ -624,6 +630,33 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
inputs_embeds = (inputs_embeds + text_step).reshape(-1, self.talker_config.text_config.hidden_size)
|
||||
return inputs_embeds, code_predictor_codes.squeeze(-1)
|
||||
|
||||
def _get_tts_embed(self, thinker_embed, tts_bos_thinker, tts_eos_thinker, tts_pad_thinker):
|
||||
"""Project thinker-side TTS embeddings into talker text space."""
|
||||
module_device = self._module_device(self.talker)
|
||||
|
||||
def _ensure_1x1(x: torch.Tensor) -> torch.Tensor:
|
||||
if x.ndim == 3:
|
||||
return x[0, -1:, :]
|
||||
if x.ndim == 2:
|
||||
return x[-1]
|
||||
return x.view(1, 1, -1)
|
||||
|
||||
def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor:
|
||||
if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0:
|
||||
xin = _ensure_1x1(x_opt).to(module_device)
|
||||
else:
|
||||
xin = torch.zeros(
|
||||
(1, thinker_embed.shape[-1]),
|
||||
device=module_device,
|
||||
dtype=thinker_embed.dtype,
|
||||
)
|
||||
return self.talker.text_projection(xin).to(module_device)
|
||||
|
||||
self.tts_bos_embed = _proj_from_thinker(tts_bos_thinker)
|
||||
self.tts_eos_embed = _proj_from_thinker(tts_eos_thinker)
|
||||
self.tts_pad_embed = _proj_from_thinker(tts_pad_thinker)
|
||||
return self.tts_bos_embed, self.tts_eos_embed, self.tts_pad_embed
|
||||
|
||||
def talker_preprocess_prefill(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict):
|
||||
# Containers to return per-request updates (e.g., code_predictor_hidden_per_request)
|
||||
update_dict: dict[str, dict] = {}
|
||||
@@ -647,7 +680,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
if info_dict.get("thinker_input_ids") is None
|
||||
else torch.as_tensor(info_dict.get("thinker_input_ids"), device=self._module_device(self.talker))
|
||||
)
|
||||
# Optional thinker-side TTS token embeddings (to be projected by talker)
|
||||
|
||||
tts_bos_thinker = info_dict.get("tts_bos_embed").to(
|
||||
device=self._module_device(self.talker), dtype=torch.bfloat16
|
||||
)
|
||||
@@ -700,7 +733,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
tts_pad_thinker=tts_pad_thinker,
|
||||
)
|
||||
|
||||
# Queue tailing_text_hidden for decode (drop first for next steps),
|
||||
# Queue trailing_text_hidden for decode (drop first for next steps),
|
||||
try:
|
||||
if isinstance(trailing_text_hidden, torch.Tensor) and trailing_text_hidden.numel() > 0:
|
||||
if trailing_text_hidden.ndim == 2:
|
||||
@@ -716,7 +749,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
# compatible with old shape [1,S,D]
|
||||
rem_tail = trailing_text_hidden.squeeze(0)
|
||||
if rem_tail.shape[0] > 0:
|
||||
update_dict["tailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous()
|
||||
update_dict["trailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous()
|
||||
# Also persist projected tts_pad for decode fallback if needed
|
||||
if isinstance(tts_pad_thinker, torch.Tensor):
|
||||
pad_in = tts_pad_thinker
|
||||
@@ -762,31 +795,9 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
(thinker_result_ids == self.thinker_config.video_token_id)
|
||||
).to(input_ids.device) # [t] # fmt: skip
|
||||
|
||||
# Project thinker-side TTS embeddings into talker text space (provided by thinker stage)
|
||||
talker_dev = self._module_device(self.talker)
|
||||
|
||||
def _ensure_1x1(x: torch.Tensor) -> torch.Tensor:
|
||||
if x.ndim == 3:
|
||||
return x[0, -1:, :]
|
||||
if x.ndim == 2:
|
||||
return x[-1]
|
||||
return x.view(1, 1, -1)
|
||||
|
||||
def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor:
|
||||
if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0:
|
||||
xin = _ensure_1x1(x_opt).to(talker_dev)
|
||||
else:
|
||||
xin = torch.zeros(
|
||||
(1, thinker_embed.shape[-1]),
|
||||
device=talker_dev,
|
||||
dtype=thinker_embed.dtype,
|
||||
)
|
||||
return self.talker.text_projection(xin).to(input_ids.device)
|
||||
|
||||
tts_bos_embed = _proj_from_thinker(tts_bos_thinker)
|
||||
tts_eos_embed = _proj_from_thinker(tts_eos_thinker)
|
||||
tts_pad_embed = _proj_from_thinker(tts_pad_thinker)
|
||||
self.tts_pad_embed = tts_pad_embed
|
||||
tts_bos_embed, tts_eos_embed, tts_pad_embed = self._get_tts_embed(
|
||||
thinker_embed, tts_bos_thinker, tts_eos_thinker, tts_pad_thinker
|
||||
)
|
||||
|
||||
talker_input_embeds = [] # [1 t d]
|
||||
talker_input_ids = []
|
||||
@@ -835,24 +846,58 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
|
||||
return talker_input_id, talker_input_embed, trailing_text_hidden_all
|
||||
|
||||
def _thinker_decode_to_talker_decode(
|
||||
self,
|
||||
info_dict: dict,
|
||||
device: torch.device,
|
||||
update_dict,
|
||||
):
|
||||
"""
|
||||
Project thinker outputs to talker inputs during prefill stage.
|
||||
Returns:
|
||||
(input_ids, input_embeds) for talker
|
||||
"""
|
||||
thinker_embed = info_dict.get("thinker_embeddings", None)
|
||||
if thinker_embed is None:
|
||||
if info_dict.get("finished_flag"):
|
||||
return self.tts_pad_embed.to(device)
|
||||
update_dict["finished_flag"] = True
|
||||
return self.tts_eos_embed.to(device)
|
||||
|
||||
thinker_embed = thinker_embed.to(device)
|
||||
return self.talker.text_projection(thinker_embed).to(device)
|
||||
|
||||
def talker_preprocess_decode(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict):
|
||||
update_dict: dict[str, dict] = {}
|
||||
last_talker_hidden = None
|
||||
text_step = None
|
||||
try:
|
||||
q_tail = info_dict.get("tailing_text_hidden")
|
||||
if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0:
|
||||
use_vec = q_tail[0:1, :]
|
||||
new_q_tail = (
|
||||
q_tail[1:, :].detach().to("cpu").contiguous()
|
||||
if q_tail.shape[0] > 1
|
||||
else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
)
|
||||
text_step = use_vec.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
update_dict["tailing_text_hidden"] = new_q_tail
|
||||
if self.vllm_config.model_config.async_chunk:
|
||||
text_step = self._thinker_decode_to_talker_decode(info_dict, input_ids.device, update_dict)
|
||||
else:
|
||||
text_step = self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
q_tail = info_dict.get("trailing_text_hidden", None)
|
||||
if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0:
|
||||
use_vec = q_tail[0:1, :]
|
||||
new_q_tail = (
|
||||
q_tail[1:, :].detach().to("cpu").contiguous()
|
||||
if q_tail.shape[0] > 1
|
||||
else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
)
|
||||
text_step = use_vec.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
update_dict["trailing_text_hidden"] = new_q_tail
|
||||
else:
|
||||
text_step = self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
|
||||
last_talker_hidden = info_dict.get("last_talker_hidden").to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
last_talker_hidden = last_talker_hidden.reshape(*last_talker_hidden.shape[-2:]) # [1, hidden_size]
|
||||
last_talker_hidden_tensor = info_dict.get("last_talker_hidden")
|
||||
if last_talker_hidden_tensor is not None:
|
||||
last_talker_hidden = last_talker_hidden_tensor.to(input_embeds.device, dtype=input_embeds.dtype)
|
||||
last_talker_hidden = last_talker_hidden.reshape(*last_talker_hidden.shape[-2:]) # [1, hidden_size]
|
||||
else:
|
||||
last_talker_hidden = torch.zeros(
|
||||
(1, self.talker_config.text_config.hidden_size),
|
||||
device=input_embeds.device,
|
||||
dtype=input_embeds.dtype,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in decode: {e}")
|
||||
|
||||
@@ -866,7 +911,6 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
)
|
||||
|
||||
user_mm_mask = multimodal_mask[im_start_index:segment_end_index]
|
||||
|
||||
# Multimodal data exists
|
||||
if user_mm_mask.any():
|
||||
user_thinker_hidden_mm = thinker_hidden[im_start_index:segment_end_index][user_mm_mask]
|
||||
@@ -883,12 +927,20 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
assistant_hidden = self.talker.text_projection(thinker_embed[im_start_index:segment_end_index]).to(
|
||||
tts_pad_embed.device
|
||||
) # [t, d]
|
||||
|
||||
# [3 tokens] + [4 pad] + [1 BOS] + [1 first text] = 9 tokens
|
||||
assistant_text_hidden = torch.cat(
|
||||
(
|
||||
assistant_hidden[:3],
|
||||
tts_pad_embed.expand(4, -1),
|
||||
tts_bos_embed,
|
||||
assistant_hidden[3:4], # First text
|
||||
assistant_hidden[3:4]
|
||||
if assistant_hidden.shape[0] > 3
|
||||
else torch.zeros(
|
||||
(1, assistant_hidden.shape[1]),
|
||||
device=assistant_hidden.device,
|
||||
dtype=assistant_hidden.dtype,
|
||||
), # First text
|
||||
),
|
||||
dim=0,
|
||||
)
|
||||
@@ -904,6 +956,9 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
device=tts_pad_embed.device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
embed_input_ids = self.talker.embed_input_ids(codec_special_tokens).to(
|
||||
device=tts_pad_embed.device, dtype=torch.bfloat16
|
||||
)
|
||||
assistant_codec_hidden = torch.cat(
|
||||
(
|
||||
torch.zeros(
|
||||
@@ -911,18 +966,21 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
device=tts_pad_embed.device,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
self.talker.embed_input_ids(codec_special_tokens).to(device=tts_pad_embed.device, dtype=torch.bfloat16),
|
||||
),
|
||||
dim=0,
|
||||
)
|
||||
trailing_text_hidden = torch.cat(
|
||||
(
|
||||
assistant_hidden[4:],
|
||||
tts_eos_embed,
|
||||
embed_input_ids,
|
||||
),
|
||||
dim=0,
|
||||
)
|
||||
|
||||
if assistant_hidden.shape[0] > 4:
|
||||
trailing_text_hidden = torch.cat(
|
||||
(assistant_hidden[4:], tts_eos_embed),
|
||||
dim=0,
|
||||
)
|
||||
else:
|
||||
trailing_text_hidden = torch.zeros(
|
||||
tts_eos_embed.shape, device=tts_eos_embed.device, dtype=tts_eos_embed.dtype
|
||||
)
|
||||
|
||||
input_embeds = assistant_text_hidden + assistant_codec_hidden
|
||||
input_ids = torch.full(
|
||||
(assistant_text_hidden.shape[0],),
|
||||
|
||||
@@ -196,6 +196,39 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
|
||||
|
||||
return torch.cat(wavs, dim=-1)
|
||||
|
||||
def chunked_decode_streaming(
|
||||
self,
|
||||
codes: torch.Tensor,
|
||||
chunk_size: int = 25,
|
||||
left_context_size: int = 25,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Decode long sequences in chunks to avoid OOM.
|
||||
|
||||
Uses overlapping chunks with left context to avoid boundary artifacts.
|
||||
|
||||
Args:
|
||||
codes: [batch, num_quantizers, seq_len] - num_quantizers-layer RVQ codes
|
||||
chunk_size: Number of codec frames per chunk
|
||||
left_context_size: Number of overlapping frames for context
|
||||
|
||||
Returns:
|
||||
waveform: [batch, 1, waveform_len] - Complete waveform
|
||||
"""
|
||||
wavs = []
|
||||
end_index = codes.shape[-1]
|
||||
# TODO: need to optimize algorithms, current only support
|
||||
# chunk_size = left_context_size = 25
|
||||
if end_index <= chunk_size:
|
||||
context_size = 0
|
||||
else:
|
||||
context_size = left_context_size
|
||||
# Decode chunk
|
||||
wav_chunk = self(codes)
|
||||
# Remove context from output (context_size * total_upsample samples)
|
||||
wavs.append(wav_chunk[..., context_size * self.total_upsample :])
|
||||
return torch.cat(wavs, dim=-1)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights from HuggingFace checkpoint."""
|
||||
loader = AutoWeightsLoader(
|
||||
|
||||
@@ -13,9 +13,15 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import Cache, PretrainedConfig
|
||||
from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import Qwen3OmniMoeRotaryEmbedding
|
||||
from transformers.generation.logits_process import (
|
||||
LogitsProcessorList,
|
||||
TopKLogitsWarper,
|
||||
TopPLogitsWarper,
|
||||
)
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -24,81 +30,12 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Rotary Embeddings and Helper Functions
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class Qwen3OmniCodePredictorRotaryEmbedding(nn.Module):
|
||||
"""Rotary positional embeddings for the code predictor."""
|
||||
|
||||
def __init__(self, dim, max_position_embeddings=2048, base=10000):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.base = base
|
||||
|
||||
inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim))
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
def forward(self, x: torch.Tensor, position_ids: torch.LongTensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
inv_freq = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
||||
position = position_ids[:, None, :].float()
|
||||
freqs = (inv_freq @ position).transpose(1, 2)
|
||||
emb = torch.cat((freqs, freqs), dim=-1)
|
||||
cos = emb.cos().to(x.dtype)
|
||||
sin = emb.sin().to(x.dtype)
|
||||
return cos, sin
|
||||
|
||||
|
||||
def rotate_half(x):
|
||||
"""Rotates half the hidden dims of the input."""
|
||||
x1 = x[..., : x.shape[-1] // 2]
|
||||
x2 = x[..., x.shape[-1] // 2 :]
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
||||
"""Applies Rotary Position Embedding to the query and key tensors.
|
||||
|
||||
Args:
|
||||
q (`torch.Tensor`): The query tensor.
|
||||
k (`torch.Tensor`): The key tensor.
|
||||
cos (`torch.Tensor`): The cosine part of the rotary embedding.
|
||||
sin (`torch.Tensor`): The sine part of the rotary embedding.
|
||||
position_ids (`torch.Tensor`, *optional*):
|
||||
Deprecated and unused.
|
||||
unsqueeze_dim (`int`, *optional*, defaults to 1):
|
||||
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
|
||||
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
|
||||
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
|
||||
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
|
||||
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
|
||||
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
|
||||
Returns:
|
||||
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
||||
"""
|
||||
cos = cos.unsqueeze(unsqueeze_dim)
|
||||
sin = sin.unsqueeze(unsqueeze_dim)
|
||||
q_embed = (q * cos) + (rotate_half(q) * sin)
|
||||
k_embed = (k * cos) + (rotate_half(k) * sin)
|
||||
return q_embed, k_embed
|
||||
|
||||
|
||||
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
||||
"""Repeat key/value heads for grouped query attention."""
|
||||
if n_rep == 1:
|
||||
return hidden_states
|
||||
bsz, num_kv_heads, seq_len, head_dim = hidden_states.shape
|
||||
hidden_states = hidden_states[:, :, None, :, :].expand(bsz, num_kv_heads, n_rep, seq_len, head_dim)
|
||||
return hidden_states.reshape(bsz, num_kv_heads * n_rep, seq_len, head_dim)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Code Predictor Attention Layer
|
||||
# ============================================================================
|
||||
@@ -149,6 +86,12 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
|
||||
prefix=f"{prefix}.o_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=config.code_predictor_config.max_position_embeddings,
|
||||
rope_parameters=None,
|
||||
dual_chunk_attention_config=None,
|
||||
)
|
||||
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_key_value_heads * self.head_dim
|
||||
@@ -159,12 +102,19 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
|
||||
self.is_causal = True
|
||||
self.config = config
|
||||
|
||||
self.attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"]
|
||||
cudagraph_mode = get_current_vllm_config().compilation_config.cudagraph_mode
|
||||
if "flash_attention_2" in ALL_ATTENTION_FUNCTIONS and cudagraph_mode.has_full_cudagraphs():
|
||||
logger.warning(
|
||||
f"CUDAGraphMode.{cudagraph_mode.name} is currently not supported "
|
||||
f"with flash attention for Qwen3-Omni talker MTP."
|
||||
f"removing flash attention from attention_backends"
|
||||
)
|
||||
self.attention_backends.remove("flash_attention_2")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
causal_mask: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
past_key_values: Cache | None = None,
|
||||
cache_position: torch.LongTensor | None = None,
|
||||
use_cache: bool = False,
|
||||
@@ -176,34 +126,39 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
# Reshape for attention
|
||||
q = q.view(bsz, seq_len, self.num_heads, self.head_dim)
|
||||
k = k.view(bsz, seq_len, self.num_key_value_heads, self.head_dim)
|
||||
v = v.view(bsz, seq_len, self.num_key_value_heads, self.head_dim)
|
||||
q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim)
|
||||
k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
|
||||
v = v.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
|
||||
|
||||
# Apply normalization
|
||||
q = self.q_norm(q)
|
||||
k = self.k_norm(k)
|
||||
q = self.q_norm(q).contiguous()
|
||||
k = self.k_norm(k).contiguous()
|
||||
q = q.reshape(-1, self.q_size)
|
||||
k = k.reshape(-1, self.kv_size)
|
||||
|
||||
# Apply RoPE
|
||||
q, k = self.rotary_emb(position_ids, q, k)
|
||||
|
||||
# Reshape for attention
|
||||
q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim)
|
||||
k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
|
||||
|
||||
v_heads = v.transpose(1, 2).contiguous()
|
||||
q_heads = q.transpose(1, 2).contiguous()
|
||||
k_heads = k.transpose(1, 2).contiguous()
|
||||
q_heads, k_heads = apply_rotary_pos_emb(q_heads, k_heads, cos, sin)
|
||||
v_heads = v.transpose(1, 2).contiguous()
|
||||
|
||||
if past_key_values is not None:
|
||||
sin, cos = self.rotary_emb.get_cos_sin(seq_len)
|
||||
# sin and cos are specific to RoPE models; cache_position needed for the static cache
|
||||
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
|
||||
k_heads, v_heads = past_key_values.update(k_heads, v_heads, self.layer_idx, cache_kwargs)
|
||||
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
# Try attention backends in order of preference, with runtime error handling
|
||||
# This handles cases where the backend is registered but not actually available
|
||||
attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"]
|
||||
attn_output = None
|
||||
last_error = None
|
||||
|
||||
for backend_name in attention_backends:
|
||||
for backend_name in self.attention_backends:
|
||||
if backend_name not in ALL_ATTENTION_FUNCTIONS:
|
||||
continue
|
||||
|
||||
@@ -219,15 +174,10 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
|
||||
scaling=self.head_dim**-0.5,
|
||||
sliding_window=None,
|
||||
use_cache=use_cache,
|
||||
position_ids=position_ids,
|
||||
position_ids=position_ids[:seq_len].unsqueeze(0),
|
||||
output_hidden_states=True,
|
||||
output_attentions=False,
|
||||
)
|
||||
# Success - log fallback if not using flash_attention_2
|
||||
if backend_name != "flash_attention_2":
|
||||
logger.warning_once(
|
||||
f"Using {backend_name} attention backend (flash_attention_2 not available or failed)"
|
||||
)
|
||||
break
|
||||
except (ValueError, ImportError, RuntimeError, AttributeError) as e:
|
||||
# Store error and try next backend
|
||||
@@ -335,9 +285,6 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
|
||||
def mtp_block(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
causal_mask: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
past_key_values: Cache | None = None,
|
||||
cache_position: torch.LongTensor | None = None,
|
||||
use_cache: bool = False,
|
||||
@@ -346,9 +293,7 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
|
||||
# Self-attention with residual
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(
|
||||
hidden_states, causal_mask, cos, sin, past_key_values, cache_position, use_cache, position_ids
|
||||
)
|
||||
hidden_states = self.self_attn(hidden_states, past_key_values, cache_position, use_cache, position_ids)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
# MLP with residual
|
||||
@@ -359,47 +304,6 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None, "inputs_embeds required for MTP"
|
||||
|
||||
# Mask position 0 (not needed for MTP)
|
||||
inputs_embeds[positions == 0] = 0
|
||||
|
||||
hidden_states = torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
|
||||
|
||||
# Get position info for RoPE
|
||||
batch_size, seq_len, _ = hidden_states.shape
|
||||
device = hidden_states.device
|
||||
position_ids = torch.arange(seq_len, dtype=torch.long, device=device).unsqueeze(0).expand(batch_size, -1)
|
||||
|
||||
# Get RoPE embeddings
|
||||
head_dim = self.self_attn.head_dim
|
||||
rotary_emb = Qwen3OmniCodePredictorRotaryEmbedding(
|
||||
head_dim, max_position_embeddings=self.config.code_predictor_config.max_position_embeddings
|
||||
)
|
||||
rotary_emb = Qwen3OmniMoeRotaryEmbedding(self.config)
|
||||
cos, sin = rotary_emb(hidden_states, position_ids)
|
||||
|
||||
# Create causal mask
|
||||
causal_mask = (
|
||||
torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, device=device), diagonal=1)
|
||||
.unsqueeze(0)
|
||||
.unsqueeze(0)
|
||||
)
|
||||
causal_mask = causal_mask.masked_fill(causal_mask, float("-inf"))
|
||||
|
||||
# Forward through MTP block
|
||||
hidden_states = self.mtp_block(hidden_states, causal_mask, cos, sin)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
"""
|
||||
@@ -444,9 +348,6 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# RoPE
|
||||
self.rotary_emb = Qwen3OmniMoeRotaryEmbedding(config=config)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
@@ -462,7 +363,6 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
|
||||
Args:
|
||||
inputs_embeds: [batch, seq_len, hidden_size]
|
||||
attention_mask: Optional attention mask tensor
|
||||
position_ids: Optional position IDs tensor
|
||||
past_key_values: Optional cached key-value pairs
|
||||
use_cache: Whether to use cache
|
||||
@@ -473,37 +373,11 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
Named tuple with .last_hidden_state and .past_key_values attributes
|
||||
"""
|
||||
batch_size, seq_len, _ = inputs_embeds.shape
|
||||
|
||||
# Create positions tensor if not provided
|
||||
# positions must be [num_tokens] or [batch_size, seq_len]
|
||||
if position_ids is None:
|
||||
if cache_position is not None:
|
||||
position_ids = cache_position # [num_tokens]
|
||||
else:
|
||||
position_ids = torch.arange(seq_len, device=inputs_embeds.device).unsqueeze(0) # [1, seq_len]
|
||||
else:
|
||||
position_ids = position_ids.flatten() # Ensure [num_tokens]
|
||||
|
||||
# Extract cos/sin from rotary_emb cache
|
||||
# The cos_sin_cache is [max_pos, rotary_dim * 2]
|
||||
cos, sin = self.rotary_emb(inputs_embeds, position_ids)
|
||||
|
||||
# Create causal mask
|
||||
device = inputs_embeds.device
|
||||
causal_mask = (
|
||||
torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, device=device), diagonal=1)
|
||||
.unsqueeze(0)
|
||||
.unsqueeze(0)
|
||||
)
|
||||
causal_mask = causal_mask.masked_fill(causal_mask, float("-inf"))
|
||||
|
||||
# Forward through decoder layers
|
||||
hidden_states = inputs_embeds
|
||||
|
||||
for layer in self.layers:
|
||||
hidden_states = layer.mtp_block(
|
||||
hidden_states, causal_mask, cos, sin, past_key_values, cache_position, use_cache, position_ids
|
||||
)
|
||||
hidden_states = layer.mtp_block(hidden_states, past_key_values, cache_position, use_cache, position_ids)
|
||||
|
||||
# Final norm
|
||||
hidden_states = self.norm(hidden_states)
|
||||
@@ -517,6 +391,32 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
return self.codec_embedding
|
||||
|
||||
|
||||
def code_predictor_sample(
|
||||
logits: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
forward_context = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
logits = self.logits_processors(None, logits[:, -1])
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1]
|
||||
return code
|
||||
|
||||
|
||||
def code_predictor_sample_fake(
|
||||
logits: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((logits.shape[0], 1), dtype=torch.int64, device=logits.device)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen3_omni_code_predictor_sample",
|
||||
op_func=code_predictor_sample,
|
||||
fake_impl=code_predictor_sample_fake,
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
|
||||
"""
|
||||
@@ -552,30 +452,83 @@ class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
|
||||
for _ in range(self.num_code_groups - 1)
|
||||
]
|
||||
)
|
||||
self.logits_processors = LogitsProcessorList(
|
||||
[
|
||||
TopKLogitsWarper(top_k=50),
|
||||
TopPLogitsWarper(top_p=0.8),
|
||||
]
|
||||
)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.layer_name = prefix
|
||||
|
||||
def forward(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor, # [batch, seq_len, hidden_size]
|
||||
layer_idx: int, # Which layer to predict (0-num_layers-2 for layers 1-num_layers-1)
|
||||
layer0_code: torch.Tensor,
|
||||
layer0_embed: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for code predictor.
|
||||
|
||||
Args:
|
||||
inputs_embeds: Input embeddings [batch_size, seq_len, hidden_size]
|
||||
layer_idx: Which residual layer to predict (0-num_layers-2 for layers 1-num_layers-1)
|
||||
layer0_code:
|
||||
Code index for code-group (layer) 0.
|
||||
Shape: [batch_size, 1], dtype typically int64.
|
||||
|
||||
last_talker_hidden:
|
||||
|
||||
Shape: [batch_size, hidden_size].
|
||||
|
||||
Returns:
|
||||
logits: Predicted logits [batch_size, seq_len, vocab_size]
|
||||
hidden_states: Output hidden states [batch_size, seq_len, hidden_size]
|
||||
pos_all_layers:
|
||||
Predicted codes for all code groups, including `layer0_code`.
|
||||
Shape: [batch_size, num_code_groups, 1].
|
||||
|
||||
current_input:
|
||||
The final input embedding sequence after appending embeddings of all
|
||||
predicted codes (one token per predicted layer).
|
||||
Shape: [batch_size, num_code_groups + 2, hidden_size].
|
||||
"""
|
||||
# Pass through base model
|
||||
hidden_states = self.model(inputs_embeds)
|
||||
pos_codes = [layer0_code] # Start with layer 0: [batch, 1]
|
||||
try:
|
||||
current_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [batch, 2, hidden_size]
|
||||
except Exception as e:
|
||||
print(f"Error in current_input: {e}")
|
||||
print(f"last_talker_hidden shape: {last_talker_hidden.shape}")
|
||||
print(f"prev_embed shape: {layer0_embed.shape}")
|
||||
raise e
|
||||
batch_size = current_input.shape[0]
|
||||
|
||||
# Get logits from corresponding head
|
||||
logits = self.lm_head[layer_idx](hidden_states)
|
||||
# Predict all residual layers (layers 1 to num_code_groups-1) autoregressively
|
||||
for layer_idx in range(self.num_code_groups - 1):
|
||||
seq_len = layer_idx + 2
|
||||
# Compute position_ids dynamically to avoid torch.compile specializing batch_size
|
||||
position_ids = torch.arange(seq_len, device=current_input.device, dtype=torch.int64).repeat(batch_size)
|
||||
# Forward through code_predictor model
|
||||
outputs = self.model(
|
||||
inputs_embeds=current_input,
|
||||
attention_mask=None,
|
||||
position_ids=position_ids,
|
||||
past_key_values=None,
|
||||
use_cache=False,
|
||||
cache_position=None,
|
||||
)
|
||||
hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size]
|
||||
|
||||
return logits, hidden_states
|
||||
# Use the corresponding lm_head for this layer
|
||||
logits = self.lm_head[layer_idx](hidden_state[:, -1:, :])
|
||||
code = torch.ops.vllm.qwen3_omni_code_predictor_sample(logits, self.layer_name)
|
||||
pos_codes.append(code)
|
||||
# Update prev_embed for next layer (if not last layer)
|
||||
# layer_idx=0 predicts layer 1, embed with codec_embedding[1]
|
||||
new_embed = self.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size]
|
||||
current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size]
|
||||
pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1]
|
||||
return pos_all_layers, current_input
|
||||
|
||||
def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with mapping for fused QKV and gate_up projections.
|
||||
|
||||
@@ -4,11 +4,6 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers.generation.logits_process import (
|
||||
LogitsProcessorList,
|
||||
TopKLogitsWarper,
|
||||
TopPLogitsWarper,
|
||||
)
|
||||
from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import (
|
||||
Qwen3OmniMoeTalkerConfig,
|
||||
)
|
||||
@@ -140,7 +135,13 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
|
||||
self.code_predictor = Qwen3OmniMoeTalkerCodePredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "code_predictor")
|
||||
)
|
||||
self.empty_code = torch.empty((1, 0), dtype=torch.long)
|
||||
max_batch_size = max(
|
||||
vllm_config.scheduler_config.max_num_seqs, vllm_config.compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
self.layer0_embed_buffer = torch.zeros(
|
||||
(max_batch_size, 1, self.config.text_config.hidden_size),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
|
||||
def code_predictor_forward(
|
||||
self,
|
||||
@@ -192,62 +193,18 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
|
||||
all_codes_per_position = []
|
||||
middle_hidden_states = [] # Collect hidden states for each position
|
||||
|
||||
logits_processors = LogitsProcessorList(
|
||||
[
|
||||
TopKLogitsWarper(top_k=top_k),
|
||||
TopPLogitsWarper(top_p=top_p),
|
||||
]
|
||||
)
|
||||
# Generate residual layers for each position
|
||||
for pos in range(seq_len):
|
||||
layer0_code = input_ids[:, pos : pos + 1] # [batch, 1]
|
||||
|
||||
# Predict all residual layers (layers 1 to num_code_groups-1) autoregressively
|
||||
pos_codes = [layer0_code] # Start with layer 0: [batch, 1]
|
||||
|
||||
# Initial input: [last_talker_hidden, layer0_embed]
|
||||
layer0_embed = self.embed_input_ids(layer0_code)
|
||||
prev_embed = layer0_embed # Track previous layer embedding
|
||||
try:
|
||||
current_input = torch.cat([last_talker_hidden, prev_embed], dim=1) # [batch, 2, hidden_size]
|
||||
except Exception as e:
|
||||
print(f"Error in current_input: {e}")
|
||||
print(f"last_talker_hidden shape: {last_talker_hidden.shape}")
|
||||
print(f"prev_embed shape: {prev_embed.shape}")
|
||||
raise e
|
||||
|
||||
for layer_idx in range(self.num_code_groups - 1):
|
||||
# Input for this layer: [last_talker_hidden, prev_layer_embed]
|
||||
|
||||
# Forward through code_predictor model
|
||||
outputs = self.code_predictor.model(
|
||||
inputs_embeds=current_input,
|
||||
attention_mask=None,
|
||||
position_ids=None,
|
||||
past_key_values=None,
|
||||
use_cache=False,
|
||||
cache_position=None,
|
||||
)
|
||||
|
||||
hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size]
|
||||
|
||||
# Use the corresponding lm_head for this layer
|
||||
logits = self.code_predictor.lm_head[layer_idx](hidden_state[:, -1:, :]) # [batch, 1, vocab_size]
|
||||
|
||||
logits = logits_processors(None, logits[:, -1])
|
||||
|
||||
# Sample from the filtered distribution
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1]
|
||||
pos_codes.append(code)
|
||||
|
||||
# Update prev_embed for next layer (if not last layer)
|
||||
# layer_idx=0 predicts layer 1, embed with codec_embedding[1]
|
||||
new_embed = self.code_predictor.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size]
|
||||
current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size]
|
||||
self.layer0_embed_buffer[:batch_size].copy_(layer0_embed)
|
||||
pos_all_layers, current_input = self.code_predictor(
|
||||
layer0_code, self.layer0_embed_buffer[:batch_size], last_talker_hidden
|
||||
)
|
||||
|
||||
# Stack all layers for this position: [batch, num_code_groups, 1]
|
||||
pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1]
|
||||
all_codes_per_position.append(pos_all_layers)
|
||||
middle_hidden_states.append(current_input[:, 2:-1, :])
|
||||
|
||||
@@ -563,7 +520,10 @@ class Qwen3OmniMoeTalkerSharedExpertWrapper(nn.Module):
|
||||
- mlp.shared_expert.{gate_proj, up_proj, down_proj}.weight
|
||||
- mlp.shared_expert_gate.weight (sibling, not child)
|
||||
|
||||
The wrapper applies: sigmoid(shared_expert_gate(x)) * shared_expert(x)
|
||||
The wrapper applies: sigmoid(shared_expert_gate(x)) * shared_expert(x).
|
||||
|
||||
It also exposes the underlying shared_expert interface to keep
|
||||
compatibility with backends that split shared-expert computation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -575,9 +535,30 @@ class Qwen3OmniMoeTalkerSharedExpertWrapper(nn.Module):
|
||||
self._shared_expert = shared_expert
|
||||
self._shared_expert_gate = shared_expert_gate
|
||||
|
||||
@property
|
||||
def gate_up_proj(self):
|
||||
return self._shared_expert.gate_up_proj
|
||||
|
||||
@property
|
||||
def down_proj(self):
|
||||
return self._shared_expert.down_proj
|
||||
|
||||
@property
|
||||
def act_fn(self):
|
||||
return self._shared_expert.act_fn
|
||||
|
||||
def expert_gate(self, x: torch.Tensor):
|
||||
gate_out = self._shared_expert_gate(x)
|
||||
if isinstance(gate_out, tuple):
|
||||
return gate_out
|
||||
return gate_out, None
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
out = self._shared_expert(x)
|
||||
gate_values = F.sigmoid(self._shared_expert_gate(x)) # [batch, 1]
|
||||
gate_out = self._shared_expert_gate(x)
|
||||
if isinstance(gate_out, tuple):
|
||||
gate_out = gate_out[0]
|
||||
gate_values = F.sigmoid(gate_out) # [batch, 1]
|
||||
return gate_values * out # Broadcasting: [batch, 1] * [batch, hidden]
|
||||
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ from transformers.processing_utils import Unpack
|
||||
from transformers.utils import can_return_tuple, logging
|
||||
from transformers.utils.hub import cached_file
|
||||
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
|
||||
from .configuration_qwen3_tts import (
|
||||
Qwen3TTSConfig,
|
||||
Qwen3TTSSpeakerEncoderConfig,
|
||||
@@ -1860,6 +1862,15 @@ class Qwen3TTSForConditionalGeneration(Qwen3TTSPreTrainedModel, GenerationMixin)
|
||||
weights_only=weights_only,
|
||||
**kwargs,
|
||||
)
|
||||
if not local_files_only and not os.path.isdir(pretrained_model_name_or_path):
|
||||
download_cache_dir = kwargs.get("cache_dir", cache_dir)
|
||||
download_revision = kwargs.get("revision", revision)
|
||||
download_weights_from_hf_specific(
|
||||
pretrained_model_name_or_path,
|
||||
cache_dir=download_cache_dir,
|
||||
allow_patterns=["speech_tokenizer/*"],
|
||||
revision=download_revision,
|
||||
)
|
||||
speech_tokenizer_path = cached_file(
|
||||
pretrained_model_name_or_path,
|
||||
"speech_tokenizer/config.json",
|
||||
|
||||
@@ -125,6 +125,14 @@ class Qwen3TTSModelForGeneration(nn.Module):
|
||||
if isinstance(value, list) and len(value) > 0:
|
||||
runtime_additional_information[key] = value[0]
|
||||
|
||||
# During profile/warmup runs, text is empty and no real inputs exist.
|
||||
# Cap generation steps so the full pipeline executes (preserving
|
||||
# KV-cache profiling behaviour) but exits quickly even if the model
|
||||
# cannot converge from degenerate dummy inputs.
|
||||
if not text:
|
||||
logger.info("Profile run detected (empty text). Capping max_new_tokens to 2.")
|
||||
runtime_additional_information["max_new_tokens"] = 2
|
||||
|
||||
# Call the appropriate generation method based on task_type
|
||||
if task_type == "CustomVoice":
|
||||
result = self.model.generate_custom_voice(
|
||||
|
||||
+2
-2
@@ -39,7 +39,6 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
|
||||
from transformers.processing_utils import Unpack
|
||||
from transformers.utils import ModelOutput, auto_docstring, logging
|
||||
from transformers.utils.deprecation import deprecate_kwarg
|
||||
from transformers.utils.generic import check_model_inputs
|
||||
|
||||
from .configuration_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2Config,
|
||||
@@ -495,7 +494,8 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr
|
||||
# Initialize weights and apply final processing
|
||||
self.post_init()
|
||||
|
||||
@check_model_inputs()
|
||||
# Note: @check_model_inputs decorator removed for vLLM compatibility
|
||||
# The decorator causes "unexpected keyword argument 'inputs_embeds'" error
|
||||
@auto_docstring
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in new issue
Block a user