[CI] fix ci and format existing code (#71)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu authored and GitHub committed 2025-11-18 20:54:26 +08:00
1 parent 09fab2ecc9
commit 5707fc78d5
52 files changed
+813 -2155

No files matched your search

+55 -142
View File
@@ -1,7 +1,8 @@
import glob
import os
from collections.abc import Iterable
from functools import cached_property
from typing import Dict, Iterable, NamedTuple, Optional, Set, Tuple, Union
from typing import NamedTuple, Optional, Union
import numpy as np
import torch
@@ -11,7 +12,6 @@ from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
Qwen2_5OmniTalkerConfig,
Qwen2_5OmniThinkerConfig,
)
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
@@ -29,9 +29,8 @@ from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
TALKER_CODEC_EOS_TOKEN_ID = 8294
@@ -57,7 +56,6 @@ logger = init_logger(__name__)
class Qwen2_5OmniForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, Qwen2_5OmniConditionalGenerationMixin
):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.have_multimodal_outputs = True
@@ -117,17 +115,15 @@ class Qwen2_5OmniForConditionalGeneration(
architectures=["Qwen2_5OmniToken2WavModel"],
)
# voice resources (loaded on demand)
self._token2wav_conds: Dict[str, torch.Tensor] = {}
self._token2wav_ref_mels: Dict[str, torch.Tensor] = {}
self._token2wav_conds: dict[str, torch.Tensor] = {}
self._token2wav_ref_mels: dict[str, torch.Tensor] = {}
self.model = self.token2wav
else:
raise ValueError("Invalid model stage")
# Set up intermediate tensors
self.make_empty_intermediate_tensors = (
(self.thinker.make_empty_intermediate_tensors)
if self.model_stage == "thinker"
else lambda: None
(self.thinker.make_empty_intermediate_tensors) if self.model_stage == "thinker" else lambda: None
)
# -------------------- Device utilities --------------------
@@ -176,11 +172,7 @@ class Qwen2_5OmniForConditionalGeneration(
multimodal_embeddings=None,
) -> torch.Tensor:
if self.model_stage == "code2wav":
return (
torch.zeros_like(input_ids)
.reshape(-1, 1)
.repeat(1, self.vllm_config.model_config.get_hidden_size())
)
return torch.zeros_like(input_ids).reshape(-1, 1).repeat(1, self.vllm_config.model_config.get_hidden_size())
return self.model.get_input_embeddings(input_ids, multimodal_embeddings)
def get_multimodal_embeddings(self, **kwargs):
@@ -229,9 +221,7 @@ class Qwen2_5OmniForConditionalGeneration(
# if input_ids is None, set it to a zero tensor, in the length of the
# same as the embedding seq length
if input_ids is None:
input_ids = torch.zeros(
inputs_embeds.shape[1], dtype=torch.long, device=thinker_dev
).unsqueeze(
input_ids = torch.zeros(inputs_embeds.shape[1], dtype=torch.long, device=thinker_dev).unsqueeze(
0
) # (1, 0)
added_batch_dim = True
@@ -259,11 +249,7 @@ class Qwen2_5OmniForConditionalGeneration(
# Text-only path
return OmniOutput(
text_hidden_states=(
text_hidden_states.squeeze(0)
if added_batch_dim
else text_hidden_states
),
text_hidden_states=(text_hidden_states.squeeze(0) if added_batch_dim else text_hidden_states),
multimodal_outputs=None,
)
@@ -273,9 +259,11 @@ class Qwen2_5OmniForConditionalGeneration(
# Rules:
# - Prefill segments are wrapped with special tokens: [BOS][PAD...][EOS]
# - Decode segments consist of a single non-special token.
# - If additional_information is provided (can be a list split by request or a concatenated tensor plus a list of shapes),
# - If additional_information is provided
# (can be a list split by request or a concatenated tensor plus a list of shapes),
# then for each request, reconstruct the thinker→talker input embeddings for the Prefill segments;
# - For Decode segments, if per-request auxiliary decode embeddings are provided (optional), add them; otherwise, keep the original embedding.
# - For Decode segments, if per-request auxiliary decode embeddings are provided (optional), add them;
# otherwise, keep the original embedding.
if input_ids is None and additional_information is None:
input_ids = torch.zeros(
@@ -295,9 +283,7 @@ class Qwen2_5OmniForConditionalGeneration(
# ------- Request-scoped additional information (no cross-request concat) -------
request_ids: Optional[list[str]] = kwargs.get("request_ids") # ordered
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get(
"request_token_spans"
)
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get("request_token_spans")
addi_by_req: Optional[dict] = kwargs.get("additional_information_by_req_id")
runtime_addi = kwargs.get("runtime_additional_information")
@@ -360,35 +346,24 @@ class Qwen2_5OmniForConditionalGeneration(
req_input_ids, req_embeds = self._thinker_to_talker_prefill(
voice_type=voice_type,
output_prompt_embeds=tr.to(inputs_embeds.dtype).to(
self._module_device(self.model)
),
output_prompt_embeds=tr.to(inputs_embeds.dtype).to(self._module_device(self.model)),
output_token_ids=otoks,
thinker_prompt_embeds=pe.to(inputs_embeds.dtype).to(
self._module_device(self.model)
),
thinker_prompt_embeds=pe.to(inputs_embeds.dtype).to(self._module_device(self.model)),
prompt_token_ids=ptoks,
)
seg_len = min(span_len, req_embeds.shape[0])
inputs_embeds[s : s + seg_len] = req_embeds[:seg_len]
if (
isinstance(req_input_ids, torch.Tensor)
and req_input_ids.numel() == seg_len
):
if isinstance(req_input_ids, torch.Tensor) and req_input_ids.numel() == seg_len:
input_ids[s : s + seg_len] = req_input_ids
# Prepare per-request reply queue for subsequent decode: drop first row
if tr.ndim == 2 and tr.shape[0] > 0:
update_by_req_id.setdefault(rid, {})[
"thinker_reply_part_per_request"
] = (tr[1:].detach().to("cpu").contiguous())
update_by_req_id.setdefault(rid, {})["thinker_reply_part_per_request"] = (
tr[1:].detach().to("cpu").contiguous()
)
# ------- Decode: span_len == 1 -------
if (
not is_profile
and isinstance(request_ids, list)
and isinstance(request_token_spans, list)
):
if not is_profile and isinstance(request_ids, list) and isinstance(request_token_spans, list):
for idx_req, rid in enumerate(request_ids):
s, e = request_token_spans[idx_req]
if (int(e) - int(s)) != 1:
@@ -398,27 +373,15 @@ class Qwen2_5OmniForConditionalGeneration(
# A) runtime queue
q = None
if isinstance(rid, str):
q = runtime_addi_by_req.get(rid, {}).get(
"thinker_reply_part_per_request"
)
q = runtime_addi_by_req.get(rid, {}).get("thinker_reply_part_per_request")
if isinstance(q, torch.Tensor) and q.numel() > 0:
step_vec = q[0:1]
new_q = q[1:].detach().to("cpu").contiguous()
update_by_req_id.setdefault(rid, {})[
"thinker_reply_part_per_request"
] = new_q
update_by_req_id.setdefault(rid, {})["thinker_reply_part_per_request"] = new_q
else:
# B) per-request provided decode vector (optional)
info = (
addi_by_req.get(rid, {})
if isinstance(addi_by_req, dict)
else {}
)
dv = (
info.get("decode_output_prompt_embeds")
if isinstance(info, dict)
else None
)
info = addi_by_req.get(rid, {}) if isinstance(addi_by_req, dict) else {}
dv = info.get("decode_output_prompt_embeds") if isinstance(info, dict) else None
if isinstance(dv, torch.Tensor) and dv.numel() > 0:
step_vec = dv[0:1] if dv.ndim == 2 else dv.view(1, -1)
elif (
@@ -433,9 +396,7 @@ class Qwen2_5OmniForConditionalGeneration(
if isinstance(step_vec, torch.Tensor) and step_vec.numel() > 0:
one_id = input_ids[s : s + 1]
_, one_embed = self._thinker_to_talker_decode_one_step(
output_prompt_embeds=step_vec.to(inputs_embeds.dtype).to(
self._module_device(self.model)
),
output_prompt_embeds=step_vec.to(inputs_embeds.dtype).to(self._module_device(self.model)),
output_token_ids=one_id,
)
inputs_embeds[s] = one_embed[0]
@@ -449,15 +410,12 @@ class Qwen2_5OmniForConditionalGeneration(
multimodal_outputs: dict = None
# Return updates if any
if update_by_req_id:
multimodal_outputs = {
"additional_information_update_by_req_id": update_by_req_id
}
multimodal_outputs = {"additional_information_update_by_req_id": update_by_req_id}
if sampling_metadata is not None:
# the padding token id is set to text model's pad token id, which do not match with the talker model's word embedding size
sampling_metadata.prompt_token_ids[
sampling_metadata.prompt_token_ids == 152064
] = 8448
# the padding token id is set to text model's pad token id,
# which do not match with the talker model's word embedding size
sampling_metadata.prompt_token_ids[sampling_metadata.prompt_token_ids == 152064] = 8448
return OmniOutput(
text_hidden_states=talker_hidden,
@@ -479,9 +437,7 @@ class Qwen2_5OmniForConditionalGeneration(
code = code[1:] if code[0] == TALKER_CODEC_BOS_TOKEN_ID else code
audio_tensor = self.generate_audio(code, voice_type)
return OmniOutput(
text_hidden_states=None, multimodal_outputs={"audio": audio_tensor}
)
return OmniOutput(text_hidden_states=None, multimodal_outputs={"audio": audio_tensor})
return OmniOutput(
text_hidden_states=torch.cat(
@@ -492,9 +448,7 @@ class Qwen2_5OmniForConditionalGeneration(
).to(self._module_device(self.model)),
self.talker.thinker_to_talker_proj(
self.talker.get_input_embeddings(
torch.tensor(
[TALKER_CODEC_BOS_TOKEN_ID, TALKER_CODEC_EOS_TOKEN_ID]
)
torch.tensor([TALKER_CODEC_BOS_TOKEN_ID, TALKER_CODEC_EOS_TOKEN_ID])
.to(torch.bfloat16)
.to(self._module_device(self.model))
)
@@ -621,7 +575,6 @@ class Qwen2_5OmniForConditionalGeneration(
thinker_prompt_embeds,
prompt_token_ids,
):
talker_hf_config = self.talker_config
if hasattr(talker_hf_config, "talker_config"):
talker_hf_config = talker_hf_config.talker_config
@@ -644,9 +597,7 @@ class Qwen2_5OmniForConditionalGeneration(
input_tokens_len = len(prompt_token_ids_processed)
# the code below is from model runner in Qwen, may need to further discuss later
if input_tokens_len > 2:
prompt_token_ids_processed = [
self.talker_config.tts_codec_mask_token_id
] * (input_tokens_len - 2) + [
prompt_token_ids_processed = [self.talker_config.tts_codec_mask_token_id] * (input_tokens_len - 2) + [
self.talker_config.tts_codec_pad_token_id,
self.talker_config.tts_codec_start_token_id,
]
@@ -657,9 +608,7 @@ class Qwen2_5OmniForConditionalGeneration(
][-input_tokens_len:]
if isinstance(prompt_token_ids_processed, list):
prompt_token_ids_processed = (
torch.Tensor(prompt_token_ids_processed)
.to(torch.int64)
.to(self._module_device(self.talker))
torch.Tensor(prompt_token_ids_processed).to(torch.int64).to(self._module_device(self.talker))
)
return prompt_token_ids_processed, prompt_embeds
@@ -668,14 +617,12 @@ class Qwen2_5OmniForConditionalGeneration(
output_prompt_embeds,
output_token_ids,
):
processed_output_token_embeds = (
output_prompt_embeds + self.talker.get_input_embeddings(output_token_ids)
processed_output_token_embeds = output_prompt_embeds + self.talker.get_input_embeddings(
output_token_ids
) # for decode
return output_token_ids, processed_output_token_embeds
def compute_logits(
self, hidden_states: Union[torch.Tensor, OmniOutput]
) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: Union[torch.Tensor, OmniOutput]) -> Optional[torch.Tensor]:
# Handle OmniOutput type
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
@@ -691,9 +638,7 @@ class Qwen2_5OmniForConditionalGeneration(
# Use thinker model for sampling
return self.model.sample(logits, sampling_metadata)
def generate_speech(
self, text_tokens: torch.Tensor, voice_type: str = "default"
) -> torch.Tensor:
def generate_speech(self, text_tokens: torch.Tensor, voice_type: str = "default") -> torch.Tensor:
"""
Generate speech from text tokens using the talker and token2wav models.
This method is kept for backward compatibility and direct speech generation.
@@ -706,9 +651,7 @@ class Qwen2_5OmniForConditionalGeneration(
Audio tensor
"""
# Generate codec tokens using talker model
talker_output = self.talker(
input_ids=None, positions=None, inputs_embeds=text_tokens
)
talker_output = self.talker(input_ids=None, positions=None, inputs_embeds=text_tokens)
# Convert talker output to codec tokens
codec_tokens = self._convert_to_codec_tokens(talker_output)
@@ -735,9 +678,7 @@ class Qwen2_5OmniForConditionalGeneration(
# Suppress only codec_bos, consistent with HF generate's
# suppress_tokens behavior
bos_id = None
if hasattr(self, "talker_config") and hasattr(
self.talker_config, "tts_codec_start_token_id"
):
if hasattr(self, "talker_config") and hasattr(self.talker_config, "tts_codec_start_token_id"):
bos_id = int(getattr(self.talker_config, "tts_codec_start_token_id"))
if bos_id is not None:
logits[..., bos_id] = -1e9
@@ -756,12 +697,8 @@ class Qwen2_5OmniForConditionalGeneration(
conds = getattr(self.token2wav_config, "conds", None)
ref_mels = getattr(self.token2wav_config, "ref_mels", None)
if isinstance(conds, dict) and isinstance(ref_mels, dict):
self._token2wav_conds = {
k: torch.as_tensor(v, device=device) for k, v in conds.items()
}
self._token2wav_ref_mels = {
k: torch.as_tensor(v, device=device) for k, v in ref_mels.items()
}
self._token2wav_conds = {k: torch.as_tensor(v, device=device) for k, v in conds.items()}
self._token2wav_ref_mels = {k: torch.as_tensor(v, device=device) for k, v in ref_mels.items()}
# legacy: load from directory if provided
model_path = hf_model_folder
if isinstance(model_path, str) and os.path.isdir(model_path):
@@ -773,24 +710,14 @@ class Qwen2_5OmniForConditionalGeneration(
self._token2wav_ref_mels[key] = value["ref_mel"].to(device)
else:
# legacy npy inputs
for f in sorted(
glob.glob(os.path.join(model_path, "inputs", "*spk_emb.npy"))
):
for f in sorted(glob.glob(os.path.join(model_path, "inputs", "*spk_emb.npy"))):
key = os.path.basename(f).split("_")[0].lower()
self._token2wav_conds[key] = torch.as_tensor(
np.load(f), device=device
)
for f in sorted(
glob.glob(os.path.join(model_path, "inputs", "*ref_mel.npy"))
):
self._token2wav_conds[key] = torch.as_tensor(np.load(f), device=device)
for f in sorted(glob.glob(os.path.join(model_path, "inputs", "*ref_mel.npy"))):
key = os.path.basename(f).split("_")[0].lower()
self._token2wav_ref_mels[key] = torch.as_tensor(
np.load(f), device=device
)
self._token2wav_ref_mels[key] = torch.as_tensor(np.load(f), device=device)
def _codec_to_audio(
self, codec_tokens: torch.Tensor, voice_type: str = "default"
) -> Optional[torch.Tensor]:
def _codec_to_audio(self, codec_tokens: torch.Tensor, voice_type: str = "default") -> Optional[torch.Tensor]:
if self.token2wav is None:
self._init_token2wav_model()
if self.token2wav is None:
@@ -824,9 +751,7 @@ class Qwen2_5OmniForConditionalGeneration(
if codec.ndim == 1:
codec = codec.unsqueeze(0)
else:
codec = torch.as_tensor(
codec_tokens, dtype=torch.long, device=token2wav_dev
).unsqueeze(0)
codec = torch.as_tensor(codec_tokens, dtype=torch.long, device=token2wav_dev).unsqueeze(0)
# Streaming with chunked process and boundary alignment
# (rely on token2wav.process_chunk)
@@ -841,9 +766,7 @@ class Qwen2_5OmniForConditionalGeneration(
steps = 10
# Prepare initial noise for the whole sequence
y_all = torch.randn(
(1, total_mel, mel_dim), dtype=ref_mel.dtype, device=token2wav_dev
)
y_all = torch.randn((1, total_mel, mel_dim), dtype=ref_mel.dtype, device=token2wav_dev)
logger.info(
"Currently, we do not use the chunked process, we only use the "
@@ -855,9 +778,7 @@ class Qwen2_5OmniForConditionalGeneration(
for i in range(codec.shape[1]):
chunk_code_length = i * 2 - 24
finished = i == (codec.shape[1] - 1)
if (
chunk_code_length > 0 and chunk_code_length % chunk_size == 0
) or finished:
if (chunk_code_length > 0 and chunk_code_length % chunk_size == 0) or finished:
chunk_ends.append(i)
# Number of chunks in mel domain
@@ -886,7 +807,7 @@ class Qwen2_5OmniForConditionalGeneration(
waveform = np.concatenate(wav_chunks)
return torch.as_tensor(waveform, device=token2wav_dev)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights for all components of the omni model."""
loaded_weights = set()
thinker_weights = []
@@ -915,20 +836,14 @@ class Qwen2_5OmniForConditionalGeneration(
if talker_weights and self.talker is not None:
# Map talker weights to appropriate components
if self.thinker is None:
thinker_embedding_weights = [
w
for n, w in thinker_weights
if n == "thinker.model.embed_tokens.weight"
]
thinker_embedding_weights = [w for n, w in thinker_weights if n == "thinker.model.embed_tokens.weight"]
if thinker_embedding_weights:
self.thinker_embedding = nn.Embedding(
thinker_embedding_weights[0].shape[0],
thinker_embedding_weights[0].shape[1],
)
self.thinker_embedding.weight = nn.Parameter(
thinker_embedding_weights[0].to(
self._module_device(self.talker)
)
thinker_embedding_weights[0].to(self._module_device(self.talker))
)
talker_loaded = self.talker.load_weights(talker_weights)
talker_loaded = add_prefix_to_loaded_weights(talker_loaded, "talker")
@@ -949,9 +864,7 @@ class Qwen2_5OmniForConditionalGeneration(
allow_patterns=["*.pt"],
)
self._init_token2wav_model(hf_model_folder)
t2w_loaded = self.token2wav.load_weights(
token2wav_weights, os.path.join(hf_model_folder, "spk_dict.pt")
)
t2w_loaded = self.token2wav.load_weights(token2wav_weights, os.path.join(hf_model_folder, "spk_dict.pt"))
t2w_loaded = add_prefix_to_loaded_weights(t2w_loaded, "token2wav")
loaded_weights.update(t2w_loaded)
@@ -1,24 +1,17 @@
from collections.abc import Iterable
from functools import cached_property
from typing import Iterable, Optional, Set, Tuple, Union
from typing import Optional, Union
import torch
import torch.nn as nn
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
Qwen2_5OmniTalkerConfig,
)
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
Qwen2_5OmniAudioEncoder,
)
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniTalkerConfig
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
# from vllm.attention import AttentionMetadata # unused import
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniConditionalGenerationMixin,
Qwen2_5OmniThinkerDummyInputsBuilder,
@@ -72,9 +65,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
if hasattr(config, "talker_config"):
self.config = config.talker_config
vllm_config.model_config.hf_text_config = (
vllm_config.model_config.hf_config.talker_config
)
vllm_config.model_config.hf_text_config = vllm_config.model_config.hf_config.talker_config
else:
self.config = config
@@ -92,9 +83,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
hf_config=getattr(self.config, "text_config", self.config),
architectures=["Qwen2ForCausalLM_old"],
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
def init_multi_modal(self, thinker_config):
self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
@@ -122,7 +111,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
) -> torch.Tensor:
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
# TODO (ywang96): support overlapping modalitiy embeddings so that
# `use_audio_in_video` will work on V1.
inputs_embeds = merge_multimodal_embeddings(
@@ -145,9 +133,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
inputs_embeds: Optional[torch.Tensor] = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
assert (
input_ids is not None or inputs_embeds is not None
), "input_ids or inputs_embeds must be provided"
assert input_ids is not None or inputs_embeds is not None, "input_ids or inputs_embeds must be provided"
# forward_context: ForwardContext = get_forward_context() # unused variable
if intermediate_tensors is not None:
@@ -184,7 +170,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
) -> Optional[SamplerOutput]:
return self.language_model.sample(logits, sampling_metadata)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=["thinker.", "token2wav."],
@@ -220,31 +206,15 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
for input_key in kwargs:
if (
input_key in ("pixel_values", "image_embeds")
and "image" not in mm_input_by_modality
):
mm_input_by_modality["image"] = self._parse_and_validate_image_input(
**kwargs
)
if (
input_key in ("pixel_values_videos", "video_embeds")
and "video" not in mm_input_by_modality
):
mm_input_by_modality["video"] = self._parse_and_validate_video_input(
**kwargs
)
if (
input_key in ("input_audio_features")
and "audio" not in mm_input_by_modality
):
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
**kwargs
)
if input_key in ("pixel_values", "image_embeds") and "image" not in mm_input_by_modality:
mm_input_by_modality["image"] = self._parse_and_validate_image_input(**kwargs)
if input_key in ("pixel_values_videos", "video_embeds") and "video" not in mm_input_by_modality:
mm_input_by_modality["video"] = self._parse_and_validate_video_input(**kwargs)
if input_key in ("input_audio_features") and "audio" not in mm_input_by_modality:
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(**kwargs)
return mm_input_by_modality
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return []
@@ -6,25 +6,13 @@ from typing import Any, Optional, Union
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
Qwen2_5OmniConfig,
Qwen2_5OmniThinkerConfig,
)
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
Qwen2_5OmniAudioEncoder,
)
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
Qwen2_5OmniProcessor,
)
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniConfig, Qwen2_5OmniThinkerConfig
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from vllm.model_executor.models.qwen2_5_vl import (
Qwen2_5_VisionTransformer,
Qwen2_5_VLImageEmbeddingInputs,
@@ -64,15 +52,11 @@ from vllm.multimodal.parse import (
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.processing import (
BaseMultiModalProcessor,
PlaceholderFeaturesInfo,
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing import BaseMultiModalProcessor, PlaceholderFeaturesInfo, PromptReplacement, PromptUpdate
from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
from vllm_omni.model_executor.layers.mrope import MRotaryEmbedding
try:
@@ -93,17 +77,13 @@ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
video_grid_sizes = video_grid_thw.prod(-1)
return dict(
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
"audio", audio_feature_lengths, dim=1
),
input_audio_features=MultiModalFieldConfig.flat_from_sizes("audio", audio_feature_lengths, dim=1),
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
image_grid_thw=MultiModalFieldConfig.batched("image"),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
"video", video_grid_sizes
),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
video_grid_thw=MultiModalFieldConfig.batched("video"),
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
@@ -111,7 +91,6 @@ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
def _parse_audio_data(
self,
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
@@ -127,10 +106,7 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
return super()._parse_audio_data(data)
class Qwen2_5OmniThinkerProcessingInfo(
Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo
):
class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(Qwen2_5OmniConfig).thinker_config
@@ -151,10 +127,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
return {"audio": None, "image": None, "video": None}
class Qwen2_5OmniThinkerDummyInputsBuilder(
BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]
):
class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
num_images = mm_counts.get("image", 0)
@@ -166,11 +139,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
image_token: str = hf_processor.image_token
video_token: str = hf_processor.video_token
return (
audio_token * num_audios
+ image_token * num_images
+ video_token * num_videos
)
return audio_token * num_audios + image_token * num_images + video_token * num_videos
def get_dummy_mm_data(
self,
@@ -191,17 +160,11 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
* feature_extractor.sampling_rate
)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
target_num_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts)
mm_data = {
"audio": self._get_dummy_audios(
length=target_audio_length, num_audios=num_audios
),
"image": self._get_dummy_images(
width=target_width, height=target_height, num_images=num_images
),
"audio": self._get_dummy_audios(length=target_audio_length, num_audios=num_audios),
"image": self._get_dummy_images(width=target_width, height=target_height, num_images=num_images),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
@@ -213,15 +176,10 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
return mm_data
class Qwen2_5OmniThinkerMultiModalProcessor(
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
):
class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]):
def _get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.info.get_feature_extractor()
return Qwen2_5OmniThinkerMultiModalDataParser(
target_sr=feature_extractor.sampling_rate
)
return Qwen2_5OmniThinkerMultiModalDataParser(target_sr=feature_extractor.sampling_rate)
def _call_hf_processor(
self,
@@ -252,14 +210,9 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
input_features = input_features.permute(0, 2, 1)[feature_attention_mask.bool()].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
and feature_attention_mask is not None
):
if "audio_feature_lengths" not in hf_inputs and feature_attention_mask is not None:
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
return hf_inputs
@@ -299,9 +252,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
prompt_ids,
mm_item_counts,
)
self._validate_mm_placeholders(
mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video)
tokenizer = self.info.get_tokenizer()
prompt = decode_tokens(tokenizer, prompt_ids)
@@ -315,9 +266,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
mm_prompt_updates,
mm_item_counts,
)
self._validate_mm_placeholders(
mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video)
tokenizer = self.info.get_tokenizer()
prompt = decode_tokens(tokenizer, prompt_ids)
@@ -350,15 +299,11 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
if audio_feature_lengths is None and feature_attention_mask is None:
audio_output_lengths = []
elif audio_feature_lengths is not None:
_, audio_output_lens = _get_feat_extract_output_lengths(
audio_feature_lengths
)
_, audio_output_lens = _get_feat_extract_output_lengths(audio_feature_lengths)
audio_output_lengths = audio_output_lens.tolist()
elif feature_attention_mask is not None:
assert isinstance(feature_attention_mask, torch.Tensor)
_, audio_output_lens = _get_feat_extract_output_lengths(
feature_attention_mask.sum(-1)
)
_, audio_output_lens = _get_feat_extract_output_lengths(feature_attention_mask.sum(-1))
audio_output_lengths = audio_output_lens.tolist()
# number of audios read from video.
@@ -372,8 +317,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
audios = mm_items.get_items("audio", AudioProcessorItems)
audio = audios.get(item_idx)
raise ValueError(
f"The audio {audio} (len={len(audio)}) is too short "
"to be represented inside the model"
f"The audio {audio} (len={len(audio)}) is too short to be represented inside the model"
)
return [audio_token_id] * num_features
@@ -392,9 +336,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
nonlocal audio_in_video_item_idx
audio_num_features = audio_output_lengths[
audio_in_video_item_idx + item_idx
]
audio_num_features = audio_output_lengths[audio_in_video_item_idx + item_idx]
video_grid_thw = out_mm_kwargs["video_grid_thw"][item_idx]
audio_in_video_item_idx += 1
@@ -509,39 +451,27 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
class Qwen2_5OmniConditionalGenerationMixin:
def _validate_and_reshape_mm_tensor(
self, mm_input: object, name: str, dim: int = 0
) -> torch.Tensor:
def _validate_and_reshape_mm_tensor(self, mm_input: object, name: str, dim: int = 0) -> torch.Tensor:
if not isinstance(mm_input, (torch.Tensor, list)):
raise ValueError(
f"Incorrect type of {name}. " f"Got type: {type(mm_input)}"
)
raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
if isinstance(mm_input, torch.Tensor):
return torch.concat(list(mm_input), dim=dim)
else:
return torch.concat(mm_input, dim=dim)
def _parse_and_validate_audio_input(
self, **kwargs: object
) -> Optional[Qwen2AudioInputs]:
def _parse_and_validate_audio_input(self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
input_audio_features = kwargs.pop("input_audio_features", None)
audio_feature_lengths = kwargs.pop("audio_feature_lengths", None)
feature_attention_mask = kwargs.pop("feature_attention_mask", None)
if input_audio_features is None:
return None
input_audio_features = self._validate_and_reshape_mm_tensor(
input_audio_features, "input_audio_features", dim=1
)
input_audio_features = self._validate_and_reshape_mm_tensor(input_audio_features, "input_audio_features", dim=1)
if feature_attention_mask is not None:
feature_attention_mask = self._validate_and_reshape_mm_tensor(
feature_attention_mask, "feature_attention_mask"
)
if not isinstance(input_audio_features, (torch.Tensor, list)):
raise ValueError(
"Incorrect type of audio input features. "
f"Got type: {type(input_audio_features)}"
)
raise ValueError(f"Incorrect type of audio input features. Got type: {type(input_audio_features)}")
return Qwen2AudioInputs(
input_features=input_audio_features,
audio_feature_lengths=audio_feature_lengths,
@@ -560,18 +490,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
return None
if pixel_values is not None:
pixel_values = self._validate_and_reshape_mm_tensor(
pixel_values, "image pixel values"
)
image_grid_thw = self._validate_and_reshape_mm_tensor(
image_grid_thw, "image grid_thw"
)
pixel_values = self._validate_and_reshape_mm_tensor(pixel_values, "image pixel values")
image_grid_thw = self._validate_and_reshape_mm_tensor(image_grid_thw, "image grid_thw")
if not isinstance(pixel_values, (torch.Tensor, list)):
raise ValueError(
"Incorrect type of image pixel values. "
f"Got type: {type(pixel_values)}"
)
raise ValueError(f"Incorrect type of image pixel values. Got type: {type(pixel_values)}")
return Qwen2_5_VLImagePixelInputs(
type="pixel_values",
@@ -580,18 +503,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
)
if image_embeds is not None:
image_embeds = self._validate_and_reshape_mm_tensor(
image_embeds, "image embeds"
)
image_grid_thw = self._validate_and_reshape_mm_tensor(
image_grid_thw, "image grid_thw"
)
image_embeds = self._validate_and_reshape_mm_tensor(image_embeds, "image embeds")
image_grid_thw = self._validate_and_reshape_mm_tensor(image_grid_thw, "image grid_thw")
if not isinstance(image_embeds, torch.Tensor):
raise ValueError(
"Incorrect type of image embeddings. "
f"Got type: {type(image_embeds)}"
)
raise ValueError(f"Incorrect type of image embeddings. Got type: {type(image_embeds)}")
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
@@ -610,12 +526,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
return None
if pixel_values_videos is not None:
pixel_values_videos = self._validate_and_reshape_mm_tensor(
pixel_values_videos, "video pixel values"
)
video_grid_thw = self._validate_and_reshape_mm_tensor(
video_grid_thw, "video grid_thw"
)
pixel_values_videos = self._validate_and_reshape_mm_tensor(pixel_values_videos, "video pixel values")
video_grid_thw = self._validate_and_reshape_mm_tensor(video_grid_thw, "video grid_thw")
return Qwen2_5_VLVideoPixelInputs(
type="pixel_values_videos",
@@ -624,18 +536,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
)
if video_embeds is not None:
video_embeds = self._validate_and_reshape_mm_tensor(
video_embeds, "video embeds"
)
video_grid_thw = self._validate_and_reshape_mm_tensor(
video_grid_thw, "video grid_thw"
)
video_embeds = self._validate_and_reshape_mm_tensor(video_embeds, "video embeds")
video_grid_thw = self._validate_and_reshape_mm_tensor(video_grid_thw, "video grid_thw")
if not isinstance(video_embeds, torch.Tensor):
raise ValueError(
"Incorrect type of video embeddings. "
f"Got type: {type(video_embeds)}"
)
raise ValueError(f"Incorrect type of video embeddings. Got type: {type(video_embeds)}")
return Qwen2_5_VLVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
@@ -648,24 +553,20 @@ class Qwen2_5OmniConditionalGenerationMixin:
audio_hashes: list[str] = None,
cached_audio_features: torch.Tensor = None,
) -> torch.Tensor:
input_features = audio_input["input_features"]
audio_feature_lengths = audio_input["audio_feature_lengths"]
if input_features.ndim == 3:
assert input_features.shape[0] == 1
input_features = input_features.squeeze(0)
if audio_feature_lengths.ndim == 2:
assert (
audio_feature_lengths.shape[0] == 1
or audio_feature_lengths.shape[1] == 1
)
assert audio_feature_lengths.shape[0] == 1 or audio_feature_lengths.shape[1] == 1
if audio_feature_lengths.shape[0] == 1:
audio_feature_lengths = audio_feature_lengths.squeeze(0)
else:
audio_feature_lengths = audio_feature_lengths.squeeze(1)
audio_feat_lengths, audio_output_lengths = (
self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths)
audio_feat_lengths, audio_output_lengths = self.audio_tower._get_feat_extract_output_lengths(
audio_feature_lengths
)
audio_outputs = self.audio_tower(
@@ -676,9 +577,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
audio_features = audio_outputs.last_hidden_state
return audio_features.split(audio_output_lengths.tolist())
def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs
) -> tuple[torch.Tensor, ...]:
def _process_image_input(self, image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:
if image_input["type"] == "image_embeds":
return image_input["image_embeds"].type(self.visual.dtype)
@@ -758,8 +657,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
else:
logger.warning(
"flash_attn is not available, the model may not yield the "
"exactly same result as the transformers implementation "
"in the audio tower part."
"exactly same result as the transformers implementation in the audio tower part."
)
self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
@@ -777,9 +675,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
architectures=["Qwen2ForCausalLM"],
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
@@ -787,34 +683,18 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
for input_key in kwargs:
if (
input_key in ("pixel_values", "image_embeds")
and "image" not in mm_input_by_modality
):
mm_input_by_modality["image"] = self._parse_and_validate_image_input(
**kwargs
)
if (
input_key in ("pixel_values_videos", "video_embeds")
and "video" not in mm_input_by_modality
):
mm_input_by_modality["video"] = self._parse_and_validate_video_input(
**kwargs
)
if (
input_key in ("input_audio_features")
and "audio" not in mm_input_by_modality
):
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
**kwargs
)
if input_key in ("pixel_values", "image_embeds") and "image" not in mm_input_by_modality:
mm_input_by_modality["image"] = self._parse_and_validate_image_input(**kwargs)
if input_key in ("pixel_values_videos", "video_embeds") and "video" not in mm_input_by_modality:
mm_input_by_modality["video"] = self._parse_and_validate_video_input(**kwargs)
if input_key in ("input_audio_features") and "audio" not in mm_input_by_modality:
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(**kwargs)
return mm_input_by_modality
def get_language_model(self) -> torch.nn.Module:
return self.language_model
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return []
@@ -845,7 +725,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
) -> torch.Tensor:
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
# TODO (ywang96): support overlapping modalitiy embeddings so that
# `use_audio_in_video` will work on V1.
inputs_embeds = merge_multimodal_embeddings(
@@ -897,9 +776,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
placeholder_token_id = self.config.image_token_index
if modality == "video":
placeholder_token_id = self.config.video_token_index
inputs_embeds = merge_multimodal_embeddings(
input_ids, inputs_embeds, embeddings, placeholder_token_id
)
inputs_embeds = merge_multimodal_embeddings(input_ids, inputs_embeds, embeddings, placeholder_token_id)
return inputs_embeds
def forward(
@@ -921,10 +798,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
text_inputs_embeds = self.get_input_embeddings(
input_ids,
(
[
(torch.zeros_like(embeddings), modality)
for embeddings, modality in multimodal_embeddings
]
[(torch.zeros_like(embeddings), modality) for embeddings, modality in multimodal_embeddings]
if multimodal_embeddings is not None
else None
),
@@ -3,7 +3,8 @@
############################
import math
from typing import Iterable, List, Optional, Set, Tuple, Union
from collections.abc import Iterable
from typing import Optional, Union
import numpy as np
import torch
@@ -16,24 +17,17 @@ from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
Qwen2_5OmniDiTConfig,
Qwen2_5OmniToken2WavConfig,
)
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
Qwen2_5OmniPreTrainedModel,
)
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniPreTrainedModel
# Bring in HF base classes, configs and utilities used below
from transformers.utils.logging import get_logger as _hf_get_logger
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
AutoWeightsLoader as _Vllm_AutoWeightsLoader,
)
from vllm.model_executor.models.utils import AutoWeightsLoader as _Vllm_AutoWeightsLoader
from vllm.model_executor.models.utils import WeightsMapper as _Vllm_WeightsMapper
from vllm.model_executor.models.utils import (
init_vllm_registered_model as _Vllm_init_vllm_registered_model,
)
from vllm.model_executor.models.utils import maybe_prefix as _Vllm_maybe_prefix
from vllm.model_executor.models.utils import init_vllm_registered_model as _vllm_init_vllm_registered_model
from vllm.model_executor.models.utils import maybe_prefix as _vllm_maybe_prefix
from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
@@ -218,9 +212,7 @@ class AttentiveStatisticsPooling(nn.Module):
def _compute_statistics(self, x, m, dim=2):
mean = (m * x).sum(dim)
std = torch.sqrt(
(m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(self.eps)
)
std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(self.eps))
return mean, std
def forward(self, hidden_states):
@@ -282,9 +274,7 @@ class SqueezeExcitationRes2NetBlock(nn.Module):
kernel_size=1,
dilation=1,
)
self.res2net_block = Res2NetBlock(
out_channels, out_channels, res2net_scale, kernel_size, dilation
)
self.res2net_block = Res2NetBlock(out_channels, out_channels, res2net_scale, kernel_size, dilation)
self.tdnn2 = TimeDelayNetBlock(
out_channels,
out_channels,
@@ -312,13 +302,10 @@ class ECAPA_TimeDelayNet(torch.nn.Module):
def __init__(self, config: Qwen2_5OmniDiTConfig):
super().__init__()
if len(config.enc_channels) != len(config.enc_kernel_sizes) or len(
config.enc_channels
) != len(config.enc_dilations):
raise ValueError(
"enc_channels, enc_kernel_sizes and enc_dilations should have "
"same length"
)
if len(config.enc_channels) != len(config.enc_kernel_sizes) or len(config.enc_channels) != len(
config.enc_dilations
):
raise ValueError("enc_channels, enc_kernel_sizes and enc_dilations should have same length")
self.channels = config.enc_channels
self.blocks = nn.ModuleList()
@@ -412,26 +399,14 @@ class DiTInputEmbedding(nn.Module):
):
if apply_cfg:
hidden_states = torch.cat([hidden_states, hidden_states], dim=0)
speaker_embedding = torch.cat(
[speaker_embedding, torch.zeros_like(speaker_embedding)], dim=0
)
condition_vector = torch.cat(
[condition_vector, torch.zeros_like(condition_vector)], dim=0
)
speaker_embedding = torch.cat([speaker_embedding, torch.zeros_like(speaker_embedding)], dim=0)
condition_vector = torch.cat([condition_vector, torch.zeros_like(condition_vector)], dim=0)
code_embed = torch.cat([code_embed, code_embed_uncond], dim=0)
elif drop_audio_cond: # cfg for cond audio
condition_vector = torch.zeros_like(condition_vector)
speaker_embedding = torch.zeros_like(speaker_embedding)
condition_vector = (
self.spk_encoder(condition_vector)
.unsqueeze(1)
.repeat(1, hidden_states.size(1), 1)
)
hidden_states = self.proj(
torch.cat(
(hidden_states, condition_vector, code_embed, speaker_embedding), dim=-1
)
)
condition_vector = self.spk_encoder(condition_vector).unsqueeze(1).repeat(1, hidden_states.size(1), 1)
hidden_states = self.proj(torch.cat((hidden_states, condition_vector, code_embed, speaker_embedding), dim=-1))
return hidden_states
@@ -465,13 +440,9 @@ class Qwen2_5_OmniAdaLayerNormZero(nn.Module):
def forward(self, hidden_states, emb=None):
emb = self.linear(self.silu(emb))
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = torch.chunk(
emb, 6, dim=1
)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = torch.chunk(emb, 6, dim=1)
hidden_states = (
self.norm(hidden_states) * (1 + scale_msa[:, None]) + shift_msa[:, None]
)
hidden_states = self.norm(hidden_states) * (1 + scale_msa[:, None]) + shift_msa[:, None]
return hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp
@@ -490,9 +461,7 @@ class Qwen2_5_OmniAdaLayerNormZero_Final(nn.Module):
emb = self.linear(self.silu(emb))
scale, shift = torch.chunk(emb, 2, dim=1)
hidden_states = (
self.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :]
)
hidden_states = self.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :]
return hidden_states
@@ -572,9 +541,7 @@ class DiTAttention(nn.Module):
self.to_k = nn.Linear(config.hidden_size, self.inner_dim)
self.to_v = nn.Linear(config.hidden_size, self.inner_dim)
self.to_out = nn.ModuleList(
[nn.Linear(self.inner_dim, config.hidden_size), nn.Dropout(config.dropout)]
)
self.to_out = nn.ModuleList([nn.Linear(self.inner_dim, config.hidden_size), nn.Dropout(config.dropout)])
def forward(
self,
@@ -600,9 +567,7 @@ class DiTAttention(nn.Module):
# Due to training process, only first head is applied with RoPE,
# will be fixed at next release
cos, sin = position_embeddings
query[:, :1], key[:, :1] = apply_rotary_pos_emb(
query[:, :1], key[:, :1], cos, sin
)
query[:, :1], key[:, :1] = apply_rotary_pos_emb(query[:, :1], key[:, :1], cos, sin)
attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
attention_weights, _ = attention_interface(
@@ -616,9 +581,7 @@ class DiTAttention(nn.Module):
# mask. e.g. inference got a batch with different target durations,
# mask out the padding
attention_weights = attention_weights.reshape(
batch_size, -1, self.heads * head_dim
)
attention_weights = attention_weights.reshape(batch_size, -1, self.heads * head_dim)
attention_weights = attention_weights.to(query.dtype)
# linear proj
@@ -648,9 +611,7 @@ class DiTTimestepEmbedding(nn.Module):
def __init__(self, dim, freq_embed_dim=256):
super().__init__()
self.time_embed = SinusPositionEmbedding(freq_embed_dim)
self.time_mlp = nn.ModuleList(
[nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim)]
)
self.time_mlp = nn.ModuleList([nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim)])
def forward(self, timestep): # noqa: F821
time_hidden = self.time_embed(timestep)
@@ -661,29 +622,21 @@ class DiTTimestepEmbedding(nn.Module):
class DiTDecoderLayer(nn.Module):
def __init__(
self, config: Qwen2_5OmniDiTConfig, look_ahead_block=0, look_backward_block=0
):
def __init__(self, config: Qwen2_5OmniDiTConfig, look_ahead_block=0, look_backward_block=0):
super().__init__()
self.attn_norm = Qwen2_5_OmniAdaLayerNormZero(config.hidden_size)
self.attn = DiTAttention(config)
self.look_ahead_block = look_ahead_block
self.look_backward_block = look_backward_block
self.ff_norm = nn.LayerNorm(
config.hidden_size, elementwise_affine=False, eps=1e-6
)
self.ff = DiTMLP(
dim=config.hidden_size, mult=config.ff_mult, dropout=config.dropout
)
self.ff_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False, eps=1e-6)
self.ff = DiTMLP(dim=config.hidden_size, mult=config.ff_mult, dropout=config.dropout)
def forward(
self, hidden_states, timestep, position_embeddings=None, block_diff=None
): # x: noised input, t: time embedding
# pre-norm & modulation for attention input
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(
hidden_states, emb=timestep
)
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(hidden_states, emb=timestep)
# attention
attn_output = self.attn(
@@ -696,9 +649,7 @@ class DiTDecoderLayer(nn.Module):
# process attention output for input x
hidden_states = hidden_states + gate_msa.unsqueeze(1) * attn_output
norm = (
self.ff_norm(hidden_states) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
)
norm = self.ff_norm(hidden_states) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
ff_output = self.ff(norm)
hidden_states = hidden_states + gate_mlp.unsqueeze(1) * ff_output
@@ -741,16 +692,14 @@ class SnakeBeta(nn.Module):
beta = self.beta.unsqueeze(0).unsqueeze(-1)
alpha = torch.exp(alpha)
beta = torch.exp(beta)
hidden_states = hidden_states + (
1.0 / (beta + self.no_div_by_zero)
) * torch.pow(torch.sin(hidden_states * alpha), 2)
hidden_states = hidden_states + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(
torch.sin(hidden_states * alpha), 2
)
return hidden_states
def kaiser_sinc_filter1d(
cutoff: float, half_width: float, kernel_size: int
) -> torch.Tensor:
def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) -> torch.Tensor:
"""Generates a 1D Kaiser-windowed sinc filter.
Args:
@@ -775,9 +724,7 @@ def kaiser_sinc_filter1d(
else:
beta = 0.0
kaiser_window = torch.kaiser_window(
kernel_size, beta=beta, periodic=False, dtype=torch.float32
)
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
# Compute time indices
if is_even:
@@ -787,9 +734,7 @@ def kaiser_sinc_filter1d(
# Compute sinc filter
if cutoff == 0:
return torch.zeros(
(1, 1, kernel_size), dtype=torch.float32
) # Ensures correct shape
return torch.zeros((1, 1, kernel_size), dtype=torch.float32) # Ensures correct shape
sinc_filter = torch.sinc(2 * cutoff * time_indices)
normalized_filter = 2 * cutoff * kaiser_window * sinc_filter
@@ -804,27 +749,19 @@ class UpSample1d(nn.Module):
def __init__(self, ratio=2, kernel_size=None):
super().__init__()
self.ratio = ratio
self.kernel_size = (
int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
)
self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
self.stride = ratio
self.pad = self.kernel_size // ratio - 1
self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
self.pad_right = (
self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
)
self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
filter = kaiser_sinc_filter1d(
cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size
)
filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size)
self.register_buffer("filter", filter, persistent=False)
def forward(self, hidden_states):
channels = hidden_states.shape[1]
hidden_states_dtype = hidden_states.dtype
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(
self.filter.dtype
)
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(self.filter.dtype)
hidden_states = self.ratio * F.conv_transpose1d(
hidden_states,
self.filter.expand(channels, -1, -1),
@@ -857,9 +794,7 @@ class DownSample1d(nn.Module):
def forward(self, hidden_states):
channels = hidden_states.shape[1]
hidden_states_dtype = hidden_states.dtype
hidden_states = F.pad(
hidden_states, (self.pad_left, self.pad_right), mode="replicate"
).to(self.filter.dtype)
hidden_states = F.pad(hidden_states, (self.pad_left, self.pad_right), mode="replicate").to(self.filter.dtype)
out = F.conv1d(
hidden_states,
self.filter.expand(channels, -1, -1),
@@ -960,15 +895,10 @@ class AMPBlock(torch.nn.Module):
]
)
self.num_layers = len(self.convs1) + len(
self.convs2
) # total number of conv layers
self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers
self.activations = nn.ModuleList(
[
TorchActivation1d(activation=SnakeBeta(channels))
for _ in range(self.num_layers)
]
[TorchActivation1d(activation=SnakeBeta(channels)) for _ in range(self.num_layers)]
)
def _get_padding(self, kernel_size, dilation=1):
@@ -1001,9 +931,7 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
self.num_residual_blocks = len(config.resblock_kernel_sizes)
self.num_upsample_layers = len(config.upsample_rates)
self.conv_pre = nn.Conv1d(
config.mel_dim, config.upsample_initial_channel, 7, 1, padding=3
)
self.conv_pre = nn.Conv1d(config.mel_dim, config.upsample_initial_channel, 7, 1, padding=3)
# Removing extra ModuleList breaks official state dict
ups = [
@@ -1018,9 +946,7 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
)
]
)
for layer_idx, (stride, kernel_size) in enumerate(
zip(config.upsample_rates, config.upsample_kernel_sizes)
)
for layer_idx, (stride, kernel_size) in enumerate(zip(config.upsample_rates, config.upsample_kernel_sizes))
]
self.ups = nn.ModuleList(ups)
@@ -1032,16 +958,12 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
dilation,
)
for layer_idx in range(self.num_upsample_layers)
for kernel_size, dilation in zip(
config.resblock_kernel_sizes, config.resblock_dilation_sizes
)
for kernel_size, dilation in zip(config.resblock_kernel_sizes, config.resblock_dilation_sizes)
]
)
self.activation_post = TorchActivation1d(
activation=SnakeBeta(
config.upsample_initial_channel // (2**self.num_upsample_layers)
)
activation=SnakeBeta(config.upsample_initial_channel // (2**self.num_upsample_layers))
)
self.conv_post = nn.Conv1d(
config.upsample_initial_channel // (2**self.num_upsample_layers),
@@ -1081,9 +1003,7 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
for layer_index in range(self.num_upsample_layers):
hidden_representation = self.ups[layer_index][0](hidden_representation)
residual_output = sum(
self.resblocks[layer_index * self.num_residual_blocks + block_index](
hidden_representation
)
self.resblocks[layer_index * self.num_residual_blocks + block_index](hidden_representation)
for block_index in range(self.num_residual_blocks)
)
residual_output = residual_output / self.num_residual_blocks
@@ -1111,11 +1031,7 @@ class RungeKutta4ODESolver:
value_start,
function_value_start=None,
):
k1 = (
function_value_start
if function_value_start is not None
else function(time_start, value_start)
)
k1 = function_value_start if function_value_start is not None else function(time_start, value_start)
k2 = function(
time_start + time_step * self._one_third,
value_start + time_step * k1 * self._one_third,
@@ -1141,9 +1057,7 @@ class RungeKutta4ODESolver:
function_value_start,
)
def _linear_interpolation(
self, time_start, time_end, value_start, value_end, time_point
):
def _linear_interpolation(self, time_start, time_end, value_start, value_end, time_point):
if time_point == time_start:
return value_start
if time_point == time_end:
@@ -1164,15 +1078,10 @@ class RungeKutta4ODESolver:
current_value = self.initial_value
for time_start, time_end in zip(time_points[:-1], time_points[1:]):
time_step = time_end - time_start
delta_value, _ = self._compute_step(
self.function, time_start, time_step, time_end, current_value
)
delta_value, _ = self._compute_step(self.function, time_start, time_step, time_end, current_value)
next_value = current_value + delta_value
while (
current_index < len(time_points)
and time_end >= time_points[current_index]
):
while current_index < len(time_points) and time_end >= time_points[current_index]:
solution[current_index] = self._linear_interpolation(
time_start,
time_end,
@@ -1203,9 +1112,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
self.repeats = config.repeats
self.time_embed = DiTTimestepEmbedding(config.hidden_size)
self.text_embed = DiTCodecEmbedding(
config.num_embeds, config.emb_dim, config.repeats
)
self.text_embed = DiTCodecEmbedding(config.num_embeds, config.emb_dim, config.repeats)
self.input_embed = DiTInputEmbedding(config)
self.rotary_embed = Qwen2_5OmniDiTRotaryEmbedding(config.head_dim)
@@ -1225,16 +1132,12 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
)
)
self.norm_out = Qwen2_5_OmniAdaLayerNormZero_Final(
config.hidden_size
) # final modulation
self.norm_out = Qwen2_5_OmniAdaLayerNormZero_Final(config.hidden_size) # final modulation
self.proj_out = nn.Linear(config.hidden_size, config.mel_dim)
def _create_block_diff(self, hidden_states):
batch, seq_len = hidden_states.shape[0], hidden_states.shape[1]
block_indices = (
torch.arange(seq_len, device=hidden_states.device) // self.block_size
) # [seq_length]
block_indices = torch.arange(seq_len, device=hidden_states.device) // self.block_size # [seq_length]
block_i = block_indices.unsqueeze(1) # [seq_length, 1]
block_j = block_indices.unsqueeze(0) # [1, seq_length]
@@ -1259,12 +1162,8 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
# Compute embeddings
time_embedding = self.time_embed(time_step)
text_embedding = self.text_embed(
quantized_code, drop_code=False if apply_cfg else drop_code
)
text_embedding_unconditioned = (
self.text_embed(quantized_code, drop_code=True) if apply_cfg else None
)
text_embedding = self.text_embed(quantized_code, drop_code=False if apply_cfg else drop_code)
text_embedding_unconditioned = self.text_embed(quantized_code, drop_code=True) if apply_cfg else None
hidden_states = self.input_embed(
hidden_states,
@@ -1304,17 +1203,11 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
guidance_scale=0.5,
sway_coefficient=-1.0,
):
noise_initialization = torch.randn(
[1, 30000, self.mel_dim], dtype=reference_mel_spectrogram.dtype
)
noise_initialization = torch.randn([1, 30000, self.mel_dim], dtype=reference_mel_spectrogram.dtype)
maximum_duration = quantized_code.shape[1] * self.repeats
initial_state = noise_initialization[:, :maximum_duration].to(
quantized_code.device
)
initial_state = noise_initialization[:, :maximum_duration].to(quantized_code.device)
batch_size = reference_mel_spectrogram.shape[0]
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(
1, maximum_duration, 1
)
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(1, maximum_duration, 1)
if batch_size != 1:
raise ValueError("Only batch size = 1 is currently supported")
@@ -1341,10 +1234,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
apply_cfg=True,
)
guided_prediction, null_prediction = torch.chunk(model_output, 2, dim=0)
return (
guided_prediction
+ (guided_prediction - null_prediction) * guidance_scale
)
return guided_prediction + (guided_prediction - null_prediction) * guidance_scale
initial_time = 0
time_embedding = torch.linspace(
@@ -1356,13 +1246,9 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
)
if sway_coefficient is not None:
time_embedding += sway_coefficient * (
torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding
)
time_embedding += sway_coefficient * (torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding)
ode_solver = RungeKutta4ODESolver(
function=ode_function, initial_value=initial_state
)
ode_solver = RungeKutta4ODESolver(function=ode_function, initial_value=initial_state)
solution_trajectory = ode_solver.integrate(time_embedding)
generated_waveform = solution_trajectory[-1]
@@ -1393,9 +1279,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
"""
initial_state = y0.to(quantized_code.device)
batch_size = reference_mel_spectrogram.shape[0]
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(
1, initial_state.shape[1], 1
)
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(1, initial_state.shape[1], 1)
if batch_size != 1:
raise ValueError("Only batch size = 1 is currently supported")
@@ -1422,10 +1306,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
apply_cfg=True,
)
guided_prediction, null_prediction = torch.chunk(model_output, 2, dim=0)
return (
guided_prediction
+ (guided_prediction - null_prediction) * guidance_scale
)
return guided_prediction + (guided_prediction - null_prediction) * guidance_scale
initial_time = 0
time_embedding = torch.linspace(
@@ -1437,13 +1318,9 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
)
if sway_coefficient is not None:
time_embedding += sway_coefficient * (
torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding
)
time_embedding += sway_coefficient * (torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding)
ode_solver = RungeKutta4ODESolver(
function=ode_function, initial_value=initial_state
)
ode_solver = RungeKutta4ODESolver(function=ode_function, initial_value=initial_state)
solution_trajectory = ode_solver.integrate(time_embedding)
generated_waveform = solution_trajectory[-1]
@@ -1479,8 +1356,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
attn_impl = "sdpa"
elif config._attn_implementation == "eager":
logger.warning_once(
"Qwen2_5OmniToken2WavModel does not support eager attention "
"implementation, fall back to sdpa"
"Qwen2_5OmniToken2WavModel does not support eager attention implementation, fall back to sdpa"
)
attn_impl = "sdpa"
self.code2wav_dit_model = Qwen2_5OmniToken2WavDiTModel._from_config(
@@ -1503,9 +1379,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
# codec embedding size for masking EOS out-of-range
try:
self.codec_embed_size = (
self.code2wav_dit_model.text_embed.codec_embed.weight.size(0)
)
self.codec_embed_size = self.code2wav_dit_model.text_embed.codec_embed.weight.size(0)
except Exception:
self.codec_embed_size = -1
@@ -1587,7 +1461,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
steps: int,
prev_generated: torch.Tensor,
finished: bool = False,
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
"""Streaming per small chunk: returns (mel_or_None, audio_slice)."""
start_index = max(i * self.chunk_size - self.past_cache_size, 0)
end_index = min(
@@ -1595,16 +1469,8 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
codec_all.shape[1] * self.factor,
)
y0 = (
y_all[:, start_index:end_index]
.reshape(1, -1, self.code2wav_dit_model.mel_dim)
.contiguous()
)
codec = (
codec_all[:, start_index // self.factor : end_index // self.factor]
.reshape(1, -1)
.contiguous()
)
y0 = y_all[:, start_index:end_index].reshape(1, -1, self.code2wav_dit_model.mel_dim).contiguous()
codec = codec_all[:, start_index // self.factor : end_index // self.factor].reshape(1, -1).contiguous()
# generate mel for current window (B, mel_dim, T)
generated = self.process_chunk_dit_batch(
@@ -1634,9 +1500,9 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
y_all: torch.Tensor,
i: int,
steps: int,
prev_generated: Union[torch.Tensor, List[torch.Tensor]],
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
finished: bool = False,
) -> Tuple[Union[torch.Tensor, List[torch.Tensor]], torch.Tensor]:
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
"""High-level chunk API aligning to qwen2_code2wav_dit signature."""
if not isinstance(prev_generated, torch.Tensor):
prev_generated = prev_generated[0] if len(prev_generated) > 0 else None
@@ -1661,7 +1527,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
finished: bool,
prev_generated: Optional[torch.Tensor],
generated: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Align mel and audio boundaries for 50Hz-like streaming.
@@ -1678,21 +1544,13 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
mel = generated[:, :, : self.chunk_size]
elif finished:
mel_trim = generated[:, :, self.past_cache_size :]
mel = torch.cat(
[prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2
)
mel = torch.cat([prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2)
else:
if start_index == 0:
mel_trim = generated[
:, :, i * self.chunk_size : -self.future_cache_size
]
mel_trim = generated[:, :, i * self.chunk_size : -self.future_cache_size]
else:
mel_trim = generated[
:, :, self.past_cache_size : -self.future_cache_size
]
mel = torch.cat(
[prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2
)
mel_trim = generated[:, :, self.past_cache_size : -self.future_cache_size]
mel = torch.cat([prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2)
audio = self.code2wav_bigvgan_model(mel)
if i == 0:
@@ -1700,11 +1558,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
elif finished:
audio_output = audio[self.future_size * self.vocoder_hop :]
else:
audio_output = audio[
self.future_size
* self.vocoder_hop : -self.future_size
* self.vocoder_hop
]
audio_output = audio[self.future_size * self.vocoder_hop : -self.future_size * self.vocoder_hop]
return mel, audio_output
@@ -1728,9 +1582,9 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
self.config = vllm_config.model_config.hf_config
# Initialize underlying HF Token2Wav model via registry
self.token2wav = _Vllm_init_vllm_registered_model(
self.token2wav = _vllm_init_vllm_registered_model(
vllm_config=vllm_config,
prefix=_Vllm_maybe_prefix(prefix, "token2wav_model"),
prefix=_vllm_maybe_prefix(prefix, "token2wav_model"),
hf_config=self.config,
architectures=["Qwen2_5OmniToken2WavDiTModel"],
)
@@ -1782,9 +1636,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
) -> Optional[SamplerOutput]:
return None
def load_weights_without_buffers(
self, weights: Iterable[Tuple[str, torch.Tensor]]
) -> Set[str]:
def load_weights_without_buffers(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = _Vllm_AutoWeightsLoader(self)
loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
# Log load summary
@@ -1822,9 +1674,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
return registers
# remove buffers from the weights and reload them after loading weights
def remove_buffers_from_weights(
self, weights: Iterable[Tuple[str, torch.Tensor]], buffers: dict
):
def remove_buffers_from_weights(self, weights: Iterable[tuple[str, torch.Tensor]], buffers: dict):
weights_to_load = []
for key, value in weights:
if key in buffers:
@@ -1844,9 +1694,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
loaded_buffers.add(name)
return loaded_buffers
def load_weights(
self, weights: Iterable[Tuple[str, torch.Tensor]], spk_dict_path: str
) -> Set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]], spk_dict_path: str) -> set[str]:
buffers = self.find_all_registers()
weights_to_load = self.remove_buffers_from_weights(weights, buffers)
loaded = self.load_weights_without_buffers(weights_to_load)
@@ -1874,9 +1722,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
)
@torch.inference_mode()
def process_chunk_bigvgan_batch(
self, mel_batch: torch.Tensor
) -> Optional[torch.Tensor]:
def process_chunk_bigvgan_batch(self, mel_batch: torch.Tensor) -> Optional[torch.Tensor]:
# BigVGAN is not part of this wrapper; return None for parity.
return None
@@ -1891,7 +1737,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
steps: int,
prev_generated: torch.Tensor,
finished: bool = False,
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
mel = self.token2wav(
code=codec_all,
conditioning=conditioning,
@@ -1909,9 +1755,9 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
y_all: torch.Tensor,
i: int,
steps: int,
prev_generated: Union[torch.Tensor, List[torch.Tensor]],
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
finished: bool = False,
) -> Tuple[Union[torch.Tensor, List[torch.Tensor]], torch.Tensor]:
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
_mel, out = self.process_little_chunk(
conditioning=conditioning,
reference_mel=reference_mel,
@@ -1919,9 +1765,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
y_all=y_all,
i=i,
steps=steps,
prev_generated=(
prev_generated if isinstance(prev_generated, torch.Tensor) else None
),
prev_generated=(prev_generated if isinstance(prev_generated, torch.Tensor) else None),
finished=finished,
)
return _mel if _mel is not None else prev_generated, out
+26 -68
View File
@@ -1,9 +1,9 @@
from typing import Iterable, Optional, Set, Tuple, Union
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch import nn
from transformers import Qwen2Config
from vllm.attention import Attention, AttentionType
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -11,23 +11,13 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.linear import MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.pooler import Pooler, PoolingType
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead, VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import default_weight_loader, maybe_remap_kv_scale_name
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -48,7 +38,6 @@ logger = init_logger(__name__)
class Qwen2MLP(nn.Module):
def __init__(
self,
hidden_size: int,
@@ -73,10 +62,7 @@ class Qwen2MLP(nn.Module):
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. "
"Only silu is supported for now."
)
raise ValueError(f"Unsupported activation: {hidden_act}. Only silu is supported for now.")
self.act_fn = SiluAndMul()
def forward(self, x):
@@ -87,7 +73,6 @@ class Qwen2MLP(nn.Module):
class Qwen2Attention(nn.Module):
def __init__(
self,
hidden_size: int,
@@ -98,7 +83,7 @@ class Qwen2Attention(nn.Module):
head_dim: Optional[int] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
rope_scaling: Optional[Tuple] = None,
rope_scaling: Optional[tuple] = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
@@ -173,7 +158,6 @@ class Qwen2Attention(nn.Module):
class Qwen2DecoderLayer(nn.Module):
def __init__(
self,
config: Qwen2Config,
@@ -217,16 +201,14 @@ class Qwen2DecoderLayer(nn.Module):
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
residual = hidden_states
@@ -255,7 +237,6 @@ class Qwen2DecoderLayer(nn.Module):
}
)
class Qwen2Model(nn.Module):
def __init__(
self,
*,
@@ -270,27 +251,20 @@ class Qwen2Model(nn.Module):
quant_config = vllm_config.quant_config
# TODO (@robertgshaw2): see if this can be moved out
if cache_config.sliding_window is not None and hasattr(
config, "max_window_layers"
):
if cache_config.sliding_window is not None and hasattr(config, "max_window_layers"):
raise ValueError(
"Sliding window for some but all layers is not "
"supported. This model uses sliding window "
"but `max_window_layers` = {} is less than "
"`num_hidden_layers` = {}. Please open an issue "
"to discuss this feature.".format(
config.max_window_layers,
config.num_hidden_layers,
)
f"but `max_window_layers` = {config.max_window_layers} is less than "
f"`num_hidden_layers` = {config.num_hidden_layers}. Please open an issue "
"to discuss this feature."
)
self.config = config
self.quant_config = quant_config
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
):
if get_pp_group().is_first_rank or (config.tie_word_embeddings and get_pp_group().is_last_rank):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
getattr(config, "embedding_size", config.hidden_size),
@@ -348,13 +322,11 @@ class Qwen2Model(nn.Module):
residual,
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return IntermediateTensors({"hidden_states": hidden_states, "residual": residual})
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
@@ -364,19 +336,15 @@ class Qwen2Model(nn.Module):
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: Set[str] = set()
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if self.quant_config is not None and (
scale_name := self.quant_config.get_cache_scale(name)
):
if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):
# Loading kv cache quantization scales
param = params_dict[scale_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = (
loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
)
loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
weight_loader(param, loaded_weight)
loaded_params.add(scale_name)
continue
@@ -433,9 +401,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self.lora_config = lora_config
self.quant_config = quant_config
self.model = Qwen2Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.model = Qwen2Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
if get_pp_group().is_last_rank:
if config.tie_word_embeddings:
@@ -453,9 +419,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self.logits_processor = LogitsProcessor(config.vocab_size)
self.sampler = Sampler()
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.get_input_embeddings(input_ids)
@@ -467,9 +431,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
hidden_states = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
return hidden_states
def compute_logits(
@@ -487,7 +449,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
next_tokens = self.sampler(logits, sampling_metadata)
return next_tokens
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
@@ -521,9 +483,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
self.lora_config = lora_config
self.quant_config = quant_config
self.model = Qwen2Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.model = Qwen2Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
# TODO: Replace this model class with as_embedding_model(
# Qwen2ForCausalLM) after changing the default pooling method
@@ -554,9 +514,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
) -> Optional[PoolerOutput]:
return self._pooler(hidden_states, pooling_metadata)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
weights = self.hf_to_vllm_mapper.apply(weights)
weights = (
(name, data) for name, data in weights if not name.startswith("lm_head.")
)
weights = ((name, data) for name, data in weights if not name.startswith("lm_head."))
self.model.load_weights(weights)
+1 -5
View File
@@ -1,8 +1,4 @@
from vllm.model_executor.models.registry import (
_VLLM_MODELS,
_LazyRegisteredModel,
_ModelRegistry,
)
from vllm.model_executor.models.registry import _VLLM_MODELS, _LazyRegisteredModel, _ModelRegistry
_OMNI_MODELS = {
"Qwen2_5OmniForConditionalGeneration": (