[CI] fix ci and format existing code (#71)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
09fab2ecc9
commit
5707fc78d5
52 files changed
+813
-2155
No files matched your search
@@ -1,7 +1,8 @@
|
||||
import glob
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Dict, Iterable, NamedTuple, Optional, Set, Tuple, Union
|
||||
from typing import NamedTuple, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -11,7 +12,6 @@ from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
||||
Qwen2_5OmniTalkerConfig,
|
||||
Qwen2_5OmniThinkerConfig,
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
|
||||
@@ -29,9 +29,8 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
|
||||
|
||||
TALKER_CODEC_EOS_TOKEN_ID = 8294
|
||||
@@ -57,7 +56,6 @@ logger = init_logger(__name__)
|
||||
class Qwen2_5OmniForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, Qwen2_5OmniConditionalGenerationMixin
|
||||
):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.have_multimodal_outputs = True
|
||||
@@ -117,17 +115,15 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
architectures=["Qwen2_5OmniToken2WavModel"],
|
||||
)
|
||||
# voice resources (loaded on demand)
|
||||
self._token2wav_conds: Dict[str, torch.Tensor] = {}
|
||||
self._token2wav_ref_mels: Dict[str, torch.Tensor] = {}
|
||||
self._token2wav_conds: dict[str, torch.Tensor] = {}
|
||||
self._token2wav_ref_mels: dict[str, torch.Tensor] = {}
|
||||
self.model = self.token2wav
|
||||
else:
|
||||
raise ValueError("Invalid model stage")
|
||||
|
||||
# Set up intermediate tensors
|
||||
self.make_empty_intermediate_tensors = (
|
||||
(self.thinker.make_empty_intermediate_tensors)
|
||||
if self.model_stage == "thinker"
|
||||
else lambda: None
|
||||
(self.thinker.make_empty_intermediate_tensors) if self.model_stage == "thinker" else lambda: None
|
||||
)
|
||||
|
||||
# -------------------- Device utilities --------------------
|
||||
@@ -176,11 +172,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
multimodal_embeddings=None,
|
||||
) -> torch.Tensor:
|
||||
if self.model_stage == "code2wav":
|
||||
return (
|
||||
torch.zeros_like(input_ids)
|
||||
.reshape(-1, 1)
|
||||
.repeat(1, self.vllm_config.model_config.get_hidden_size())
|
||||
)
|
||||
return torch.zeros_like(input_ids).reshape(-1, 1).repeat(1, self.vllm_config.model_config.get_hidden_size())
|
||||
return self.model.get_input_embeddings(input_ids, multimodal_embeddings)
|
||||
|
||||
def get_multimodal_embeddings(self, **kwargs):
|
||||
@@ -229,9 +221,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
# if input_ids is None, set it to a zero tensor, in the length of the
|
||||
# same as the embedding seq length
|
||||
if input_ids is None:
|
||||
input_ids = torch.zeros(
|
||||
inputs_embeds.shape[1], dtype=torch.long, device=thinker_dev
|
||||
).unsqueeze(
|
||||
input_ids = torch.zeros(inputs_embeds.shape[1], dtype=torch.long, device=thinker_dev).unsqueeze(
|
||||
0
|
||||
) # (1, 0)
|
||||
added_batch_dim = True
|
||||
@@ -259,11 +249,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
|
||||
# Text-only path
|
||||
return OmniOutput(
|
||||
text_hidden_states=(
|
||||
text_hidden_states.squeeze(0)
|
||||
if added_batch_dim
|
||||
else text_hidden_states
|
||||
),
|
||||
text_hidden_states=(text_hidden_states.squeeze(0) if added_batch_dim else text_hidden_states),
|
||||
multimodal_outputs=None,
|
||||
)
|
||||
|
||||
@@ -273,9 +259,11 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
# Rules:
|
||||
# - Prefill segments are wrapped with special tokens: [BOS][PAD...][EOS]
|
||||
# - Decode segments consist of a single non-special token.
|
||||
# - If additional_information is provided (can be a list split by request or a concatenated tensor plus a list of shapes),
|
||||
# - If additional_information is provided
|
||||
# (can be a list split by request or a concatenated tensor plus a list of shapes),
|
||||
# then for each request, reconstruct the thinker→talker input embeddings for the Prefill segments;
|
||||
# - For Decode segments, if per-request auxiliary decode embeddings are provided (optional), add them; otherwise, keep the original embedding.
|
||||
# - For Decode segments, if per-request auxiliary decode embeddings are provided (optional), add them;
|
||||
# otherwise, keep the original embedding.
|
||||
|
||||
if input_ids is None and additional_information is None:
|
||||
input_ids = torch.zeros(
|
||||
@@ -295,9 +283,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
|
||||
# ------- Request-scoped additional information (no cross-request concat) -------
|
||||
request_ids: Optional[list[str]] = kwargs.get("request_ids") # ordered
|
||||
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get(
|
||||
"request_token_spans"
|
||||
)
|
||||
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get("request_token_spans")
|
||||
addi_by_req: Optional[dict] = kwargs.get("additional_information_by_req_id")
|
||||
runtime_addi = kwargs.get("runtime_additional_information")
|
||||
|
||||
@@ -360,35 +346,24 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
|
||||
req_input_ids, req_embeds = self._thinker_to_talker_prefill(
|
||||
voice_type=voice_type,
|
||||
output_prompt_embeds=tr.to(inputs_embeds.dtype).to(
|
||||
self._module_device(self.model)
|
||||
),
|
||||
output_prompt_embeds=tr.to(inputs_embeds.dtype).to(self._module_device(self.model)),
|
||||
output_token_ids=otoks,
|
||||
thinker_prompt_embeds=pe.to(inputs_embeds.dtype).to(
|
||||
self._module_device(self.model)
|
||||
),
|
||||
thinker_prompt_embeds=pe.to(inputs_embeds.dtype).to(self._module_device(self.model)),
|
||||
prompt_token_ids=ptoks,
|
||||
)
|
||||
seg_len = min(span_len, req_embeds.shape[0])
|
||||
inputs_embeds[s : s + seg_len] = req_embeds[:seg_len]
|
||||
if (
|
||||
isinstance(req_input_ids, torch.Tensor)
|
||||
and req_input_ids.numel() == seg_len
|
||||
):
|
||||
if isinstance(req_input_ids, torch.Tensor) and req_input_ids.numel() == seg_len:
|
||||
input_ids[s : s + seg_len] = req_input_ids
|
||||
|
||||
# Prepare per-request reply queue for subsequent decode: drop first row
|
||||
if tr.ndim == 2 and tr.shape[0] > 0:
|
||||
update_by_req_id.setdefault(rid, {})[
|
||||
"thinker_reply_part_per_request"
|
||||
] = (tr[1:].detach().to("cpu").contiguous())
|
||||
update_by_req_id.setdefault(rid, {})["thinker_reply_part_per_request"] = (
|
||||
tr[1:].detach().to("cpu").contiguous()
|
||||
)
|
||||
|
||||
# ------- Decode: span_len == 1 -------
|
||||
if (
|
||||
not is_profile
|
||||
and isinstance(request_ids, list)
|
||||
and isinstance(request_token_spans, list)
|
||||
):
|
||||
if not is_profile and isinstance(request_ids, list) and isinstance(request_token_spans, list):
|
||||
for idx_req, rid in enumerate(request_ids):
|
||||
s, e = request_token_spans[idx_req]
|
||||
if (int(e) - int(s)) != 1:
|
||||
@@ -398,27 +373,15 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
# A) runtime queue
|
||||
q = None
|
||||
if isinstance(rid, str):
|
||||
q = runtime_addi_by_req.get(rid, {}).get(
|
||||
"thinker_reply_part_per_request"
|
||||
)
|
||||
q = runtime_addi_by_req.get(rid, {}).get("thinker_reply_part_per_request")
|
||||
if isinstance(q, torch.Tensor) and q.numel() > 0:
|
||||
step_vec = q[0:1]
|
||||
new_q = q[1:].detach().to("cpu").contiguous()
|
||||
update_by_req_id.setdefault(rid, {})[
|
||||
"thinker_reply_part_per_request"
|
||||
] = new_q
|
||||
update_by_req_id.setdefault(rid, {})["thinker_reply_part_per_request"] = new_q
|
||||
else:
|
||||
# B) per-request provided decode vector (optional)
|
||||
info = (
|
||||
addi_by_req.get(rid, {})
|
||||
if isinstance(addi_by_req, dict)
|
||||
else {}
|
||||
)
|
||||
dv = (
|
||||
info.get("decode_output_prompt_embeds")
|
||||
if isinstance(info, dict)
|
||||
else None
|
||||
)
|
||||
info = addi_by_req.get(rid, {}) if isinstance(addi_by_req, dict) else {}
|
||||
dv = info.get("decode_output_prompt_embeds") if isinstance(info, dict) else None
|
||||
if isinstance(dv, torch.Tensor) and dv.numel() > 0:
|
||||
step_vec = dv[0:1] if dv.ndim == 2 else dv.view(1, -1)
|
||||
elif (
|
||||
@@ -433,9 +396,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
if isinstance(step_vec, torch.Tensor) and step_vec.numel() > 0:
|
||||
one_id = input_ids[s : s + 1]
|
||||
_, one_embed = self._thinker_to_talker_decode_one_step(
|
||||
output_prompt_embeds=step_vec.to(inputs_embeds.dtype).to(
|
||||
self._module_device(self.model)
|
||||
),
|
||||
output_prompt_embeds=step_vec.to(inputs_embeds.dtype).to(self._module_device(self.model)),
|
||||
output_token_ids=one_id,
|
||||
)
|
||||
inputs_embeds[s] = one_embed[0]
|
||||
@@ -449,15 +410,12 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
multimodal_outputs: dict = None
|
||||
# Return updates if any
|
||||
if update_by_req_id:
|
||||
multimodal_outputs = {
|
||||
"additional_information_update_by_req_id": update_by_req_id
|
||||
}
|
||||
multimodal_outputs = {"additional_information_update_by_req_id": update_by_req_id}
|
||||
|
||||
if sampling_metadata is not None:
|
||||
# the padding token id is set to text model's pad token id, which do not match with the talker model's word embedding size
|
||||
sampling_metadata.prompt_token_ids[
|
||||
sampling_metadata.prompt_token_ids == 152064
|
||||
] = 8448
|
||||
# the padding token id is set to text model's pad token id,
|
||||
# which do not match with the talker model's word embedding size
|
||||
sampling_metadata.prompt_token_ids[sampling_metadata.prompt_token_ids == 152064] = 8448
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=talker_hidden,
|
||||
@@ -479,9 +437,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
code = code[1:] if code[0] == TALKER_CODEC_BOS_TOKEN_ID else code
|
||||
|
||||
audio_tensor = self.generate_audio(code, voice_type)
|
||||
return OmniOutput(
|
||||
text_hidden_states=None, multimodal_outputs={"audio": audio_tensor}
|
||||
)
|
||||
return OmniOutput(text_hidden_states=None, multimodal_outputs={"audio": audio_tensor})
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=torch.cat(
|
||||
@@ -492,9 +448,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
).to(self._module_device(self.model)),
|
||||
self.talker.thinker_to_talker_proj(
|
||||
self.talker.get_input_embeddings(
|
||||
torch.tensor(
|
||||
[TALKER_CODEC_BOS_TOKEN_ID, TALKER_CODEC_EOS_TOKEN_ID]
|
||||
)
|
||||
torch.tensor([TALKER_CODEC_BOS_TOKEN_ID, TALKER_CODEC_EOS_TOKEN_ID])
|
||||
.to(torch.bfloat16)
|
||||
.to(self._module_device(self.model))
|
||||
)
|
||||
@@ -621,7 +575,6 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
thinker_prompt_embeds,
|
||||
prompt_token_ids,
|
||||
):
|
||||
|
||||
talker_hf_config = self.talker_config
|
||||
if hasattr(talker_hf_config, "talker_config"):
|
||||
talker_hf_config = talker_hf_config.talker_config
|
||||
@@ -644,9 +597,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
input_tokens_len = len(prompt_token_ids_processed)
|
||||
# the code below is from model runner in Qwen, may need to further discuss later
|
||||
if input_tokens_len > 2:
|
||||
prompt_token_ids_processed = [
|
||||
self.talker_config.tts_codec_mask_token_id
|
||||
] * (input_tokens_len - 2) + [
|
||||
prompt_token_ids_processed = [self.talker_config.tts_codec_mask_token_id] * (input_tokens_len - 2) + [
|
||||
self.talker_config.tts_codec_pad_token_id,
|
||||
self.talker_config.tts_codec_start_token_id,
|
||||
]
|
||||
@@ -657,9 +608,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
][-input_tokens_len:]
|
||||
if isinstance(prompt_token_ids_processed, list):
|
||||
prompt_token_ids_processed = (
|
||||
torch.Tensor(prompt_token_ids_processed)
|
||||
.to(torch.int64)
|
||||
.to(self._module_device(self.talker))
|
||||
torch.Tensor(prompt_token_ids_processed).to(torch.int64).to(self._module_device(self.talker))
|
||||
)
|
||||
return prompt_token_ids_processed, prompt_embeds
|
||||
|
||||
@@ -668,14 +617,12 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
output_prompt_embeds,
|
||||
output_token_ids,
|
||||
):
|
||||
processed_output_token_embeds = (
|
||||
output_prompt_embeds + self.talker.get_input_embeddings(output_token_ids)
|
||||
processed_output_token_embeds = output_prompt_embeds + self.talker.get_input_embeddings(
|
||||
output_token_ids
|
||||
) # for decode
|
||||
return output_token_ids, processed_output_token_embeds
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: Union[torch.Tensor, OmniOutput]
|
||||
) -> Optional[torch.Tensor]:
|
||||
def compute_logits(self, hidden_states: Union[torch.Tensor, OmniOutput]) -> Optional[torch.Tensor]:
|
||||
# Handle OmniOutput type
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
@@ -691,9 +638,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
# Use thinker model for sampling
|
||||
return self.model.sample(logits, sampling_metadata)
|
||||
|
||||
def generate_speech(
|
||||
self, text_tokens: torch.Tensor, voice_type: str = "default"
|
||||
) -> torch.Tensor:
|
||||
def generate_speech(self, text_tokens: torch.Tensor, voice_type: str = "default") -> torch.Tensor:
|
||||
"""
|
||||
Generate speech from text tokens using the talker and token2wav models.
|
||||
This method is kept for backward compatibility and direct speech generation.
|
||||
@@ -706,9 +651,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
Audio tensor
|
||||
"""
|
||||
# Generate codec tokens using talker model
|
||||
talker_output = self.talker(
|
||||
input_ids=None, positions=None, inputs_embeds=text_tokens
|
||||
)
|
||||
talker_output = self.talker(input_ids=None, positions=None, inputs_embeds=text_tokens)
|
||||
|
||||
# Convert talker output to codec tokens
|
||||
codec_tokens = self._convert_to_codec_tokens(talker_output)
|
||||
@@ -735,9 +678,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
# Suppress only codec_bos, consistent with HF generate's
|
||||
# suppress_tokens behavior
|
||||
bos_id = None
|
||||
if hasattr(self, "talker_config") and hasattr(
|
||||
self.talker_config, "tts_codec_start_token_id"
|
||||
):
|
||||
if hasattr(self, "talker_config") and hasattr(self.talker_config, "tts_codec_start_token_id"):
|
||||
bos_id = int(getattr(self.talker_config, "tts_codec_start_token_id"))
|
||||
if bos_id is not None:
|
||||
logits[..., bos_id] = -1e9
|
||||
@@ -756,12 +697,8 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
conds = getattr(self.token2wav_config, "conds", None)
|
||||
ref_mels = getattr(self.token2wav_config, "ref_mels", None)
|
||||
if isinstance(conds, dict) and isinstance(ref_mels, dict):
|
||||
self._token2wav_conds = {
|
||||
k: torch.as_tensor(v, device=device) for k, v in conds.items()
|
||||
}
|
||||
self._token2wav_ref_mels = {
|
||||
k: torch.as_tensor(v, device=device) for k, v in ref_mels.items()
|
||||
}
|
||||
self._token2wav_conds = {k: torch.as_tensor(v, device=device) for k, v in conds.items()}
|
||||
self._token2wav_ref_mels = {k: torch.as_tensor(v, device=device) for k, v in ref_mels.items()}
|
||||
# legacy: load from directory if provided
|
||||
model_path = hf_model_folder
|
||||
if isinstance(model_path, str) and os.path.isdir(model_path):
|
||||
@@ -773,24 +710,14 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
self._token2wav_ref_mels[key] = value["ref_mel"].to(device)
|
||||
else:
|
||||
# legacy npy inputs
|
||||
for f in sorted(
|
||||
glob.glob(os.path.join(model_path, "inputs", "*spk_emb.npy"))
|
||||
):
|
||||
for f in sorted(glob.glob(os.path.join(model_path, "inputs", "*spk_emb.npy"))):
|
||||
key = os.path.basename(f).split("_")[0].lower()
|
||||
self._token2wav_conds[key] = torch.as_tensor(
|
||||
np.load(f), device=device
|
||||
)
|
||||
for f in sorted(
|
||||
glob.glob(os.path.join(model_path, "inputs", "*ref_mel.npy"))
|
||||
):
|
||||
self._token2wav_conds[key] = torch.as_tensor(np.load(f), device=device)
|
||||
for f in sorted(glob.glob(os.path.join(model_path, "inputs", "*ref_mel.npy"))):
|
||||
key = os.path.basename(f).split("_")[0].lower()
|
||||
self._token2wav_ref_mels[key] = torch.as_tensor(
|
||||
np.load(f), device=device
|
||||
)
|
||||
self._token2wav_ref_mels[key] = torch.as_tensor(np.load(f), device=device)
|
||||
|
||||
def _codec_to_audio(
|
||||
self, codec_tokens: torch.Tensor, voice_type: str = "default"
|
||||
) -> Optional[torch.Tensor]:
|
||||
def _codec_to_audio(self, codec_tokens: torch.Tensor, voice_type: str = "default") -> Optional[torch.Tensor]:
|
||||
if self.token2wav is None:
|
||||
self._init_token2wav_model()
|
||||
if self.token2wav is None:
|
||||
@@ -824,9 +751,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
if codec.ndim == 1:
|
||||
codec = codec.unsqueeze(0)
|
||||
else:
|
||||
codec = torch.as_tensor(
|
||||
codec_tokens, dtype=torch.long, device=token2wav_dev
|
||||
).unsqueeze(0)
|
||||
codec = torch.as_tensor(codec_tokens, dtype=torch.long, device=token2wav_dev).unsqueeze(0)
|
||||
|
||||
# Streaming with chunked process and boundary alignment
|
||||
# (rely on token2wav.process_chunk)
|
||||
@@ -841,9 +766,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
steps = 10
|
||||
|
||||
# Prepare initial noise for the whole sequence
|
||||
y_all = torch.randn(
|
||||
(1, total_mel, mel_dim), dtype=ref_mel.dtype, device=token2wav_dev
|
||||
)
|
||||
y_all = torch.randn((1, total_mel, mel_dim), dtype=ref_mel.dtype, device=token2wav_dev)
|
||||
|
||||
logger.info(
|
||||
"Currently, we do not use the chunked process, we only use the "
|
||||
@@ -855,9 +778,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
for i in range(codec.shape[1]):
|
||||
chunk_code_length = i * 2 - 24
|
||||
finished = i == (codec.shape[1] - 1)
|
||||
if (
|
||||
chunk_code_length > 0 and chunk_code_length % chunk_size == 0
|
||||
) or finished:
|
||||
if (chunk_code_length > 0 and chunk_code_length % chunk_size == 0) or finished:
|
||||
chunk_ends.append(i)
|
||||
|
||||
# Number of chunks in mel domain
|
||||
@@ -886,7 +807,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
waveform = np.concatenate(wav_chunks)
|
||||
return torch.as_tensor(waveform, device=token2wav_dev)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights for all components of the omni model."""
|
||||
loaded_weights = set()
|
||||
thinker_weights = []
|
||||
@@ -915,20 +836,14 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
if talker_weights and self.talker is not None:
|
||||
# Map talker weights to appropriate components
|
||||
if self.thinker is None:
|
||||
thinker_embedding_weights = [
|
||||
w
|
||||
for n, w in thinker_weights
|
||||
if n == "thinker.model.embed_tokens.weight"
|
||||
]
|
||||
thinker_embedding_weights = [w for n, w in thinker_weights if n == "thinker.model.embed_tokens.weight"]
|
||||
if thinker_embedding_weights:
|
||||
self.thinker_embedding = nn.Embedding(
|
||||
thinker_embedding_weights[0].shape[0],
|
||||
thinker_embedding_weights[0].shape[1],
|
||||
)
|
||||
self.thinker_embedding.weight = nn.Parameter(
|
||||
thinker_embedding_weights[0].to(
|
||||
self._module_device(self.talker)
|
||||
)
|
||||
thinker_embedding_weights[0].to(self._module_device(self.talker))
|
||||
)
|
||||
talker_loaded = self.talker.load_weights(talker_weights)
|
||||
talker_loaded = add_prefix_to_loaded_weights(talker_loaded, "talker")
|
||||
@@ -949,9 +864,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
allow_patterns=["*.pt"],
|
||||
)
|
||||
self._init_token2wav_model(hf_model_folder)
|
||||
t2w_loaded = self.token2wav.load_weights(
|
||||
token2wav_weights, os.path.join(hf_model_folder, "spk_dict.pt")
|
||||
)
|
||||
t2w_loaded = self.token2wav.load_weights(token2wav_weights, os.path.join(hf_model_folder, "spk_dict.pt"))
|
||||
t2w_loaded = add_prefix_to_loaded_weights(t2w_loaded, "token2wav")
|
||||
loaded_weights.update(t2w_loaded)
|
||||
|
||||
|
||||
@@ -1,24 +1,17 @@
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Iterable, Optional, Set, Tuple, Union
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
||||
Qwen2_5OmniTalkerConfig,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
|
||||
Qwen2_5OmniAudioEncoder,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniTalkerConfig
|
||||
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
|
||||
|
||||
# from vllm.attention import AttentionMetadata # unused import
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniConditionalGenerationMixin,
|
||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||
@@ -72,9 +65,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
|
||||
if hasattr(config, "talker_config"):
|
||||
self.config = config.talker_config
|
||||
vllm_config.model_config.hf_text_config = (
|
||||
vllm_config.model_config.hf_config.talker_config
|
||||
)
|
||||
vllm_config.model_config.hf_text_config = vllm_config.model_config.hf_config.talker_config
|
||||
else:
|
||||
self.config = config
|
||||
|
||||
@@ -92,9 +83,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
hf_config=getattr(self.config, "text_config", self.config),
|
||||
architectures=["Qwen2ForCausalLM_old"],
|
||||
)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
|
||||
|
||||
def init_multi_modal(self, thinker_config):
|
||||
self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
|
||||
@@ -122,7 +111,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
|
||||
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
|
||||
|
||||
# TODO (ywang96): support overlapping modalitiy embeddings so that
|
||||
# `use_audio_in_video` will work on V1.
|
||||
inputs_embeds = merge_multimodal_embeddings(
|
||||
@@ -145,9 +133,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs: object,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
assert (
|
||||
input_ids is not None or inputs_embeds is not None
|
||||
), "input_ids or inputs_embeds must be provided"
|
||||
assert input_ids is not None or inputs_embeds is not None, "input_ids or inputs_embeds must be provided"
|
||||
# forward_context: ForwardContext = get_forward_context() # unused variable
|
||||
|
||||
if intermediate_tensors is not None:
|
||||
@@ -184,7 +170,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
) -> Optional[SamplerOutput]:
|
||||
return self.language_model.sample(logits, sampling_metadata)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=["thinker.", "token2wav."],
|
||||
@@ -220,31 +206,15 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
for input_key in kwargs:
|
||||
if (
|
||||
input_key in ("pixel_values", "image_embeds")
|
||||
and "image" not in mm_input_by_modality
|
||||
):
|
||||
mm_input_by_modality["image"] = self._parse_and_validate_image_input(
|
||||
**kwargs
|
||||
)
|
||||
if (
|
||||
input_key in ("pixel_values_videos", "video_embeds")
|
||||
and "video" not in mm_input_by_modality
|
||||
):
|
||||
mm_input_by_modality["video"] = self._parse_and_validate_video_input(
|
||||
**kwargs
|
||||
)
|
||||
if (
|
||||
input_key in ("input_audio_features")
|
||||
and "audio" not in mm_input_by_modality
|
||||
):
|
||||
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
|
||||
**kwargs
|
||||
)
|
||||
if input_key in ("pixel_values", "image_embeds") and "image" not in mm_input_by_modality:
|
||||
mm_input_by_modality["image"] = self._parse_and_validate_image_input(**kwargs)
|
||||
if input_key in ("pixel_values_videos", "video_embeds") and "video" not in mm_input_by_modality:
|
||||
mm_input_by_modality["video"] = self._parse_and_validate_video_input(**kwargs)
|
||||
if input_key in ("input_audio_features") and "audio" not in mm_input_by_modality:
|
||||
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(**kwargs)
|
||||
return mm_input_by_modality
|
||||
|
||||
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
|
||||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
if not mm_input_by_modality:
|
||||
return []
|
||||
|
||||
@@ -6,25 +6,13 @@ from typing import Any, Optional, Union
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
||||
Qwen2_5OmniConfig,
|
||||
Qwen2_5OmniThinkerConfig,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
|
||||
Qwen2_5OmniAudioEncoder,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
|
||||
Qwen2_5OmniProcessor,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniConfig, Qwen2_5OmniThinkerConfig
|
||||
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
|
||||
from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessor
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from vllm.model_executor.models.qwen2_5_vl import (
|
||||
Qwen2_5_VisionTransformer,
|
||||
Qwen2_5_VLImageEmbeddingInputs,
|
||||
@@ -64,15 +52,11 @@ from vllm.multimodal.parse import (
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, PlaceholderFeaturesInfo, PromptReplacement, PromptUpdate
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
|
||||
|
||||
from vllm_omni.model_executor.layers.mrope import MRotaryEmbedding
|
||||
|
||||
try:
|
||||
@@ -93,17 +77,13 @@ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
video_grid_sizes = video_grid_thw.prod(-1)
|
||||
|
||||
return dict(
|
||||
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", audio_feature_lengths, dim=1
|
||||
),
|
||||
input_audio_features=MultiModalFieldConfig.flat_from_sizes("audio", audio_feature_lengths, dim=1),
|
||||
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
||||
audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
|
||||
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
||||
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
||||
image_grid_thw=MultiModalFieldConfig.batched("image"),
|
||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
||||
"video", video_grid_sizes
|
||||
),
|
||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
|
||||
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
|
||||
video_grid_thw=MultiModalFieldConfig.batched("video"),
|
||||
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
|
||||
@@ -111,7 +91,6 @@ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
|
||||
@@ -127,10 +106,7 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
return super()._parse_audio_data(data)
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerProcessingInfo(
|
||||
Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo
|
||||
):
|
||||
|
||||
class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config(Qwen2_5OmniConfig).thinker_config
|
||||
|
||||
@@ -151,10 +127,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
return {"audio": None, "image": None, "video": None}
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]
|
||||
):
|
||||
|
||||
class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
num_images = mm_counts.get("image", 0)
|
||||
@@ -166,11 +139,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
image_token: str = hf_processor.image_token
|
||||
video_token: str = hf_processor.video_token
|
||||
|
||||
return (
|
||||
audio_token * num_audios
|
||||
+ image_token * num_images
|
||||
+ video_token * num_videos
|
||||
)
|
||||
return audio_token * num_audios + image_token * num_images + video_token * num_videos
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
@@ -191,17 +160,11 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
* feature_extractor.sampling_rate
|
||||
)
|
||||
target_width, target_height = self.info.get_image_size_with_most_features()
|
||||
target_num_frames = self.info.get_num_frames_with_most_features(
|
||||
seq_len, mm_counts
|
||||
)
|
||||
target_num_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts)
|
||||
|
||||
mm_data = {
|
||||
"audio": self._get_dummy_audios(
|
||||
length=target_audio_length, num_audios=num_audios
|
||||
),
|
||||
"image": self._get_dummy_images(
|
||||
width=target_width, height=target_height, num_images=num_images
|
||||
),
|
||||
"audio": self._get_dummy_audios(length=target_audio_length, num_audios=num_audios),
|
||||
"image": self._get_dummy_images(width=target_width, height=target_height, num_images=num_images),
|
||||
"video": self._get_dummy_videos(
|
||||
width=target_width,
|
||||
height=target_height,
|
||||
@@ -213,15 +176,10 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
return mm_data
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
|
||||
):
|
||||
|
||||
class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]):
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
return Qwen2_5OmniThinkerMultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate
|
||||
)
|
||||
return Qwen2_5OmniThinkerMultiModalDataParser(target_sr=feature_extractor.sampling_rate)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
@@ -252,14 +210,9 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
|
||||
if "input_audio_features" not in hf_inputs and input_features is not None:
|
||||
if feature_attention_mask is not None:
|
||||
input_features = input_features.permute(0, 2, 1)[
|
||||
feature_attention_mask.bool()
|
||||
].permute(1, 0)
|
||||
input_features = input_features.permute(0, 2, 1)[feature_attention_mask.bool()].permute(1, 0)
|
||||
hf_inputs["input_audio_features"] = input_features
|
||||
if (
|
||||
"audio_feature_lengths" not in hf_inputs
|
||||
and feature_attention_mask is not None
|
||||
):
|
||||
if "audio_feature_lengths" not in hf_inputs and feature_attention_mask is not None:
|
||||
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
||||
return hf_inputs
|
||||
|
||||
@@ -299,9 +252,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
prompt_ids,
|
||||
mm_item_counts,
|
||||
)
|
||||
self._validate_mm_placeholders(
|
||||
mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video
|
||||
)
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt = decode_tokens(tokenizer, prompt_ids)
|
||||
@@ -315,9 +266,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
mm_prompt_updates,
|
||||
mm_item_counts,
|
||||
)
|
||||
self._validate_mm_placeholders(
|
||||
mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video
|
||||
)
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts, use_audio_in_video=use_audio_in_video)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt = decode_tokens(tokenizer, prompt_ids)
|
||||
@@ -350,15 +299,11 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
if audio_feature_lengths is None and feature_attention_mask is None:
|
||||
audio_output_lengths = []
|
||||
elif audio_feature_lengths is not None:
|
||||
_, audio_output_lens = _get_feat_extract_output_lengths(
|
||||
audio_feature_lengths
|
||||
)
|
||||
_, audio_output_lens = _get_feat_extract_output_lengths(audio_feature_lengths)
|
||||
audio_output_lengths = audio_output_lens.tolist()
|
||||
elif feature_attention_mask is not None:
|
||||
assert isinstance(feature_attention_mask, torch.Tensor)
|
||||
_, audio_output_lens = _get_feat_extract_output_lengths(
|
||||
feature_attention_mask.sum(-1)
|
||||
)
|
||||
_, audio_output_lens = _get_feat_extract_output_lengths(feature_attention_mask.sum(-1))
|
||||
audio_output_lengths = audio_output_lens.tolist()
|
||||
|
||||
# number of audios read from video.
|
||||
@@ -372,8 +317,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
audio = audios.get(item_idx)
|
||||
raise ValueError(
|
||||
f"The audio {audio} (len={len(audio)}) is too short "
|
||||
"to be represented inside the model"
|
||||
f"The audio {audio} (len={len(audio)}) is too short to be represented inside the model"
|
||||
)
|
||||
|
||||
return [audio_token_id] * num_features
|
||||
@@ -392,9 +336,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
|
||||
nonlocal audio_in_video_item_idx
|
||||
|
||||
audio_num_features = audio_output_lengths[
|
||||
audio_in_video_item_idx + item_idx
|
||||
]
|
||||
audio_num_features = audio_output_lengths[audio_in_video_item_idx + item_idx]
|
||||
video_grid_thw = out_mm_kwargs["video_grid_thw"][item_idx]
|
||||
|
||||
audio_in_video_item_idx += 1
|
||||
@@ -509,39 +451,27 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
|
||||
|
||||
class Qwen2_5OmniConditionalGenerationMixin:
|
||||
|
||||
def _validate_and_reshape_mm_tensor(
|
||||
self, mm_input: object, name: str, dim: int = 0
|
||||
) -> torch.Tensor:
|
||||
def _validate_and_reshape_mm_tensor(self, mm_input: object, name: str, dim: int = 0) -> torch.Tensor:
|
||||
if not isinstance(mm_input, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
f"Incorrect type of {name}. " f"Got type: {type(mm_input)}"
|
||||
)
|
||||
raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
|
||||
if isinstance(mm_input, torch.Tensor):
|
||||
return torch.concat(list(mm_input), dim=dim)
|
||||
else:
|
||||
return torch.concat(mm_input, dim=dim)
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
self, **kwargs: object
|
||||
) -> Optional[Qwen2AudioInputs]:
|
||||
def _parse_and_validate_audio_input(self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
|
||||
input_audio_features = kwargs.pop("input_audio_features", None)
|
||||
audio_feature_lengths = kwargs.pop("audio_feature_lengths", None)
|
||||
feature_attention_mask = kwargs.pop("feature_attention_mask", None)
|
||||
if input_audio_features is None:
|
||||
return None
|
||||
input_audio_features = self._validate_and_reshape_mm_tensor(
|
||||
input_audio_features, "input_audio_features", dim=1
|
||||
)
|
||||
input_audio_features = self._validate_and_reshape_mm_tensor(input_audio_features, "input_audio_features", dim=1)
|
||||
if feature_attention_mask is not None:
|
||||
feature_attention_mask = self._validate_and_reshape_mm_tensor(
|
||||
feature_attention_mask, "feature_attention_mask"
|
||||
)
|
||||
if not isinstance(input_audio_features, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
"Incorrect type of audio input features. "
|
||||
f"Got type: {type(input_audio_features)}"
|
||||
)
|
||||
raise ValueError(f"Incorrect type of audio input features. Got type: {type(input_audio_features)}")
|
||||
return Qwen2AudioInputs(
|
||||
input_features=input_audio_features,
|
||||
audio_feature_lengths=audio_feature_lengths,
|
||||
@@ -560,18 +490,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
return None
|
||||
|
||||
if pixel_values is not None:
|
||||
pixel_values = self._validate_and_reshape_mm_tensor(
|
||||
pixel_values, "image pixel values"
|
||||
)
|
||||
image_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
image_grid_thw, "image grid_thw"
|
||||
)
|
||||
pixel_values = self._validate_and_reshape_mm_tensor(pixel_values, "image pixel values")
|
||||
image_grid_thw = self._validate_and_reshape_mm_tensor(image_grid_thw, "image grid_thw")
|
||||
|
||||
if not isinstance(pixel_values, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
"Incorrect type of image pixel values. "
|
||||
f"Got type: {type(pixel_values)}"
|
||||
)
|
||||
raise ValueError(f"Incorrect type of image pixel values. Got type: {type(pixel_values)}")
|
||||
|
||||
return Qwen2_5_VLImagePixelInputs(
|
||||
type="pixel_values",
|
||||
@@ -580,18 +503,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
)
|
||||
|
||||
if image_embeds is not None:
|
||||
image_embeds = self._validate_and_reshape_mm_tensor(
|
||||
image_embeds, "image embeds"
|
||||
)
|
||||
image_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
image_grid_thw, "image grid_thw"
|
||||
)
|
||||
image_embeds = self._validate_and_reshape_mm_tensor(image_embeds, "image embeds")
|
||||
image_grid_thw = self._validate_and_reshape_mm_tensor(image_grid_thw, "image grid_thw")
|
||||
|
||||
if not isinstance(image_embeds, torch.Tensor):
|
||||
raise ValueError(
|
||||
"Incorrect type of image embeddings. "
|
||||
f"Got type: {type(image_embeds)}"
|
||||
)
|
||||
raise ValueError(f"Incorrect type of image embeddings. Got type: {type(image_embeds)}")
|
||||
return Qwen2_5_VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds,
|
||||
@@ -610,12 +526,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
return None
|
||||
|
||||
if pixel_values_videos is not None:
|
||||
pixel_values_videos = self._validate_and_reshape_mm_tensor(
|
||||
pixel_values_videos, "video pixel values"
|
||||
)
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
video_grid_thw, "video grid_thw"
|
||||
)
|
||||
pixel_values_videos = self._validate_and_reshape_mm_tensor(pixel_values_videos, "video pixel values")
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(video_grid_thw, "video grid_thw")
|
||||
|
||||
return Qwen2_5_VLVideoPixelInputs(
|
||||
type="pixel_values_videos",
|
||||
@@ -624,18 +536,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
video_embeds = self._validate_and_reshape_mm_tensor(
|
||||
video_embeds, "video embeds"
|
||||
)
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
video_grid_thw, "video grid_thw"
|
||||
)
|
||||
video_embeds = self._validate_and_reshape_mm_tensor(video_embeds, "video embeds")
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(video_grid_thw, "video grid_thw")
|
||||
|
||||
if not isinstance(video_embeds, torch.Tensor):
|
||||
raise ValueError(
|
||||
"Incorrect type of video embeddings. "
|
||||
f"Got type: {type(video_embeds)}"
|
||||
)
|
||||
raise ValueError(f"Incorrect type of video embeddings. Got type: {type(video_embeds)}")
|
||||
return Qwen2_5_VLVideoEmbeddingInputs(
|
||||
type="video_embeds",
|
||||
video_embeds=video_embeds,
|
||||
@@ -648,24 +553,20 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
audio_hashes: list[str] = None,
|
||||
cached_audio_features: torch.Tensor = None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
input_features = audio_input["input_features"]
|
||||
audio_feature_lengths = audio_input["audio_feature_lengths"]
|
||||
if input_features.ndim == 3:
|
||||
assert input_features.shape[0] == 1
|
||||
input_features = input_features.squeeze(0)
|
||||
if audio_feature_lengths.ndim == 2:
|
||||
assert (
|
||||
audio_feature_lengths.shape[0] == 1
|
||||
or audio_feature_lengths.shape[1] == 1
|
||||
)
|
||||
assert audio_feature_lengths.shape[0] == 1 or audio_feature_lengths.shape[1] == 1
|
||||
if audio_feature_lengths.shape[0] == 1:
|
||||
audio_feature_lengths = audio_feature_lengths.squeeze(0)
|
||||
else:
|
||||
audio_feature_lengths = audio_feature_lengths.squeeze(1)
|
||||
|
||||
audio_feat_lengths, audio_output_lengths = (
|
||||
self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths)
|
||||
audio_feat_lengths, audio_output_lengths = self.audio_tower._get_feat_extract_output_lengths(
|
||||
audio_feature_lengths
|
||||
)
|
||||
|
||||
audio_outputs = self.audio_tower(
|
||||
@@ -676,9 +577,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
audio_features = audio_outputs.last_hidden_state
|
||||
return audio_features.split(audio_output_lengths.tolist())
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: Qwen2_5_VLImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
def _process_image_input(self, image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["image_embeds"].type(self.visual.dtype)
|
||||
|
||||
@@ -758,8 +657,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
else:
|
||||
logger.warning(
|
||||
"flash_attn is not available, the model may not yield the "
|
||||
"exactly same result as the transformers implementation "
|
||||
"in the audio tower part."
|
||||
"exactly same result as the transformers implementation in the audio tower part."
|
||||
)
|
||||
|
||||
self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
|
||||
@@ -777,9 +675,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
architectures=["Qwen2ForCausalLM"],
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
@@ -787,34 +683,18 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
for input_key in kwargs:
|
||||
if (
|
||||
input_key in ("pixel_values", "image_embeds")
|
||||
and "image" not in mm_input_by_modality
|
||||
):
|
||||
mm_input_by_modality["image"] = self._parse_and_validate_image_input(
|
||||
**kwargs
|
||||
)
|
||||
if (
|
||||
input_key in ("pixel_values_videos", "video_embeds")
|
||||
and "video" not in mm_input_by_modality
|
||||
):
|
||||
mm_input_by_modality["video"] = self._parse_and_validate_video_input(
|
||||
**kwargs
|
||||
)
|
||||
if (
|
||||
input_key in ("input_audio_features")
|
||||
and "audio" not in mm_input_by_modality
|
||||
):
|
||||
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
|
||||
**kwargs
|
||||
)
|
||||
if input_key in ("pixel_values", "image_embeds") and "image" not in mm_input_by_modality:
|
||||
mm_input_by_modality["image"] = self._parse_and_validate_image_input(**kwargs)
|
||||
if input_key in ("pixel_values_videos", "video_embeds") and "video" not in mm_input_by_modality:
|
||||
mm_input_by_modality["video"] = self._parse_and_validate_video_input(**kwargs)
|
||||
if input_key in ("input_audio_features") and "audio" not in mm_input_by_modality:
|
||||
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(**kwargs)
|
||||
return mm_input_by_modality
|
||||
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.language_model
|
||||
|
||||
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
|
||||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
if not mm_input_by_modality:
|
||||
return []
|
||||
@@ -845,7 +725,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
|
||||
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
|
||||
|
||||
# TODO (ywang96): support overlapping modalitiy embeddings so that
|
||||
# `use_audio_in_video` will work on V1.
|
||||
inputs_embeds = merge_multimodal_embeddings(
|
||||
@@ -897,9 +776,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
placeholder_token_id = self.config.image_token_index
|
||||
if modality == "video":
|
||||
placeholder_token_id = self.config.video_token_index
|
||||
inputs_embeds = merge_multimodal_embeddings(
|
||||
input_ids, inputs_embeds, embeddings, placeholder_token_id
|
||||
)
|
||||
inputs_embeds = merge_multimodal_embeddings(input_ids, inputs_embeds, embeddings, placeholder_token_id)
|
||||
return inputs_embeds
|
||||
|
||||
def forward(
|
||||
@@ -921,10 +798,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
text_inputs_embeds = self.get_input_embeddings(
|
||||
input_ids,
|
||||
(
|
||||
[
|
||||
(torch.zeros_like(embeddings), modality)
|
||||
for embeddings, modality in multimodal_embeddings
|
||||
]
|
||||
[(torch.zeros_like(embeddings), modality) for embeddings, modality in multimodal_embeddings]
|
||||
if multimodal_embeddings is not None
|
||||
else None
|
||||
),
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
############################
|
||||
|
||||
import math
|
||||
from typing import Iterable, List, Optional, Set, Tuple, Union
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -16,24 +17,17 @@ from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
||||
Qwen2_5OmniDiTConfig,
|
||||
Qwen2_5OmniToken2WavConfig,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
|
||||
Qwen2_5OmniPreTrainedModel,
|
||||
)
|
||||
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniPreTrainedModel
|
||||
|
||||
# Bring in HF base classes, configs and utilities used below
|
||||
from transformers.utils.logging import get_logger as _hf_get_logger
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader as _Vllm_AutoWeightsLoader,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader as _Vllm_AutoWeightsLoader
|
||||
from vllm.model_executor.models.utils import WeightsMapper as _Vllm_WeightsMapper
|
||||
from vllm.model_executor.models.utils import (
|
||||
init_vllm_registered_model as _Vllm_init_vllm_registered_model,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix as _Vllm_maybe_prefix
|
||||
from vllm.model_executor.models.utils import init_vllm_registered_model as _vllm_init_vllm_registered_model
|
||||
from vllm.model_executor.models.utils import maybe_prefix as _vllm_maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
@@ -218,9 +212,7 @@ class AttentiveStatisticsPooling(nn.Module):
|
||||
|
||||
def _compute_statistics(self, x, m, dim=2):
|
||||
mean = (m * x).sum(dim)
|
||||
std = torch.sqrt(
|
||||
(m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(self.eps)
|
||||
)
|
||||
std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(self.eps))
|
||||
return mean, std
|
||||
|
||||
def forward(self, hidden_states):
|
||||
@@ -282,9 +274,7 @@ class SqueezeExcitationRes2NetBlock(nn.Module):
|
||||
kernel_size=1,
|
||||
dilation=1,
|
||||
)
|
||||
self.res2net_block = Res2NetBlock(
|
||||
out_channels, out_channels, res2net_scale, kernel_size, dilation
|
||||
)
|
||||
self.res2net_block = Res2NetBlock(out_channels, out_channels, res2net_scale, kernel_size, dilation)
|
||||
self.tdnn2 = TimeDelayNetBlock(
|
||||
out_channels,
|
||||
out_channels,
|
||||
@@ -312,13 +302,10 @@ class ECAPA_TimeDelayNet(torch.nn.Module):
|
||||
|
||||
def __init__(self, config: Qwen2_5OmniDiTConfig):
|
||||
super().__init__()
|
||||
if len(config.enc_channels) != len(config.enc_kernel_sizes) or len(
|
||||
config.enc_channels
|
||||
) != len(config.enc_dilations):
|
||||
raise ValueError(
|
||||
"enc_channels, enc_kernel_sizes and enc_dilations should have "
|
||||
"same length"
|
||||
)
|
||||
if len(config.enc_channels) != len(config.enc_kernel_sizes) or len(config.enc_channels) != len(
|
||||
config.enc_dilations
|
||||
):
|
||||
raise ValueError("enc_channels, enc_kernel_sizes and enc_dilations should have same length")
|
||||
self.channels = config.enc_channels
|
||||
self.blocks = nn.ModuleList()
|
||||
|
||||
@@ -412,26 +399,14 @@ class DiTInputEmbedding(nn.Module):
|
||||
):
|
||||
if apply_cfg:
|
||||
hidden_states = torch.cat([hidden_states, hidden_states], dim=0)
|
||||
speaker_embedding = torch.cat(
|
||||
[speaker_embedding, torch.zeros_like(speaker_embedding)], dim=0
|
||||
)
|
||||
condition_vector = torch.cat(
|
||||
[condition_vector, torch.zeros_like(condition_vector)], dim=0
|
||||
)
|
||||
speaker_embedding = torch.cat([speaker_embedding, torch.zeros_like(speaker_embedding)], dim=0)
|
||||
condition_vector = torch.cat([condition_vector, torch.zeros_like(condition_vector)], dim=0)
|
||||
code_embed = torch.cat([code_embed, code_embed_uncond], dim=0)
|
||||
elif drop_audio_cond: # cfg for cond audio
|
||||
condition_vector = torch.zeros_like(condition_vector)
|
||||
speaker_embedding = torch.zeros_like(speaker_embedding)
|
||||
condition_vector = (
|
||||
self.spk_encoder(condition_vector)
|
||||
.unsqueeze(1)
|
||||
.repeat(1, hidden_states.size(1), 1)
|
||||
)
|
||||
hidden_states = self.proj(
|
||||
torch.cat(
|
||||
(hidden_states, condition_vector, code_embed, speaker_embedding), dim=-1
|
||||
)
|
||||
)
|
||||
condition_vector = self.spk_encoder(condition_vector).unsqueeze(1).repeat(1, hidden_states.size(1), 1)
|
||||
hidden_states = self.proj(torch.cat((hidden_states, condition_vector, code_embed, speaker_embedding), dim=-1))
|
||||
|
||||
return hidden_states
|
||||
|
||||
@@ -465,13 +440,9 @@ class Qwen2_5_OmniAdaLayerNormZero(nn.Module):
|
||||
|
||||
def forward(self, hidden_states, emb=None):
|
||||
emb = self.linear(self.silu(emb))
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = torch.chunk(
|
||||
emb, 6, dim=1
|
||||
)
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = torch.chunk(emb, 6, dim=1)
|
||||
|
||||
hidden_states = (
|
||||
self.norm(hidden_states) * (1 + scale_msa[:, None]) + shift_msa[:, None]
|
||||
)
|
||||
hidden_states = self.norm(hidden_states) * (1 + scale_msa[:, None]) + shift_msa[:, None]
|
||||
return hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp
|
||||
|
||||
|
||||
@@ -490,9 +461,7 @@ class Qwen2_5_OmniAdaLayerNormZero_Final(nn.Module):
|
||||
emb = self.linear(self.silu(emb))
|
||||
scale, shift = torch.chunk(emb, 2, dim=1)
|
||||
|
||||
hidden_states = (
|
||||
self.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :]
|
||||
)
|
||||
hidden_states = self.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :]
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -572,9 +541,7 @@ class DiTAttention(nn.Module):
|
||||
self.to_k = nn.Linear(config.hidden_size, self.inner_dim)
|
||||
self.to_v = nn.Linear(config.hidden_size, self.inner_dim)
|
||||
|
||||
self.to_out = nn.ModuleList(
|
||||
[nn.Linear(self.inner_dim, config.hidden_size), nn.Dropout(config.dropout)]
|
||||
)
|
||||
self.to_out = nn.ModuleList([nn.Linear(self.inner_dim, config.hidden_size), nn.Dropout(config.dropout)])
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -600,9 +567,7 @@ class DiTAttention(nn.Module):
|
||||
# Due to training process, only first head is applied with RoPE,
|
||||
# will be fixed at next release
|
||||
cos, sin = position_embeddings
|
||||
query[:, :1], key[:, :1] = apply_rotary_pos_emb(
|
||||
query[:, :1], key[:, :1], cos, sin
|
||||
)
|
||||
query[:, :1], key[:, :1] = apply_rotary_pos_emb(query[:, :1], key[:, :1], cos, sin)
|
||||
|
||||
attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
|
||||
attention_weights, _ = attention_interface(
|
||||
@@ -616,9 +581,7 @@ class DiTAttention(nn.Module):
|
||||
|
||||
# mask. e.g. inference got a batch with different target durations,
|
||||
# mask out the padding
|
||||
attention_weights = attention_weights.reshape(
|
||||
batch_size, -1, self.heads * head_dim
|
||||
)
|
||||
attention_weights = attention_weights.reshape(batch_size, -1, self.heads * head_dim)
|
||||
attention_weights = attention_weights.to(query.dtype)
|
||||
|
||||
# linear proj
|
||||
@@ -648,9 +611,7 @@ class DiTTimestepEmbedding(nn.Module):
|
||||
def __init__(self, dim, freq_embed_dim=256):
|
||||
super().__init__()
|
||||
self.time_embed = SinusPositionEmbedding(freq_embed_dim)
|
||||
self.time_mlp = nn.ModuleList(
|
||||
[nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim)]
|
||||
)
|
||||
self.time_mlp = nn.ModuleList([nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim)])
|
||||
|
||||
def forward(self, timestep): # noqa: F821
|
||||
time_hidden = self.time_embed(timestep)
|
||||
@@ -661,29 +622,21 @@ class DiTTimestepEmbedding(nn.Module):
|
||||
|
||||
|
||||
class DiTDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self, config: Qwen2_5OmniDiTConfig, look_ahead_block=0, look_backward_block=0
|
||||
):
|
||||
def __init__(self, config: Qwen2_5OmniDiTConfig, look_ahead_block=0, look_backward_block=0):
|
||||
super().__init__()
|
||||
self.attn_norm = Qwen2_5_OmniAdaLayerNormZero(config.hidden_size)
|
||||
|
||||
self.attn = DiTAttention(config)
|
||||
self.look_ahead_block = look_ahead_block
|
||||
self.look_backward_block = look_backward_block
|
||||
self.ff_norm = nn.LayerNorm(
|
||||
config.hidden_size, elementwise_affine=False, eps=1e-6
|
||||
)
|
||||
self.ff = DiTMLP(
|
||||
dim=config.hidden_size, mult=config.ff_mult, dropout=config.dropout
|
||||
)
|
||||
self.ff_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False, eps=1e-6)
|
||||
self.ff = DiTMLP(dim=config.hidden_size, mult=config.ff_mult, dropout=config.dropout)
|
||||
|
||||
def forward(
|
||||
self, hidden_states, timestep, position_embeddings=None, block_diff=None
|
||||
): # x: noised input, t: time embedding
|
||||
# pre-norm & modulation for attention input
|
||||
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(
|
||||
hidden_states, emb=timestep
|
||||
)
|
||||
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(hidden_states, emb=timestep)
|
||||
|
||||
# attention
|
||||
attn_output = self.attn(
|
||||
@@ -696,9 +649,7 @@ class DiTDecoderLayer(nn.Module):
|
||||
# process attention output for input x
|
||||
hidden_states = hidden_states + gate_msa.unsqueeze(1) * attn_output
|
||||
|
||||
norm = (
|
||||
self.ff_norm(hidden_states) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
|
||||
)
|
||||
norm = self.ff_norm(hidden_states) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
|
||||
ff_output = self.ff(norm)
|
||||
hidden_states = hidden_states + gate_mlp.unsqueeze(1) * ff_output
|
||||
|
||||
@@ -741,16 +692,14 @@ class SnakeBeta(nn.Module):
|
||||
beta = self.beta.unsqueeze(0).unsqueeze(-1)
|
||||
alpha = torch.exp(alpha)
|
||||
beta = torch.exp(beta)
|
||||
hidden_states = hidden_states + (
|
||||
1.0 / (beta + self.no_div_by_zero)
|
||||
) * torch.pow(torch.sin(hidden_states * alpha), 2)
|
||||
hidden_states = hidden_states + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(
|
||||
torch.sin(hidden_states * alpha), 2
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
def kaiser_sinc_filter1d(
|
||||
cutoff: float, half_width: float, kernel_size: int
|
||||
) -> torch.Tensor:
|
||||
def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) -> torch.Tensor:
|
||||
"""Generates a 1D Kaiser-windowed sinc filter.
|
||||
|
||||
Args:
|
||||
@@ -775,9 +724,7 @@ def kaiser_sinc_filter1d(
|
||||
else:
|
||||
beta = 0.0
|
||||
|
||||
kaiser_window = torch.kaiser_window(
|
||||
kernel_size, beta=beta, periodic=False, dtype=torch.float32
|
||||
)
|
||||
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
|
||||
|
||||
# Compute time indices
|
||||
if is_even:
|
||||
@@ -787,9 +734,7 @@ def kaiser_sinc_filter1d(
|
||||
|
||||
# Compute sinc filter
|
||||
if cutoff == 0:
|
||||
return torch.zeros(
|
||||
(1, 1, kernel_size), dtype=torch.float32
|
||||
) # Ensures correct shape
|
||||
return torch.zeros((1, 1, kernel_size), dtype=torch.float32) # Ensures correct shape
|
||||
|
||||
sinc_filter = torch.sinc(2 * cutoff * time_indices)
|
||||
normalized_filter = 2 * cutoff * kaiser_window * sinc_filter
|
||||
@@ -804,27 +749,19 @@ class UpSample1d(nn.Module):
|
||||
def __init__(self, ratio=2, kernel_size=None):
|
||||
super().__init__()
|
||||
self.ratio = ratio
|
||||
self.kernel_size = (
|
||||
int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
|
||||
)
|
||||
self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
|
||||
self.stride = ratio
|
||||
self.pad = self.kernel_size // ratio - 1
|
||||
self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
|
||||
self.pad_right = (
|
||||
self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
|
||||
)
|
||||
self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
|
||||
|
||||
filter = kaiser_sinc_filter1d(
|
||||
cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size
|
||||
)
|
||||
filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size)
|
||||
self.register_buffer("filter", filter, persistent=False)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
hidden_states_dtype = hidden_states.dtype
|
||||
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(
|
||||
self.filter.dtype
|
||||
)
|
||||
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(self.filter.dtype)
|
||||
hidden_states = self.ratio * F.conv_transpose1d(
|
||||
hidden_states,
|
||||
self.filter.expand(channels, -1, -1),
|
||||
@@ -857,9 +794,7 @@ class DownSample1d(nn.Module):
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
hidden_states_dtype = hidden_states.dtype
|
||||
hidden_states = F.pad(
|
||||
hidden_states, (self.pad_left, self.pad_right), mode="replicate"
|
||||
).to(self.filter.dtype)
|
||||
hidden_states = F.pad(hidden_states, (self.pad_left, self.pad_right), mode="replicate").to(self.filter.dtype)
|
||||
out = F.conv1d(
|
||||
hidden_states,
|
||||
self.filter.expand(channels, -1, -1),
|
||||
@@ -960,15 +895,10 @@ class AMPBlock(torch.nn.Module):
|
||||
]
|
||||
)
|
||||
|
||||
self.num_layers = len(self.convs1) + len(
|
||||
self.convs2
|
||||
) # total number of conv layers
|
||||
self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers
|
||||
|
||||
self.activations = nn.ModuleList(
|
||||
[
|
||||
TorchActivation1d(activation=SnakeBeta(channels))
|
||||
for _ in range(self.num_layers)
|
||||
]
|
||||
[TorchActivation1d(activation=SnakeBeta(channels)) for _ in range(self.num_layers)]
|
||||
)
|
||||
|
||||
def _get_padding(self, kernel_size, dilation=1):
|
||||
@@ -1001,9 +931,7 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
|
||||
self.num_residual_blocks = len(config.resblock_kernel_sizes)
|
||||
self.num_upsample_layers = len(config.upsample_rates)
|
||||
|
||||
self.conv_pre = nn.Conv1d(
|
||||
config.mel_dim, config.upsample_initial_channel, 7, 1, padding=3
|
||||
)
|
||||
self.conv_pre = nn.Conv1d(config.mel_dim, config.upsample_initial_channel, 7, 1, padding=3)
|
||||
|
||||
# Removing extra ModuleList breaks official state dict
|
||||
ups = [
|
||||
@@ -1018,9 +946,7 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
|
||||
)
|
||||
]
|
||||
)
|
||||
for layer_idx, (stride, kernel_size) in enumerate(
|
||||
zip(config.upsample_rates, config.upsample_kernel_sizes)
|
||||
)
|
||||
for layer_idx, (stride, kernel_size) in enumerate(zip(config.upsample_rates, config.upsample_kernel_sizes))
|
||||
]
|
||||
self.ups = nn.ModuleList(ups)
|
||||
|
||||
@@ -1032,16 +958,12 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
|
||||
dilation,
|
||||
)
|
||||
for layer_idx in range(self.num_upsample_layers)
|
||||
for kernel_size, dilation in zip(
|
||||
config.resblock_kernel_sizes, config.resblock_dilation_sizes
|
||||
)
|
||||
for kernel_size, dilation in zip(config.resblock_kernel_sizes, config.resblock_dilation_sizes)
|
||||
]
|
||||
)
|
||||
|
||||
self.activation_post = TorchActivation1d(
|
||||
activation=SnakeBeta(
|
||||
config.upsample_initial_channel // (2**self.num_upsample_layers)
|
||||
)
|
||||
activation=SnakeBeta(config.upsample_initial_channel // (2**self.num_upsample_layers))
|
||||
)
|
||||
self.conv_post = nn.Conv1d(
|
||||
config.upsample_initial_channel // (2**self.num_upsample_layers),
|
||||
@@ -1081,9 +1003,7 @@ class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel):
|
||||
for layer_index in range(self.num_upsample_layers):
|
||||
hidden_representation = self.ups[layer_index][0](hidden_representation)
|
||||
residual_output = sum(
|
||||
self.resblocks[layer_index * self.num_residual_blocks + block_index](
|
||||
hidden_representation
|
||||
)
|
||||
self.resblocks[layer_index * self.num_residual_blocks + block_index](hidden_representation)
|
||||
for block_index in range(self.num_residual_blocks)
|
||||
)
|
||||
residual_output = residual_output / self.num_residual_blocks
|
||||
@@ -1111,11 +1031,7 @@ class RungeKutta4ODESolver:
|
||||
value_start,
|
||||
function_value_start=None,
|
||||
):
|
||||
k1 = (
|
||||
function_value_start
|
||||
if function_value_start is not None
|
||||
else function(time_start, value_start)
|
||||
)
|
||||
k1 = function_value_start if function_value_start is not None else function(time_start, value_start)
|
||||
k2 = function(
|
||||
time_start + time_step * self._one_third,
|
||||
value_start + time_step * k1 * self._one_third,
|
||||
@@ -1141,9 +1057,7 @@ class RungeKutta4ODESolver:
|
||||
function_value_start,
|
||||
)
|
||||
|
||||
def _linear_interpolation(
|
||||
self, time_start, time_end, value_start, value_end, time_point
|
||||
):
|
||||
def _linear_interpolation(self, time_start, time_end, value_start, value_end, time_point):
|
||||
if time_point == time_start:
|
||||
return value_start
|
||||
if time_point == time_end:
|
||||
@@ -1164,15 +1078,10 @@ class RungeKutta4ODESolver:
|
||||
current_value = self.initial_value
|
||||
for time_start, time_end in zip(time_points[:-1], time_points[1:]):
|
||||
time_step = time_end - time_start
|
||||
delta_value, _ = self._compute_step(
|
||||
self.function, time_start, time_step, time_end, current_value
|
||||
)
|
||||
delta_value, _ = self._compute_step(self.function, time_start, time_step, time_end, current_value)
|
||||
next_value = current_value + delta_value
|
||||
|
||||
while (
|
||||
current_index < len(time_points)
|
||||
and time_end >= time_points[current_index]
|
||||
):
|
||||
while current_index < len(time_points) and time_end >= time_points[current_index]:
|
||||
solution[current_index] = self._linear_interpolation(
|
||||
time_start,
|
||||
time_end,
|
||||
@@ -1203,9 +1112,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
self.repeats = config.repeats
|
||||
self.time_embed = DiTTimestepEmbedding(config.hidden_size)
|
||||
|
||||
self.text_embed = DiTCodecEmbedding(
|
||||
config.num_embeds, config.emb_dim, config.repeats
|
||||
)
|
||||
self.text_embed = DiTCodecEmbedding(config.num_embeds, config.emb_dim, config.repeats)
|
||||
self.input_embed = DiTInputEmbedding(config)
|
||||
|
||||
self.rotary_embed = Qwen2_5OmniDiTRotaryEmbedding(config.head_dim)
|
||||
@@ -1225,16 +1132,12 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
)
|
||||
)
|
||||
|
||||
self.norm_out = Qwen2_5_OmniAdaLayerNormZero_Final(
|
||||
config.hidden_size
|
||||
) # final modulation
|
||||
self.norm_out = Qwen2_5_OmniAdaLayerNormZero_Final(config.hidden_size) # final modulation
|
||||
self.proj_out = nn.Linear(config.hidden_size, config.mel_dim)
|
||||
|
||||
def _create_block_diff(self, hidden_states):
|
||||
batch, seq_len = hidden_states.shape[0], hidden_states.shape[1]
|
||||
block_indices = (
|
||||
torch.arange(seq_len, device=hidden_states.device) // self.block_size
|
||||
) # [seq_length]
|
||||
block_indices = torch.arange(seq_len, device=hidden_states.device) // self.block_size # [seq_length]
|
||||
|
||||
block_i = block_indices.unsqueeze(1) # [seq_length, 1]
|
||||
block_j = block_indices.unsqueeze(0) # [1, seq_length]
|
||||
@@ -1259,12 +1162,8 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
|
||||
# Compute embeddings
|
||||
time_embedding = self.time_embed(time_step)
|
||||
text_embedding = self.text_embed(
|
||||
quantized_code, drop_code=False if apply_cfg else drop_code
|
||||
)
|
||||
text_embedding_unconditioned = (
|
||||
self.text_embed(quantized_code, drop_code=True) if apply_cfg else None
|
||||
)
|
||||
text_embedding = self.text_embed(quantized_code, drop_code=False if apply_cfg else drop_code)
|
||||
text_embedding_unconditioned = self.text_embed(quantized_code, drop_code=True) if apply_cfg else None
|
||||
|
||||
hidden_states = self.input_embed(
|
||||
hidden_states,
|
||||
@@ -1304,17 +1203,11 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
guidance_scale=0.5,
|
||||
sway_coefficient=-1.0,
|
||||
):
|
||||
noise_initialization = torch.randn(
|
||||
[1, 30000, self.mel_dim], dtype=reference_mel_spectrogram.dtype
|
||||
)
|
||||
noise_initialization = torch.randn([1, 30000, self.mel_dim], dtype=reference_mel_spectrogram.dtype)
|
||||
maximum_duration = quantized_code.shape[1] * self.repeats
|
||||
initial_state = noise_initialization[:, :maximum_duration].to(
|
||||
quantized_code.device
|
||||
)
|
||||
initial_state = noise_initialization[:, :maximum_duration].to(quantized_code.device)
|
||||
batch_size = reference_mel_spectrogram.shape[0]
|
||||
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(
|
||||
1, maximum_duration, 1
|
||||
)
|
||||
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(1, maximum_duration, 1)
|
||||
|
||||
if batch_size != 1:
|
||||
raise ValueError("Only batch size = 1 is currently supported")
|
||||
@@ -1341,10 +1234,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
apply_cfg=True,
|
||||
)
|
||||
guided_prediction, null_prediction = torch.chunk(model_output, 2, dim=0)
|
||||
return (
|
||||
guided_prediction
|
||||
+ (guided_prediction - null_prediction) * guidance_scale
|
||||
)
|
||||
return guided_prediction + (guided_prediction - null_prediction) * guidance_scale
|
||||
|
||||
initial_time = 0
|
||||
time_embedding = torch.linspace(
|
||||
@@ -1356,13 +1246,9 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
)
|
||||
|
||||
if sway_coefficient is not None:
|
||||
time_embedding += sway_coefficient * (
|
||||
torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding
|
||||
)
|
||||
time_embedding += sway_coefficient * (torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding)
|
||||
|
||||
ode_solver = RungeKutta4ODESolver(
|
||||
function=ode_function, initial_value=initial_state
|
||||
)
|
||||
ode_solver = RungeKutta4ODESolver(function=ode_function, initial_value=initial_state)
|
||||
solution_trajectory = ode_solver.integrate(time_embedding)
|
||||
|
||||
generated_waveform = solution_trajectory[-1]
|
||||
@@ -1393,9 +1279,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
"""
|
||||
initial_state = y0.to(quantized_code.device)
|
||||
batch_size = reference_mel_spectrogram.shape[0]
|
||||
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(
|
||||
1, initial_state.shape[1], 1
|
||||
)
|
||||
conditioning_vector = conditioning_vector.unsqueeze(1).repeat(1, initial_state.shape[1], 1)
|
||||
|
||||
if batch_size != 1:
|
||||
raise ValueError("Only batch size = 1 is currently supported")
|
||||
@@ -1422,10 +1306,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
apply_cfg=True,
|
||||
)
|
||||
guided_prediction, null_prediction = torch.chunk(model_output, 2, dim=0)
|
||||
return (
|
||||
guided_prediction
|
||||
+ (guided_prediction - null_prediction) * guidance_scale
|
||||
)
|
||||
return guided_prediction + (guided_prediction - null_prediction) * guidance_scale
|
||||
|
||||
initial_time = 0
|
||||
time_embedding = torch.linspace(
|
||||
@@ -1437,13 +1318,9 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
)
|
||||
|
||||
if sway_coefficient is not None:
|
||||
time_embedding += sway_coefficient * (
|
||||
torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding
|
||||
)
|
||||
time_embedding += sway_coefficient * (torch.cos(torch.pi / 2 * time_embedding) - 1 + time_embedding)
|
||||
|
||||
ode_solver = RungeKutta4ODESolver(
|
||||
function=ode_function, initial_value=initial_state
|
||||
)
|
||||
ode_solver = RungeKutta4ODESolver(function=ode_function, initial_value=initial_state)
|
||||
solution_trajectory = ode_solver.integrate(time_embedding)
|
||||
|
||||
generated_waveform = solution_trajectory[-1]
|
||||
@@ -1479,8 +1356,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
attn_impl = "sdpa"
|
||||
elif config._attn_implementation == "eager":
|
||||
logger.warning_once(
|
||||
"Qwen2_5OmniToken2WavModel does not support eager attention "
|
||||
"implementation, fall back to sdpa"
|
||||
"Qwen2_5OmniToken2WavModel does not support eager attention implementation, fall back to sdpa"
|
||||
)
|
||||
attn_impl = "sdpa"
|
||||
self.code2wav_dit_model = Qwen2_5OmniToken2WavDiTModel._from_config(
|
||||
@@ -1503,9 +1379,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
|
||||
# codec embedding size for masking EOS out-of-range
|
||||
try:
|
||||
self.codec_embed_size = (
|
||||
self.code2wav_dit_model.text_embed.codec_embed.weight.size(0)
|
||||
)
|
||||
self.codec_embed_size = self.code2wav_dit_model.text_embed.codec_embed.weight.size(0)
|
||||
except Exception:
|
||||
self.codec_embed_size = -1
|
||||
|
||||
@@ -1587,7 +1461,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
steps: int,
|
||||
prev_generated: torch.Tensor,
|
||||
finished: bool = False,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
"""Streaming per small chunk: returns (mel_or_None, audio_slice)."""
|
||||
start_index = max(i * self.chunk_size - self.past_cache_size, 0)
|
||||
end_index = min(
|
||||
@@ -1595,16 +1469,8 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
codec_all.shape[1] * self.factor,
|
||||
)
|
||||
|
||||
y0 = (
|
||||
y_all[:, start_index:end_index]
|
||||
.reshape(1, -1, self.code2wav_dit_model.mel_dim)
|
||||
.contiguous()
|
||||
)
|
||||
codec = (
|
||||
codec_all[:, start_index // self.factor : end_index // self.factor]
|
||||
.reshape(1, -1)
|
||||
.contiguous()
|
||||
)
|
||||
y0 = y_all[:, start_index:end_index].reshape(1, -1, self.code2wav_dit_model.mel_dim).contiguous()
|
||||
codec = codec_all[:, start_index // self.factor : end_index // self.factor].reshape(1, -1).contiguous()
|
||||
|
||||
# generate mel for current window (B, mel_dim, T)
|
||||
generated = self.process_chunk_dit_batch(
|
||||
@@ -1634,9 +1500,9 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
y_all: torch.Tensor,
|
||||
i: int,
|
||||
steps: int,
|
||||
prev_generated: Union[torch.Tensor, List[torch.Tensor]],
|
||||
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
|
||||
finished: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, List[torch.Tensor]], torch.Tensor]:
|
||||
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
|
||||
"""High-level chunk API aligning to qwen2_code2wav_dit signature."""
|
||||
if not isinstance(prev_generated, torch.Tensor):
|
||||
prev_generated = prev_generated[0] if len(prev_generated) > 0 else None
|
||||
@@ -1661,7 +1527,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
finished: bool,
|
||||
prev_generated: Optional[torch.Tensor],
|
||||
generated: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Align mel and audio boundaries for 50Hz-like streaming.
|
||||
|
||||
@@ -1678,21 +1544,13 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
mel = generated[:, :, : self.chunk_size]
|
||||
elif finished:
|
||||
mel_trim = generated[:, :, self.past_cache_size :]
|
||||
mel = torch.cat(
|
||||
[prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2
|
||||
)
|
||||
mel = torch.cat([prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2)
|
||||
else:
|
||||
if start_index == 0:
|
||||
mel_trim = generated[
|
||||
:, :, i * self.chunk_size : -self.future_cache_size
|
||||
]
|
||||
mel_trim = generated[:, :, i * self.chunk_size : -self.future_cache_size]
|
||||
else:
|
||||
mel_trim = generated[
|
||||
:, :, self.past_cache_size : -self.future_cache_size
|
||||
]
|
||||
mel = torch.cat(
|
||||
[prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2
|
||||
)
|
||||
mel_trim = generated[:, :, self.past_cache_size : -self.future_cache_size]
|
||||
mel = torch.cat([prev_generated[:, :, -self.future_size * 2 :], mel_trim], dim=2)
|
||||
|
||||
audio = self.code2wav_bigvgan_model(mel)
|
||||
if i == 0:
|
||||
@@ -1700,11 +1558,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
elif finished:
|
||||
audio_output = audio[self.future_size * self.vocoder_hop :]
|
||||
else:
|
||||
audio_output = audio[
|
||||
self.future_size
|
||||
* self.vocoder_hop : -self.future_size
|
||||
* self.vocoder_hop
|
||||
]
|
||||
audio_output = audio[self.future_size * self.vocoder_hop : -self.future_size * self.vocoder_hop]
|
||||
return mel, audio_output
|
||||
|
||||
|
||||
@@ -1728,9 +1582,9 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
|
||||
# Initialize underlying HF Token2Wav model via registry
|
||||
self.token2wav = _Vllm_init_vllm_registered_model(
|
||||
self.token2wav = _vllm_init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
prefix=_Vllm_maybe_prefix(prefix, "token2wav_model"),
|
||||
prefix=_vllm_maybe_prefix(prefix, "token2wav_model"),
|
||||
hf_config=self.config,
|
||||
architectures=["Qwen2_5OmniToken2WavDiTModel"],
|
||||
)
|
||||
@@ -1782,9 +1636,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
) -> Optional[SamplerOutput]:
|
||||
return None
|
||||
|
||||
def load_weights_without_buffers(
|
||||
self, weights: Iterable[Tuple[str, torch.Tensor]]
|
||||
) -> Set[str]:
|
||||
def load_weights_without_buffers(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = _Vllm_AutoWeightsLoader(self)
|
||||
loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
# Log load summary
|
||||
@@ -1822,9 +1674,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
return registers
|
||||
|
||||
# remove buffers from the weights and reload them after loading weights
|
||||
def remove_buffers_from_weights(
|
||||
self, weights: Iterable[Tuple[str, torch.Tensor]], buffers: dict
|
||||
):
|
||||
def remove_buffers_from_weights(self, weights: Iterable[tuple[str, torch.Tensor]], buffers: dict):
|
||||
weights_to_load = []
|
||||
for key, value in weights:
|
||||
if key in buffers:
|
||||
@@ -1844,9 +1694,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
loaded_buffers.add(name)
|
||||
return loaded_buffers
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[Tuple[str, torch.Tensor]], spk_dict_path: str
|
||||
) -> Set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]], spk_dict_path: str) -> set[str]:
|
||||
buffers = self.find_all_registers()
|
||||
weights_to_load = self.remove_buffers_from_weights(weights, buffers)
|
||||
loaded = self.load_weights_without_buffers(weights_to_load)
|
||||
@@ -1874,9 +1722,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def process_chunk_bigvgan_batch(
|
||||
self, mel_batch: torch.Tensor
|
||||
) -> Optional[torch.Tensor]:
|
||||
def process_chunk_bigvgan_batch(self, mel_batch: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
# BigVGAN is not part of this wrapper; return None for parity.
|
||||
return None
|
||||
|
||||
@@ -1891,7 +1737,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
steps: int,
|
||||
prev_generated: torch.Tensor,
|
||||
finished: bool = False,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
mel = self.token2wav(
|
||||
code=codec_all,
|
||||
conditioning=conditioning,
|
||||
@@ -1909,9 +1755,9 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
y_all: torch.Tensor,
|
||||
i: int,
|
||||
steps: int,
|
||||
prev_generated: Union[torch.Tensor, List[torch.Tensor]],
|
||||
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
|
||||
finished: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, List[torch.Tensor]], torch.Tensor]:
|
||||
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
|
||||
_mel, out = self.process_little_chunk(
|
||||
conditioning=conditioning,
|
||||
reference_mel=reference_mel,
|
||||
@@ -1919,9 +1765,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
y_all=y_all,
|
||||
i=i,
|
||||
steps=steps,
|
||||
prev_generated=(
|
||||
prev_generated if isinstance(prev_generated, torch.Tensor) else None
|
||||
),
|
||||
prev_generated=(prev_generated if isinstance(prev_generated, torch.Tensor) else None),
|
||||
finished=finished,
|
||||
)
|
||||
return _mel if _mel is not None else prev_generated, out
|
||||
@@ -1,9 +1,9 @@
|
||||
from typing import Iterable, Optional, Set, Tuple, Union
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen2Config
|
||||
|
||||
from vllm.attention import Attention, AttentionType
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -11,23 +11,13 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.pooler import Pooler, PoolingType
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead, VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader, maybe_remap_kv_scale_name
|
||||
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -48,7 +38,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Qwen2MLP(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
@@ -73,10 +62,7 @@ class Qwen2MLP(nn.Module):
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. "
|
||||
"Only silu is supported for now."
|
||||
)
|
||||
raise ValueError(f"Unsupported activation: {hidden_act}. Only silu is supported for now.")
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
@@ -87,7 +73,6 @@ class Qwen2MLP(nn.Module):
|
||||
|
||||
|
||||
class Qwen2Attention(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
@@ -98,7 +83,7 @@ class Qwen2Attention(nn.Module):
|
||||
head_dim: Optional[int] = None,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
rope_scaling: Optional[Tuple] = None,
|
||||
rope_scaling: Optional[tuple] = None,
|
||||
prefix: str = "",
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
) -> None:
|
||||
@@ -173,7 +158,6 @@ class Qwen2Attention(nn.Module):
|
||||
|
||||
|
||||
class Qwen2DecoderLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen2Config,
|
||||
@@ -217,16 +201,14 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
@@ -255,7 +237,6 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
}
|
||||
)
|
||||
class Qwen2Model(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -270,27 +251,20 @@ class Qwen2Model(nn.Module):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
# TODO (@robertgshaw2): see if this can be moved out
|
||||
if cache_config.sliding_window is not None and hasattr(
|
||||
config, "max_window_layers"
|
||||
):
|
||||
if cache_config.sliding_window is not None and hasattr(config, "max_window_layers"):
|
||||
raise ValueError(
|
||||
"Sliding window for some but all layers is not "
|
||||
"supported. This model uses sliding window "
|
||||
"but `max_window_layers` = {} is less than "
|
||||
"`num_hidden_layers` = {}. Please open an issue "
|
||||
"to discuss this feature.".format(
|
||||
config.max_window_layers,
|
||||
config.num_hidden_layers,
|
||||
)
|
||||
f"but `max_window_layers` = {config.max_window_layers} is less than "
|
||||
f"`num_hidden_layers` = {config.num_hidden_layers}. Please open an issue "
|
||||
"to discuss this feature."
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank or (
|
||||
config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||
):
|
||||
if get_pp_group().is_first_rank or (config.tie_word_embeddings and get_pp_group().is_last_rank):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
getattr(config, "embedding_size", config.hidden_size),
|
||||
@@ -348,13 +322,11 @@ class Qwen2Model(nn.Module):
|
||||
residual,
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return IntermediateTensors({"hidden_states": hidden_states, "residual": residual})
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
@@ -364,19 +336,15 @@ class Qwen2Model(nn.Module):
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: Set[str] = set()
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if self.quant_config is not None and (
|
||||
scale_name := self.quant_config.get_cache_scale(name)
|
||||
):
|
||||
if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):
|
||||
# Loading kv cache quantization scales
|
||||
param = params_dict[scale_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loaded_weight = (
|
||||
loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
|
||||
)
|
||||
loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(scale_name)
|
||||
continue
|
||||
@@ -433,9 +401,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self.lora_config = lora_config
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.model = Qwen2Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.model = Qwen2Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
if config.tie_word_embeddings:
|
||||
@@ -453,9 +419,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.sampler = Sampler()
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings(input_ids)
|
||||
@@ -467,9 +431,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
hidden_states = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
@@ -487,7 +449,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
next_tokens = self.sampler(logits, sampling_metadata)
|
||||
return next_tokens
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
@@ -521,9 +483,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self.lora_config = lora_config
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.model = Qwen2Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.model = Qwen2Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
|
||||
|
||||
# TODO: Replace this model class with as_embedding_model(
|
||||
# Qwen2ForCausalLM) after changing the default pooling method
|
||||
@@ -554,9 +514,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
|
||||
) -> Optional[PoolerOutput]:
|
||||
return self._pooler(hidden_states, pooling_metadata)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
weights = self.hf_to_vllm_mapper.apply(weights)
|
||||
weights = (
|
||||
(name, data) for name, data in weights if not name.startswith("lm_head.")
|
||||
)
|
||||
weights = ((name, data) for name, data in weights if not name.startswith("lm_head."))
|
||||
self.model.load_weights(weights)
|
||||
@@ -1,8 +1,4 @@
|
||||
from vllm.model_executor.models.registry import (
|
||||
_VLLM_MODELS,
|
||||
_LazyRegisteredModel,
|
||||
_ModelRegistry,
|
||||
)
|
||||
from vllm.model_executor.models.registry import _VLLM_MODELS, _LazyRegisteredModel, _ModelRegistry
|
||||
|
||||
_OMNI_MODELS = {
|
||||
"Qwen2_5OmniForConditionalGeneration": (
|
||||
|
||||
Reference in new issue
Block a user