Dev/rebase v0.15.0 (#1159)
Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk> Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
1 parent
9494d691bf
commit
70ef548448
44 files changed
+648
-554
No files matched your search
@@ -22,6 +22,9 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniThinkerProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_llm_pos_ids_for_vision,
|
||||
)
|
||||
|
||||
# from vllm.model_executor.models.qwen2_code2wav_dit import Qwen2Code2wav
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -35,7 +38,6 @@ from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
|
||||
from vllm_omni.model_executor.models.vision import get_llm_pos_ids_for_vision
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
TALKER_CODEC_EOS_TOKEN_ID = 8294
|
||||
|
||||
@@ -286,7 +286,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
multimodal_config=multimodal_config,
|
||||
)
|
||||
else:
|
||||
self.visual = None
|
||||
|
||||
@@ -82,7 +82,7 @@ from vllm.model_executor.models.vision import (
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement,
|
||||
@@ -609,7 +609,8 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
|
||||
feature_lens=audio_feature_lengths,
|
||||
aftercnn_lens=audio_output_lengths,
|
||||
)
|
||||
audio_features = audio_outputs.last_hidden_state
|
||||
# OMNI: audio_tower.forward() returns hidden_states tensor directly
|
||||
audio_features = audio_outputs
|
||||
return audio_features.split(audio_output_lengths.tolist())
|
||||
|
||||
|
||||
@@ -674,7 +675,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
multimodal_config=multimodal_config,
|
||||
)
|
||||
self.quant_config = quant_config
|
||||
|
||||
|
||||
@@ -504,12 +504,19 @@ class Qwen3TTSConfig(PretrainedConfig):
|
||||
self.tts_bos_token_id = tts_bos_token_id
|
||||
self.tts_eos_token_id = tts_eos_token_id
|
||||
|
||||
# TODO: remove these dummy values after
|
||||
self.image_token_id = 0 # dummy image token id
|
||||
self.video_token_id = 0 # dummy video token id
|
||||
self.vision_start_token_id = 0 # dummy vision start token id
|
||||
self.vision_config = PretrainedConfig() # dummy vision config
|
||||
self.vision_config.spatial_merge_size = 1
|
||||
|
||||
def get_text_config(self, **kwargs):
|
||||
# vLLM expects text config to expose hidden_size/num_attention_heads.
|
||||
# For Qwen3 TTS, the talker config is the text model config.
|
||||
config = self.talker_config
|
||||
if hasattr(config, "rope_parameters"):
|
||||
delattr(config, "rope_parameters")
|
||||
# if hasattr(config, "rope_parameters"):
|
||||
# delattr(config, "rope_parameters")
|
||||
return config
|
||||
|
||||
|
||||
|
||||
@@ -430,6 +430,22 @@ def mel_spectrogram(
|
||||
return mel_spec
|
||||
|
||||
|
||||
def _compute_default_rope_parameters(
|
||||
config,
|
||||
device,
|
||||
):
|
||||
base = config.rope_theta
|
||||
partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
|
||||
head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
|
||||
dim = int(head_dim * partial_rotary_factor)
|
||||
|
||||
attention_factor = 1.0 # Unused in this type of RoPE
|
||||
|
||||
# Compute the inverse frequencies
|
||||
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim))
|
||||
return inv_freq, attention_factor
|
||||
|
||||
|
||||
class Qwen3TTSPreTrainedModel(PreTrainedModel):
|
||||
config_class = Qwen3TTSConfig
|
||||
base_model_prefix = "model"
|
||||
@@ -502,7 +518,9 @@ class Qwen3TTSTalkerRotaryEmbedding(nn.Module):
|
||||
self.original_max_seq_len = config.max_position_embeddings
|
||||
|
||||
self.config = config
|
||||
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
self.rope_init_fn: Callable = _compute_default_rope_parameters
|
||||
if self.rope_type != "default":
|
||||
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
|
||||
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
@@ -538,7 +556,9 @@ class Qwen3TTSRotaryEmbedding(nn.Module):
|
||||
self.original_max_seq_len = config.max_position_embeddings
|
||||
|
||||
self.config = config
|
||||
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
self.rope_init_fn: Callable = _compute_default_rope_parameters
|
||||
if self.rope_type != "default":
|
||||
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
|
||||
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
@@ -1389,7 +1409,6 @@ class Qwen3TTSTalkerModel(Qwen3TTSTalkerTextPreTrainedModel):
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.padding_idx = config.pad_token_id
|
||||
self.vocab_size = config.vocab_size
|
||||
self.layers = nn.ModuleList(
|
||||
[Qwen3TTSTalkerDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
import torch
|
||||
|
||||
|
||||
def get_llm_pos_ids_for_vision(
|
||||
start_idx: int,
|
||||
vision_idx: int,
|
||||
spatial_merge_size: int,
|
||||
t_index: list[int],
|
||||
grid_hs: torch.Tensor,
|
||||
grid_ws: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
llm_pos_ids_list = []
|
||||
llm_grid_h = grid_hs[vision_idx] // spatial_merge_size
|
||||
llm_grid_w = grid_ws[vision_idx] // spatial_merge_size
|
||||
h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(len(t_index), -1, llm_grid_w).flatten()
|
||||
w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(len(t_index), llm_grid_h, -1).flatten()
|
||||
t_index_tensor = (
|
||||
torch.Tensor(t_index).to(llm_grid_h.device).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).long().flatten()
|
||||
)
|
||||
_llm_pos_ids = torch.stack([t_index_tensor, h_index, w_index])
|
||||
llm_pos_ids_list.append(_llm_pos_ids + start_idx)
|
||||
llm_pos_ids = torch.cat(llm_pos_ids_list, dim=1)
|
||||
return llm_pos_ids
|
||||
Reference in new issue
Block a user