Compare commits

..
3 Commits
6 changed files with 116 additions and 13 deletions

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
@@ -215,8 +215,8 @@ def test_mel_head_preserves_official_checkpoint_bias_and_hidden_payload_policy(
assert seen["bias"] is True
assert talker.mel_head.bias is not None
assert talker.omni_pooler_payload_include_hidden is use_gpt_latent
assert talker.omni_payload_at_request_end is (model_type == "indextts2_5")
assert talker.omni_pooler_payload_include_hidden is False
assert talker.omni_payload_at_request_end is True
def test_v2_load_weights_loads_exact_mel_head_bias_parameter(monkeypatch):
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Deepstack buffers must remain usable when the vision tower is skipped."""
import pytest
import torch
from torch import nn
from transformers import Qwen3OmniMoeThinkerConfig
from vllm.config import DeviceConfig, ModelConfig, MultiModalConfig, SchedulerConfig, VllmConfig
from vllm.model_executor.models.utils import StageMissingLayer
import vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker as thinker_module
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _LanguageModel(nn.Module):
def make_empty_intermediate_tensors(self):
raise AssertionError("No pipeline-parallel allocation is needed by this test")
class _DecoderLayer(nn.Module):
def forward(self, positions, hidden_states, residual):
return hidden_states, residual
class _Norm(nn.Module):
def forward(self, hidden_states, residual):
return hidden_states, residual
@pytest.mark.parametrize("vision_limit", [0, 1])
@pytest.mark.parametrize("num_tokens", [4, 8])
def test_deepstack_buffers_use_execution_device(mocker, vision_limit, num_tokens):
"""Exercise the real constructor, tower-skip context and LM addition."""
config = Qwen3OmniMoeThinkerConfig(
text_config={"hidden_size": 8},
vision_config={"out_hidden_size": 8, "deepstack_visual_indexes": [0, 1]},
)
multimodal_config = MultiModalConfig(limit_per_prompt={"image": vision_limit, "video": vision_limit, "audio": 1})
model_config = mocker.Mock(spec=ModelConfig, hf_config=config, multimodal_config=multimodal_config)
model_config.get_multimodal_config.return_value = multimodal_config
vllm_config = mocker.Mock(
spec=VllmConfig,
model_config=model_config,
device_config=DeviceConfig(device="cpu"),
scheduler_config=mocker.Mock(spec=SchedulerConfig, max_num_batched_tokens=4),
quant_config=None,
)
vllm_config.with_hf_config.return_value = vllm_config
# Replace only the large networks. vLLM's real _mark_tower_model context
# still builds the disabled visual module on meta and removes its weights.
mocker.patch.object(thinker_module, "Qwen3OmniMoeAudioEncoder", side_effect=lambda *args, **kwargs: nn.Linear(8, 8))
mocker.patch.object(thinker_module, "Qwen3Omni_VisionTransformer", side_effect=lambda **kwargs: nn.Linear(8, 8))
mocker.patch.object(thinker_module, "Qwen3MoeLLMForCausalLM", return_value=_LanguageModel())
thinker = thinker_module.Qwen3OmniMoeThinkerForConditionalGeneration(vllm_config=vllm_config)
assert isinstance(thinker.visual, StageMissingLayer) is (vision_limit == 0)
# Resizing must also retain the buffer device rather than the ambient one.
with torch.device("meta"):
deepstack = thinker._get_deepstack_input_embeds(num_tokens)
model = object.__new__(thinker_module.Qwen3MoeLLMModel)
nn.Module.__init__(model)
model.layers = nn.ModuleList([_DecoderLayer(), _DecoderLayer()])
model.start_layer = 0
model.end_layer = 2
model.norm = _Norm()
mocker.patch.object(thinker_module, "get_pp_group", return_value=mocker.Mock(is_first_rank=True, is_last_rank=True))
inputs = torch.ones(num_tokens, 8)
# Before the fix, audio-only construction reaches this actual forward
# with meta buffers and raises the reported cross-device addition error.
output, _ = thinker_module.Qwen3MoeLLMModel.forward(
model,
input_ids=None,
positions=torch.arange(num_tokens),
inputs_embeds=inputs,
deepstack_input_embeds=deepstack,
)
torch.testing.assert_close(output, inputs)
assert len(thinker.deepstack_input_embeds) == 2
assert thinker.deepstack_input_embeds_num_tokens == 0
for buffer in thinker.deepstack_input_embeds:
assert buffer.device == torch.device("cpu")
assert not buffer.is_meta
assert buffer.shape == (num_tokens, 8)
assert buffer.dtype == torch.get_default_dtype()
assert torch.count_nonzero(buffer).item() == 0
+6 -1
View File
@@ -603,8 +603,13 @@ def test_eager_decode_without_a_frame_fails_loudly():
state._eager_state._apply_eager_frames(batches, torch.zeros((1, _EAGER_DIM)), _EagerBatch([1]), None)
def test_run_preprocess_records_rows_that_keep_a_sample():
@pytest.mark.parametrize("has_stream_decoder", [True, False])
def test_run_preprocess_records_rows_that_keep_a_sample(has_stream_decoder):
state = _make_eager_state()
if not has_stream_decoder:
# Eager-MTP talkers without an in-Talker codec (Qwen3-Omni) do not
# define the attribute at all.
del state.model.stream_decoder
_fill_buffers(state, "chunk", "final", "decode")
state._eager_ready = {2: "decode"}
state._eager_embeds[2] = 4.0
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""IndexTTS2 Stage 0: GPT-2 AR Talker with vLLM-native PagedAttention.
Predicts mel codes autoregressively and collects hidden_states as latent
@@ -158,17 +158,18 @@ class IndexTTS2TalkerForConditionalGeneration(nn.Module):
self.has_postprocess = True
self.requires_raw_input_tokens = True
self.enable_update_additional_information = True
# Stage 1 consumes the hidden-state stream only in the legacy
# ``use_gpt_latent`` mode. IndexTTS 2.5 transports mel codes plus
# reference conditioning, so asking the generic runner to include
# hidden states would add a synchronous [num_tokens, model_dim] D2H
# copy on every decode step without adding anything to the payload.
self.omni_pooler_payload_include_hidden = self.use_gpt_latent
# S2Mel never reads the generic pooler ``hidden`` key: latent mode
# ships the aligned GPT latent as ``hidden_states.latent`` from
# make_omni_output, and no-latent mode needs no hidden state at all.
# Including it would add a synchronous [num_tokens, model_dim] D2H
# copy plus a CPU-side concat on every decode step, which also
# serializes async scheduling.
self.omni_pooler_payload_include_hidden = False
# S2Mel consumes the complete sequence only at request end. In latent
# mode the runner retains aligned code/latent deltas on GPU; no-latent
# mode reuses the CPU output-token history and retains only conditioning
# metadata here.
self.omni_payload_at_request_end = self.config.model_type == "indextts2_5"
self.omni_payload_at_request_end = True
self.omni_request_end_token_ids = ()
self._speaker_cache = get_speaker_cache()
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {
@@ -1143,10 +1143,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
# register buffer for deepstack
if self.use_deepstack:
# Create the buffers on the target device explicitly. They are
# built inside the `_mark_tower_model` block, which runs under
# vLLM's meta-device `no_init_weights` context when the image and
# video limits are both 0 (audio-only serving), so a bare
# torch.zeros(...) would stay on meta and the first profile_run
# fails with "Tensor on device meta is not on the expected device".
self.deepstack_input_embeds = [
torch.zeros(
vllm_config.scheduler_config.max_num_batched_tokens,
thinker_config.text_config.hidden_size,
device=vllm_config.device_config.device,
)
for _ in range(self.deepstack_num_level)
]
@@ -760,7 +760,8 @@ class OmniModelState(DefaultModelState):
n_tok = int(input_batch.num_scheduled_tokens[i])
if (
self._eager_mtp
and self.model.stream_decoder is not None
# Only the Qwen3-TTS Talker has an in-Talker streaming codec.
and getattr(self.model, "stream_decoder", None) is not None
and self._eager_state.replay_inputs(
buf["req_id"],
req_idx,