Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
023716ecc4 | ||
|
|
5351cd6645 | ||
|
|
091b256674 |
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -215,8 +215,8 @@ def test_mel_head_preserves_official_checkpoint_bias_and_hidden_payload_policy(
|
||||
|
||||
assert seen["bias"] is True
|
||||
assert talker.mel_head.bias is not None
|
||||
assert talker.omni_pooler_payload_include_hidden is use_gpt_latent
|
||||
assert talker.omni_payload_at_request_end is (model_type == "indextts2_5")
|
||||
assert talker.omni_pooler_payload_include_hidden is False
|
||||
assert talker.omni_payload_at_request_end is True
|
||||
|
||||
|
||||
def test_v2_load_weights_loads_exact_mel_head_bias_parameter(monkeypatch):
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Deepstack buffers must remain usable when the vision tower is skipped."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3OmniMoeThinkerConfig
|
||||
from vllm.config import DeviceConfig, ModelConfig, MultiModalConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.model_executor.models.utils import StageMissingLayer
|
||||
|
||||
import vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker as thinker_module
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _LanguageModel(nn.Module):
|
||||
def make_empty_intermediate_tensors(self):
|
||||
raise AssertionError("No pipeline-parallel allocation is needed by this test")
|
||||
|
||||
|
||||
class _DecoderLayer(nn.Module):
|
||||
def forward(self, positions, hidden_states, residual):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class _Norm(nn.Module):
|
||||
def forward(self, hidden_states, residual):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@pytest.mark.parametrize("vision_limit", [0, 1])
|
||||
@pytest.mark.parametrize("num_tokens", [4, 8])
|
||||
def test_deepstack_buffers_use_execution_device(mocker, vision_limit, num_tokens):
|
||||
"""Exercise the real constructor, tower-skip context and LM addition."""
|
||||
config = Qwen3OmniMoeThinkerConfig(
|
||||
text_config={"hidden_size": 8},
|
||||
vision_config={"out_hidden_size": 8, "deepstack_visual_indexes": [0, 1]},
|
||||
)
|
||||
multimodal_config = MultiModalConfig(limit_per_prompt={"image": vision_limit, "video": vision_limit, "audio": 1})
|
||||
model_config = mocker.Mock(spec=ModelConfig, hf_config=config, multimodal_config=multimodal_config)
|
||||
model_config.get_multimodal_config.return_value = multimodal_config
|
||||
vllm_config = mocker.Mock(
|
||||
spec=VllmConfig,
|
||||
model_config=model_config,
|
||||
device_config=DeviceConfig(device="cpu"),
|
||||
scheduler_config=mocker.Mock(spec=SchedulerConfig, max_num_batched_tokens=4),
|
||||
quant_config=None,
|
||||
)
|
||||
vllm_config.with_hf_config.return_value = vllm_config
|
||||
|
||||
# Replace only the large networks. vLLM's real _mark_tower_model context
|
||||
# still builds the disabled visual module on meta and removes its weights.
|
||||
mocker.patch.object(thinker_module, "Qwen3OmniMoeAudioEncoder", side_effect=lambda *args, **kwargs: nn.Linear(8, 8))
|
||||
mocker.patch.object(thinker_module, "Qwen3Omni_VisionTransformer", side_effect=lambda **kwargs: nn.Linear(8, 8))
|
||||
mocker.patch.object(thinker_module, "Qwen3MoeLLMForCausalLM", return_value=_LanguageModel())
|
||||
thinker = thinker_module.Qwen3OmniMoeThinkerForConditionalGeneration(vllm_config=vllm_config)
|
||||
assert isinstance(thinker.visual, StageMissingLayer) is (vision_limit == 0)
|
||||
|
||||
# Resizing must also retain the buffer device rather than the ambient one.
|
||||
with torch.device("meta"):
|
||||
deepstack = thinker._get_deepstack_input_embeds(num_tokens)
|
||||
|
||||
model = object.__new__(thinker_module.Qwen3MoeLLMModel)
|
||||
nn.Module.__init__(model)
|
||||
model.layers = nn.ModuleList([_DecoderLayer(), _DecoderLayer()])
|
||||
model.start_layer = 0
|
||||
model.end_layer = 2
|
||||
model.norm = _Norm()
|
||||
mocker.patch.object(thinker_module, "get_pp_group", return_value=mocker.Mock(is_first_rank=True, is_last_rank=True))
|
||||
inputs = torch.ones(num_tokens, 8)
|
||||
# Before the fix, audio-only construction reaches this actual forward
|
||||
# with meta buffers and raises the reported cross-device addition error.
|
||||
output, _ = thinker_module.Qwen3MoeLLMModel.forward(
|
||||
model,
|
||||
input_ids=None,
|
||||
positions=torch.arange(num_tokens),
|
||||
inputs_embeds=inputs,
|
||||
deepstack_input_embeds=deepstack,
|
||||
)
|
||||
torch.testing.assert_close(output, inputs)
|
||||
assert len(thinker.deepstack_input_embeds) == 2
|
||||
assert thinker.deepstack_input_embeds_num_tokens == 0
|
||||
for buffer in thinker.deepstack_input_embeds:
|
||||
assert buffer.device == torch.device("cpu")
|
||||
assert not buffer.is_meta
|
||||
assert buffer.shape == (num_tokens, 8)
|
||||
assert buffer.dtype == torch.get_default_dtype()
|
||||
assert torch.count_nonzero(buffer).item() == 0
|
||||
@@ -603,8 +603,13 @@ def test_eager_decode_without_a_frame_fails_loudly():
|
||||
state._eager_state._apply_eager_frames(batches, torch.zeros((1, _EAGER_DIM)), _EagerBatch([1]), None)
|
||||
|
||||
|
||||
def test_run_preprocess_records_rows_that_keep_a_sample():
|
||||
@pytest.mark.parametrize("has_stream_decoder", [True, False])
|
||||
def test_run_preprocess_records_rows_that_keep_a_sample(has_stream_decoder):
|
||||
state = _make_eager_state()
|
||||
if not has_stream_decoder:
|
||||
# Eager-MTP talkers without an in-Talker codec (Qwen3-Omni) do not
|
||||
# define the attribute at all.
|
||||
del state.model.stream_decoder
|
||||
_fill_buffers(state, "chunk", "final", "decode")
|
||||
state._eager_ready = {2: "decode"}
|
||||
state._eager_embeds[2] = 4.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""IndexTTS2 Stage 0: GPT-2 AR Talker with vLLM-native PagedAttention.
|
||||
|
||||
Predicts mel codes autoregressively and collects hidden_states as latent
|
||||
@@ -158,17 +158,18 @@ class IndexTTS2TalkerForConditionalGeneration(nn.Module):
|
||||
self.has_postprocess = True
|
||||
self.requires_raw_input_tokens = True
|
||||
self.enable_update_additional_information = True
|
||||
# Stage 1 consumes the hidden-state stream only in the legacy
|
||||
# ``use_gpt_latent`` mode. IndexTTS 2.5 transports mel codes plus
|
||||
# reference conditioning, so asking the generic runner to include
|
||||
# hidden states would add a synchronous [num_tokens, model_dim] D2H
|
||||
# copy on every decode step without adding anything to the payload.
|
||||
self.omni_pooler_payload_include_hidden = self.use_gpt_latent
|
||||
# S2Mel never reads the generic pooler ``hidden`` key: latent mode
|
||||
# ships the aligned GPT latent as ``hidden_states.latent`` from
|
||||
# make_omni_output, and no-latent mode needs no hidden state at all.
|
||||
# Including it would add a synchronous [num_tokens, model_dim] D2H
|
||||
# copy plus a CPU-side concat on every decode step, which also
|
||||
# serializes async scheduling.
|
||||
self.omni_pooler_payload_include_hidden = False
|
||||
# S2Mel consumes the complete sequence only at request end. In latent
|
||||
# mode the runner retains aligned code/latent deltas on GPU; no-latent
|
||||
# mode reuses the CPU output-token history and retains only conditioning
|
||||
# metadata here.
|
||||
self.omni_payload_at_request_end = self.config.model_type == "indextts2_5"
|
||||
self.omni_payload_at_request_end = True
|
||||
self.omni_request_end_token_ids = ()
|
||||
self._speaker_cache = get_speaker_cache()
|
||||
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {
|
||||
|
||||
@@ -1143,10 +1143,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
|
||||
# register buffer for deepstack
|
||||
if self.use_deepstack:
|
||||
# Create the buffers on the target device explicitly. They are
|
||||
# built inside the `_mark_tower_model` block, which runs under
|
||||
# vLLM's meta-device `no_init_weights` context when the image and
|
||||
# video limits are both 0 (audio-only serving), so a bare
|
||||
# torch.zeros(...) would stay on meta and the first profile_run
|
||||
# fails with "Tensor on device meta is not on the expected device".
|
||||
self.deepstack_input_embeds = [
|
||||
torch.zeros(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
thinker_config.text_config.hidden_size,
|
||||
device=vllm_config.device_config.device,
|
||||
)
|
||||
for _ in range(self.deepstack_num_level)
|
||||
]
|
||||
|
||||
@@ -760,7 +760,8 @@ class OmniModelState(DefaultModelState):
|
||||
n_tok = int(input_batch.num_scheduled_tokens[i])
|
||||
if (
|
||||
self._eager_mtp
|
||||
and self.model.stream_decoder is not None
|
||||
# Only the Qwen3-TTS Talker has an in-Talker streaming codec.
|
||||
and getattr(self.model, "stream_decoder", None) is not None
|
||||
and self._eager_state.replay_inputs(
|
||||
buf["req_id"],
|
||||
req_idx,
|
||||
|
||||
Reference in new issue
Block a user