[Model] Add IndexTTS2 text-to-speech support (#3838)

Signed-off-by: BeatSeat <wendavid552@gmail.com>
Signed-off-by: David <david@podcast.ai>
Signed-off-by: wendavid552 <wendavid552@gmail.com>
Signed-off-by: WenDavid <davidzhangcf@foxmail.com>

tests/core/sched/test_omni_scheduling_coordinator.py

@@ -115,6 +115,7 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase):
             ("MiMoAudioModel", "code2wav"),
             ("Qwen3TTSCode2Wav", "code2wav"),
             ("CosyVoice3Model", "cosyvoice3_code2wav"),
+            ("IndexTTS2S2MelDecoder", "indextts2_s2mel_decoder"),
             ("DyninOmniForConditionalGeneration", "token2image"),
             ("DyninOmniForConditionalGeneration", "token2audio"),
         }

tests/e2e/offline_inference/test_indextts2.py

@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""E2E offline inference tests for IndexTTS2 two-stage pipeline.
+
+Stage 0 (GPT AR) → Stage 1 (S2Mel + BigVGAN). Output is 22050 Hz mono WAV.
+Covers: basic TTS.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import torch
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniRunner
+from tests.helpers.stage_config import get_deploy_config_path
+from vllm_omni import Omni
+from vllm_omni.model_executor.models.indextts2.prompt_utils import (
+    build_indextts2_prefill_prompt_ids,
+)
+
+MODEL_NAME = "IndexTeam/IndexTTS-2"
+STAGE_CONFIG = get_deploy_config_path("indextts2.yaml")
+
+_OMNI_RUNNER_PARAM = (
+    MODEL_NAME,
+    STAGE_CONFIG,
+)
+
+ASSETS_DIR = Path(__file__).resolve().parents[2] / "assets" / "indextts2"
+REF_AUDIO = str(ASSETS_DIR / "ref_audio.wav")
+
+pytestmark = [
+    pytest.mark.advanced_model,
+    pytest.mark.tts,
+    pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
+]
+
+SAMPLE_RATE = 22050
+
+
+def _audio_from_mm(mm: dict) -> torch.Tensor | None:
+    audio = mm.get("audio")
+    if audio is None:
+        audio = mm.get("model_outputs")
+    if isinstance(audio, list):
+        chunks = [chunk.reshape(-1) for chunk in audio if isinstance(chunk, torch.Tensor) and chunk.numel() > 0]
+        return torch.cat(chunks, dim=0) if chunks else None
+    return audio if isinstance(audio, torch.Tensor) else None
+
+
+def _sample_rate_from_mm(mm: dict) -> int:
+    sr = mm.get("sr")
+    if isinstance(sr, list):
+        sr = sr[-1] if sr else None
+    if hasattr(sr, "item"):
+        return int(sr.item())
+    return int(sr) if sr is not None else SAMPLE_RATE
+
+
+def _build_request(text: str, ref_audio: str, **extra) -> dict:
+    info = {
+        "text": [text],
+        "voice": [ref_audio],
+    }
+    info.update(extra)
+    return {
+        "prompt_token_ids": build_indextts2_prefill_prompt_ids(MODEL_NAME, text),
+        "additional_information": info,
+    }
+
+
+def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
+    for omni_out in omni.generate(request):
+        mm = omni_out.multimodal_output
+        assert mm is not None, "Expected multimodal_output"
+        audio = _audio_from_mm(mm)
+        assert audio is not None, "Expected audio output"
+        return audio.cpu(), _sample_rate_from_mm(mm)
+    raise AssertionError("No outputs received")
+
+
+def _assert_valid_audio(audio: torch.Tensor, sr: int) -> None:
+    assert sr == SAMPLE_RATE
+    assert audio.numel() > 0
+    assert torch.any(audio != 0).item()
+
+
+@hardware_test(res={"cuda": "L4"})
+def test_basic_english(omni_runner: OmniRunner) -> None:
+    """English TTS: text + ref_audio, no emotion conditioning."""
+    req = _build_request("Hello, this is a voice synthesis test.", REF_AUDIO)
+    audio, sr = _collect_audio(omni_runner.omni, req)
+    _assert_valid_audio(audio, sr)

tests/e2e/online_serving/test_indextts2.py

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""E2E online serving tests for IndexTTS2 via /v1/audio/speech endpoint.
+
+Two-stage pipeline: GPT AR → S2Mel + BigVGAN. Output is 22050 Hz mono WAV.
+Covers: basic TTS and the speech endpoint's stream=True compatibility path.
+"""
+
+from __future__ import annotations
+
+import os
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+import pytest
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.media import load_test_audio_data_url
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path
+
+pytestmark = [pytest.mark.advanced_model, pytest.mark.tts]
+
+MODEL = "IndexTeam/IndexTTS-2"
+
+REF_AUDIO_URL = load_test_audio_data_url("indextts2/ref_audio.wav")
+
+tts_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL,
+            stage_config_path=get_deploy_config_path("indextts2.yaml"),
+            server_args=["--disable-log-stats"],
+        ),
+        id="indextts2",
+    )
+]
+
+
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_basic_english(omni_server, openai_client) -> None:
+    """
+    Basic English TTS: text + ref_audio, no emotion.
+    Deploy Setting: default yaml
+    Input Modal: text + reference audio
+    Output Modal: audio (22050 Hz, WAV)
+    Input Setting: stream=False
+    """
+    request_config = {
+        "model": omni_server.model,
+        "input": "Hello, this is a voice cloning demo.",
+        "stream": False,
+        "response_format": "wav",
+        "ref_audio": REF_AUDIO_URL,
+        "min_audio_bytes": 1024,
+    }
+    openai_client.send_audio_speech_request(request_config)
+
+
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_streaming(omni_server, openai_client) -> None:
+    """
+    stream=True compatibility: IndexTTS2 itself is non-async-chunking, but the
+    OpenAI speech endpoint should still accept the streaming request path and
+    return a valid PCM response.
+
+    Deploy Setting: default yaml
+    Input Modal: text + reference audio
+    Output Modal: audio (22050 Hz, PCM)
+    Input Setting: stream=True
+    """
+    request_config = {
+        "model": omni_server.model,
+        "input": "Hello, this is a voice cloning demo.",
+        "stream": True,
+        "response_format": "pcm",
+        "ref_audio": REF_AUDIO_URL,
+        "min_audio_bytes": 1024,
+    }
+    openai_client.send_audio_speech_request(request_config)

tests/model_executor/stage_input_processors/test_indextts2.py

@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+import torch
+
+from vllm_omni.model_executor.stage_input_processors.indextts2 import (
+    _strip_stop_token,
+    talker2s2mel,
+    talker2s2mel_full_payload,
+    talker2s2mel_token_only,
+)
+
+STOP_MEL_TOKEN = 8193
+LATENT_DIM = 16
+
+
+def _make_talker_output(
+    *,
+    mel_codes: torch.Tensor,
+    latent: torch.Tensor,
+    s_ref: torch.Tensor | None = None,
+    ref_mel: torch.Tensor | None = None,
+    style: torch.Tensor | None = None,
+    finished: bool = True,
+):
+    meta = {}
+    if s_ref is not None:
+        meta["S_ref"] = s_ref
+    if ref_mel is not None:
+        meta["ref_mel"] = ref_mel
+    if style is not None:
+        meta["style"] = style
+
+    mm = {
+        "codes": {"mel": mel_codes},
+        "hidden_states": {"latent": latent},
+        "meta": meta,
+    }
+    return SimpleNamespace(
+        finished=finished,
+        outputs=[SimpleNamespace(multimodal_output=mm)],
+    )
+
+
+# ── _strip_stop_token ──
+
+
+def test_strip_stop_token_basic():
+    codes = torch.tensor([10, 20, 30, STOP_MEL_TOKEN, 99])
+    latent = torch.randn(5, LATENT_DIM)
+
+    c, lat, lens = _strip_stop_token(codes, latent)
+
+    assert c.shape == (1, 3)
+    assert lat.shape == (1, 3, LATENT_DIM)
+    assert lens.tolist() == [3]
+    assert (c[0] == torch.tensor([10, 20, 30])).all()
+
+
+def test_strip_stop_token_no_stop():
+    codes = torch.tensor([10, 20, 30])
+    latent = torch.randn(3, LATENT_DIM)
+
+    c, lat, lens = _strip_stop_token(codes, latent)
+
+    assert c.shape == (1, 3)
+    assert lens.tolist() == [3]
+
+
+def test_strip_stop_token_at_start():
+    codes = torch.tensor([STOP_MEL_TOKEN, 10, 20])
+    latent = torch.randn(3, LATENT_DIM)
+
+    c, lat, lens = _strip_stop_token(codes, latent)
+
+    assert c.shape == (1, 0)
+    assert lat.shape == (1, 0, LATENT_DIM)
+    assert lens.tolist() == [0]
+
+
+def test_strip_stop_token_batch_with_padding():
+    codes = torch.tensor(
+        [
+            [10, 20, STOP_MEL_TOKEN, 0],
+            [30, 40, 50, STOP_MEL_TOKEN],
+        ]
+    )
+    latent = torch.randn(2, 4, LATENT_DIM)
+
+    c, lat, lens = _strip_stop_token(codes, latent)
+
+    assert c.shape == (2, 3)
+    assert lat.shape == (2, 3, LATENT_DIM)
+    assert lens.tolist() == [2, 3]
+    assert c[0, 2].item() == 0  # padded with a valid semantic code
+
+
+def test_strip_stop_token_all_stop():
+    codes = torch.tensor([STOP_MEL_TOKEN])
+    latent = torch.randn(1, LATENT_DIM)
+
+    c, lat, lens = _strip_stop_token(codes, latent)
+
+    assert lens.tolist() == [0]
+    assert c.shape[1] == 0
+    assert lat.shape[1] == 0
+
+
+# ── talker2s2mel ──
+
+
+def test_talker2s2mel_basic():
+    mel_codes = torch.tensor([10, 20, 30, STOP_MEL_TOKEN])
+    latent = torch.randn(4, LATENT_DIM)
+    s_ref = torch.randn(1, 128)
+    ref_mel = torch.randn(1, 80, 200)
+    style = torch.randn(1, 256)
+
+    outputs = talker2s2mel(
+        source_outputs=[
+            _make_talker_output(
+                mel_codes=mel_codes,
+                latent=latent,
+                s_ref=s_ref,
+                ref_mel=ref_mel,
+                style=style,
+            )
+        ]
+    )
+
+    assert len(outputs) == 1
+    info = outputs[0]["additional_information"]
+    assert info["mel_codes"].shape == (1, 3)
+    assert info["latent"].shape == (1, 3, LATENT_DIM)
+    assert info["code_lens"].tolist() == [3]
+    assert "S_ref" in info
+    assert "ref_mel" in info
+    assert "style" in info
+    assert outputs[0]["prompt_token_ids"] == [0]
+
+
+def test_talker2s2mel_skips_unfinished():
+    mel_codes = torch.tensor([10, 20])
+    latent = torch.randn(2, LATENT_DIM)
+
+    outputs = talker2s2mel(source_outputs=[_make_talker_output(mel_codes=mel_codes, latent=latent, finished=False)])
+
+    assert len(outputs) == 0
+
+
+def test_talker2s2mel_skips_empty_mel():
+    outputs = talker2s2mel(
+        source_outputs=[
+            _make_talker_output(
+                mel_codes=torch.tensor([]),
+                latent=torch.randn(0, LATENT_DIM),
+            )
+        ]
+    )
+
+    assert len(outputs) == 0
+
+
+def test_talker2s2mel_missing_optional_meta():
+    mel_codes = torch.tensor([10, 20])
+    latent = torch.randn(2, LATENT_DIM)
+
+    outputs = talker2s2mel(source_outputs=[_make_talker_output(mel_codes=mel_codes, latent=latent)])
+
+    assert len(outputs) == 1
+    info = outputs[0]["additional_information"]
+    assert "S_ref" not in info
+    assert "ref_mel" not in info
+    assert "style" not in info
+
+
+def test_talker2s2mel_batch():
+    out0 = _make_talker_output(
+        mel_codes=torch.tensor([1, 2, STOP_MEL_TOKEN]),
+        latent=torch.randn(3, LATENT_DIM),
+        s_ref=torch.randn(1, 64),
+    )
+    out1 = _make_talker_output(
+        mel_codes=torch.tensor([5, 6, 7, 8]),
+        latent=torch.randn(4, LATENT_DIM),
+        ref_mel=torch.randn(1, 80, 100),
+    )
+
+    outputs = talker2s2mel(source_outputs=[out0, out1])
+
+    assert len(outputs) == 2
+    assert outputs[0]["additional_information"]["code_lens"].tolist() == [2]
+    assert outputs[1]["additional_information"]["code_lens"].tolist() == [4]
+
+
+def test_talker2s2mel_tensors_on_cpu():
+    device = torch.device("cpu")
+    mel_codes = torch.tensor([10, 20], device=device)
+    latent = torch.randn(2, LATENT_DIM, device=device)
+    s_ref = torch.randn(1, 64, device=device)
+
+    outputs = talker2s2mel(source_outputs=[_make_talker_output(mel_codes=mel_codes, latent=latent, s_ref=s_ref)])
+
+    info = outputs[0]["additional_information"]
+    assert info["latent"].device.type == "cpu"
+    assert info["mel_codes"].device.type == "cpu"
+    assert info["S_ref"].device.type == "cpu"
+
+
+def test_talker2s2mel_no_multimodal_dict():
+    output = SimpleNamespace(
+        finished=True,
+        outputs=[SimpleNamespace(multimodal_output="not_a_dict")],
+    )
+
+    results = talker2s2mel(source_outputs=[output])
+
+    assert len(results) == 0
+
+
+# ── full-payload connector path ──
+
+
+def _conditioning_tensors():
+    return {
+        "S_ref": torch.randn(1, 3, LATENT_DIM),
+        "ref_mel": torch.randn(1, 80, 5),
+        "style": torch.randn(1, 4),
+    }
+
+
+def test_talker2s2mel_token_only_placeholder_finished_only():
+    finished = _make_talker_output(mel_codes=torch.tensor([1]), latent=torch.randn(1, LATENT_DIM), finished=True)
+    unfinished = _make_talker_output(mel_codes=torch.tensor([2]), latent=torch.randn(1, LATENT_DIM), finished=False)
+
+    outputs = talker2s2mel_token_only([unfinished, finished])
+
+    assert len(outputs) == 1
+    assert outputs[0]["prompt_token_ids"] == [0]
+    assert outputs[0]["additional_information"] is None
+    assert outputs[0]["multi_modal_data"] is None
+
+
+def test_talker2s2mel_full_payload_flat_keys_builds_s2mel_contract():
+    cond = _conditioning_tensors()
+    latent = torch.randn(3, LATENT_DIM)
+    payload = {
+        "codes.mel": torch.tensor([[10], [20], [30]]),
+        "hidden_states.latent": latent,
+        "meta.S_ref": cond["S_ref"],
+        "meta.ref_mel": cond["ref_mel"],
+        "meta.style": cond["style"],
+    }
+
+    result = talker2s2mel_full_payload(None, payload, SimpleNamespace(request_id="r-flat"))
+
+    assert result is not None
+    assert result["mel_codes"].shape == (1, 3)
+    assert result["mel_codes"].tolist() == [[10, 20, 30]]
+    assert result["latent"].shape == (1, 3, LATENT_DIM)
+    assert result["code_lens"].tolist() == [3]
+    assert result["S_ref"].device.type == "cpu"
+    assert result["ref_mel"].device.type == "cpu"
+    assert result["style"].device.type == "cpu"
+    assert result["S_ref"].data_ptr() != cond["S_ref"].data_ptr()
+
+
+def test_talker2s2mel_full_payload_nested_fallback_input():
+    cond = _conditioning_tensors()
+    payload = {
+        "codes": {"mel": torch.tensor([[4], [5]])},
+        "hidden_states": {"latent": torch.randn(2, LATENT_DIM)},
+        "meta": cond,
+    }
+
+    result = talker2s2mel_full_payload(None, payload, SimpleNamespace(request_id="r-nested"))
+
+    assert result is not None
+    assert result["mel_codes"].tolist() == [[4, 5]]
+    assert result["latent"].shape == (1, 2, LATENT_DIM)
+
+
+def test_talker2s2mel_full_payload_normalizes_one_by_t_mel_row():
+    cond = _conditioning_tensors()
+    payload = {
+        "codes.mel": torch.tensor([[7, 8, 9]]),
+        "hidden_states.latent": torch.randn(3, LATENT_DIM),
+        "meta": cond,
+    }
+
+    result = talker2s2mel_full_payload(None, payload, SimpleNamespace(request_id="r-row"))
+
+    assert result is not None
+    assert result["mel_codes"].tolist() == [[7, 8, 9]]
+
+
+def test_talker2s2mel_full_payload_trims_stop_token_and_matching_latent():
+    cond = _conditioning_tensors()
+    latent = torch.arange(4 * LATENT_DIM, dtype=torch.float32).reshape(4, LATENT_DIM)
+    payload = {
+        "codes.mel": torch.tensor([[1], [2], [STOP_MEL_TOKEN], [99]]),
+        "hidden_states.latent": latent,
+        "meta": cond,
+    }
+
+    result = talker2s2mel_full_payload(None, payload, SimpleNamespace(request_id="r-stop"))
+
+    assert result is not None
+    assert result["mel_codes"].tolist() == [[1, 2]]
+    assert result["latent"].shape == (1, 2, LATENT_DIM)
+    assert result["latent"][0, 0].tolist() == latent[0].tolist()
+    assert result["latent"][0, 1].tolist() == latent[1].tolist()
+    assert result["code_lens"].tolist() == [2]
+
+
+def test_talker2s2mel_full_payload_crops_mel_latent_to_common_length():
+    cond = _conditioning_tensors()
+    payload = {
+        "codes.mel": torch.tensor([[1], [2], [3], [4]]),
+        "hidden_states.latent": torch.randn(2, LATENT_DIM),
+        "meta": cond,
+    }
+
+    result = talker2s2mel_full_payload(None, payload, SimpleNamespace(request_id="r-crop"))
+
+    assert result is not None
+    assert result["mel_codes"].tolist() == [[1, 2]]
+    assert result["latent"].shape == (1, 2, LATENT_DIM)
+    assert result["code_lens"].tolist() == [2]
+
+
+def test_talker2s2mel_full_payload_missing_required_fields_returns_none():
+    assert talker2s2mel_full_payload(None, {"hidden_states.latent": torch.randn(1, LATENT_DIM)}, None) is None
+    assert talker2s2mel_full_payload(None, {"codes.mel": torch.tensor([[1]])}, None) is None