[Model] Add IndexTTS2 text-to-speech support (#3838)

Signed-off-by: BeatSeat <wendavid552@gmail.com>
Signed-off-by: David <david@podcast.ai>
Signed-off-by: wendavid552 <wendavid552@gmail.com>
Signed-off-by: WenDavid <davidzhangcf@foxmail.com>

examples/offline_inference/text_to_speech/README.md

@@ -23,6 +23,7 @@ list of supported architectures across all modalities, see
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
+| IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
 
 ## Common Quick Start
@@ -436,6 +437,50 @@ Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [
 
 ---
 
+## IndexTTS-2
+
+2-stage TTS pipeline (GPT AR talker + S2Mel CFM DiT + BigVGAN vocoder) at 22.05 kHz. Every request requires reference audio for zero-shot voice cloning. Supports emotion conditioning via audio, text, or 8-dim vector.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/indextts2/end2end.py \
+    --model IndexTeam/IndexTTS-2 \
+    --text "你好，这是一个语音合成测试。" \
+    --ref-audio /path/to/reference.wav
+```
+
+### Emotion control
+```bash
+# Emotion from reference audio
+python examples/offline_inference/text_to_speech/indextts2/end2end.py \
+    --model IndexTeam/IndexTTS-2 \
+    --text "今天天气真好！" \
+    --ref-audio /path/to/ref.wav \
+    --emo-audio /path/to/happy.wav
+
+# Emotion from 8-dim vector (happy angry sad afraid disgusted melancholy surprised calm)
+python examples/offline_inference/text_to_speech/indextts2/end2end.py \
+    --model IndexTeam/IndexTTS-2 \
+    --text "今天天气真好！" \
+    --ref-audio /path/to/ref.wav \
+    --emo-vector 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
+
+# Emotion from text description
+python examples/offline_inference/text_to_speech/indextts2/end2end.py \
+    --model IndexTeam/IndexTTS-2 \
+    --text "今天天气真好！" \
+    --ref-audio /path/to/ref.wav \
+    --emo-text "happy and excited"
+```
+
+### Notes
+- `--ref-audio` is **required** — IndexTTS-2 does not support text-only synthesis.
+- Stage 0 (AR Talker): GPT-2 generates mel codes from text + reference audio.
+- Stage 1 (S2Mel + BigVGAN): CFM DiT converts mel codes to waveform at 22.05 kHz.
+- Deploy config: `vllm_omni/deploy/indextts2.yaml`. Stage 1 runs with `enforce_eager: true` (DiT has dynamic shapes).
+
+---
+
 ## Voxtral TTS
 
 Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.

examples/offline_inference/text_to_speech/indextts2/end2end.py

@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Offline inference example for IndexTTS2 via vLLM-Omni.
+
+Two-stage pipeline: GPT AR (Stage 0) → S2Mel + BigVGAN (Stage 1).
+Output is 22050 Hz mono WAV.
+
+Usage:
+  python end2end.py \
+    --model /path/to/IndexTeam/IndexTTS-2 \
+    --text "你好，这是一个语音合成测试。" \
+    --ref-audio /path/to/ref.wav
+
+  # With emotion audio:
+  python end2end.py \
+    --model /path/to/IndexTeam/IndexTTS-2 \
+    --text "今天天气真好！" \
+    --ref-audio /path/to/ref.wav \
+    --emo-audio /path/to/happy.wav
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import soundfile as sf
+import torch
+from vllm import SamplingParams
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+from vllm_omni import Omni  # noqa: E402
+from vllm_omni.model_executor.models.indextts2.prompt_utils import (  # noqa: E402
+    build_indextts2_prefill_prompt_ids,
+)
+
+
+def build_request(
+    model: str,
+    text: str,
+    ref_audio_path: str | None = None,
+    emo_audio_path: str | None = None,
+    emo_text: str | None = None,
+    emo_vector: list[float] | None = None,
+    emo_alpha: float | None = None,
+    use_emo_text: bool = False,
+    use_random: bool = False,
+) -> dict:
+    additional: dict = {"text": [text]}
+    if ref_audio_path:
+        additional["voice"] = [str(ref_audio_path)]
+    if emo_audio_path:
+        additional["emo_audio"] = [str(emo_audio_path)]
+    if emo_text:
+        additional["emo_text"] = [emo_text]
+    if emo_vector is not None:
+        additional["emo_vector"] = [emo_vector]
+    if emo_alpha is not None:
+        additional["emo_alpha"] = [emo_alpha]
+    if use_emo_text:
+        additional["use_emo_text"] = [True]
+    if use_random:
+        additional["use_random"] = [True]
+    return {
+        "prompt_token_ids": build_indextts2_prefill_prompt_ids(model, text),
+        "additional_information": additional,
+    }
+
+
+def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = 22050) -> None:
+    audio_np = waveform.float().numpy()
+    sf.write(path, audio_np, sample_rate)
+    print(f"  Saved {path} ({audio_np.shape}, {sample_rate} Hz)")
+
+
+def extract_audio(mm: dict) -> tuple[torch.Tensor | None, int]:
+    audio = mm.get("audio")
+    if audio is None:
+        audio = mm.get("model_outputs")
+    if isinstance(audio, list):
+        chunks = [chunk.reshape(-1) for chunk in audio if isinstance(chunk, torch.Tensor) and chunk.numel() > 0]
+        audio = torch.cat(chunks, dim=0) if chunks else None
+
+    sr_val = mm.get("sr")
+    if isinstance(sr_val, list):
+        sr_val = sr_val[-1] if sr_val else None
+    if hasattr(sr_val, "item"):
+        sample_rate = int(sr_val.item())
+    else:
+        sample_rate = int(sr_val) if sr_val is not None else 22050
+
+    return audio if isinstance(audio, torch.Tensor) else None, sample_rate
+
+
+def main(args) -> None:
+    omni = Omni(
+        model=args.model,
+        deploy_config=args.deploy_config,
+        stage_init_timeout=args.stage_init_timeout,
+    )
+
+    # Stage 0: GPT AR (text → mel codes)
+    gpt_sampling = SamplingParams(
+        temperature=0.8,
+        top_p=0.8,
+        top_k=30,
+        max_tokens=1500,
+        repetition_penalty=10.0,
+        stop_token_ids=[8193],
+        seed=args.seed if args.seed is not None else 42,
+        detokenize=False,
+    )
+    # Stage 1: S2Mel + BigVGAN (non-AR, params mostly ignored)
+    s2mel_sampling = SamplingParams(
+        temperature=0.0,
+        max_tokens=65536,
+        detokenize=True,
+    )
+    sampling_params = [gpt_sampling, s2mel_sampling]
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Synthesizing: {args.text!r}")
+    if args.ref_audio:
+        print(f"  ref_audio: {args.ref_audio}")
+    inputs = build_request(
+        model=args.model,
+        text=args.text,
+        ref_audio_path=args.ref_audio,
+        emo_audio_path=args.emo_audio,
+        emo_text=args.emo_text,
+        emo_vector=args.emo_vector,
+        emo_alpha=args.emo_alpha,
+        use_emo_text=args.use_emo_text,
+        use_random=args.use_random,
+    )
+
+    for i, omni_out in enumerate(omni.generate(inputs, sampling_params_list=sampling_params)):
+        mm = omni_out.multimodal_output
+        if not mm:
+            print(f"  [req {i}] No multimodal output.")
+            continue
+        audio, sr = extract_audio(mm)
+        if audio is None:
+            print(f"  [req {i}] No waveform in multimodal_output.")
+            continue
+        out_path = str(output_dir / f"output_{i}.wav")
+        save_audio(audio.cpu(), out_path, sr)
+
+    print("Done.")
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="IndexTTS2 offline inference")
+    parser.add_argument("--model", required=True, help="HF model path for IndexTTS2.")
+    parser.add_argument("--text", default="你好，这是IndexTTS2语音合成测试。")
+    parser.add_argument("--ref-audio", required=True, help="Reference audio for voice cloning.")
+    parser.add_argument("--emo-audio", default=None, help="Emotion reference audio.")
+    parser.add_argument("--emo-text", default=None, help="Emotion description text.")
+    parser.add_argument(
+        "--emo-vector",
+        type=float,
+        nargs=8,
+        default=None,
+        help="8-dim emotion vector: happy angry sad afraid disgusted melancholic surprised calm.",
+    )
+    parser.add_argument("--emo-alpha", type=float, default=None, help="Emotion weight in [0, 1].")
+    parser.add_argument("--use-emo-text", action="store_true", help="Infer emotion vector from emo-text or text.")
+    parser.add_argument("--use-random", action="store_true", help="Use random emotion prototypes.")
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument(
+        "--output-dir",
+        default=os.path.join(
+            os.environ.get("XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache")),
+            "indextts2_output",
+        ),
+    )
+    parser.add_argument("--deploy-config", default=None)
+    parser.add_argument("--stage-init-timeout", type=int, default=600)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main(parse_args())

examples/online_serving/text_to_speech/README.md

@@ -16,6 +16,7 @@ For the full list of supported architectures across all modalities, see
 |---|---|---|---|---|---|
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
+| IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | compat only, non-chunk | uploaded audio voice only; no presets | — |
 | Ming-omni-tts | `inclusionAI/Ming-omni-tts-0.5B` | ✓ (`ref_audio` / `speaker_embedding`) | ✓ (PCM stream) | IP labels + structured `instructions` | — |
 | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | — (caption-controlled) | — | caption fields (`instructions`) | — |
 | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ |
@@ -34,7 +35,7 @@ Launch the server (defaults shown — adjust `--port`, `--gpu-memory-utilization
 vllm serve <hf-repo-or-local-path> --omni --port 8091
 ```
 
-Send a TTS request via curl:
+Send a TTS request via curl. These generic snippets assume a model with a preset/default voice; voice-cloning-only models such as IndexTTS-2 require `ref_audio` or an uploaded audio `voice` (see model-specific sections below).
 
 ```bash
 curl -X POST http://localhost:8091/v1/audio/speech \
@@ -90,7 +91,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
 ```
 
-Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others).
+Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for IndexTTS-2, and 24 kHz for many others).
 
 For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](../../../docs/serving/speech_api.md).
 
@@ -135,6 +136,40 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
 
 ---
 
+## IndexTTS-2
+
+2-stage TTS (GPT AR + S2Mel CFM DiT + BigVGAN) at 22.05 kHz. Requests use `ref_audio` for voice cloning, or an uploaded audio `voice` from `/v1/audio/voices`. Supports emotion conditioning via `emo_audio`, `emo_text`, or `emo_vector` passed in `extra_params`.
+
+### Launch
+```bash
+vllm serve IndexTeam/IndexTTS-2 --omni --trust-remote-code --port 8092
+# or, to pass the bundled deploy config explicitly:
+bash examples/online_serving/text_to_speech/indextts2/run_server.sh
+```
+
+### Sending requests
+```bash
+# Voice cloning (ref_audio required)
+python examples/online_serving/text_to_speech/indextts2/speech_client.py \
+    --text "你好，世界！" \
+    --ref-audio /path/to/reference.wav
+
+# With emotion audio
+python examples/online_serving/text_to_speech/indextts2/speech_client.py \
+    --text "今天心情很好！" \
+    --ref-audio /path/to/ref.wav \
+    --emo-audio /path/to/happy.wav
+```
+
+### Notes
+- Output: 22.05 kHz mono WAV.
+- Provide `ref_audio` on the documented raw request path, or pass `voice` only when it names an uploaded audio voice; IndexTTS-2 does not provide a built-in text-only preset voice.
+- Emotion params (`emo_audio`, `emo_text`, `emo_vector`, `emo_alpha`, `use_emo_text`, `use_random`) are passed via the `extra_params` field. Official precedence is `use_emo_text` > `emo_vector` > `emo_audio` > same emotion as the speaker reference.
+- `stream=true` is accepted as an OpenAI-compatible response path, but IndexTTS-2 is not async-chunk streaming; audio is produced after S2Mel receives the full mel-code sequence.
+- Deploy config: `vllm_omni/deploy/indextts2.yaml` (auto-loaded).
+
+---
+
 ## Fish Speech S2 Pro
 
 4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.

examples/online_serving/text_to_speech/indextts2/run_server.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Launch vLLM-Omni server for IndexTTS2
+#
+# Usage from repository root:
+#   examples/online_serving/text_to_speech/indextts2/run_server.sh
+#   CUDA_VISIBLE_DEVICES=0 PORT=8092 MODEL=/path/to/IndexTeam/IndexTTS-2 examples/online_serving/text_to_speech/indextts2/run_server.sh
+
+set -e
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd -- "$SCRIPT_DIR/../../../.." && pwd)"
+
+MODEL="${MODEL:-IndexTeam/IndexTTS-2}"
+PORT="${PORT:-8092}"
+DEPLOY_CONFIG="${DEPLOY_CONFIG:-$ROOT_DIR/vllm_omni/deploy/indextts2.yaml}"
+
+echo "Starting IndexTTS2 server with model: $MODEL"
+
+FLASHINFER_DISABLE_VERSION_CHECK=1 \
+vllm serve "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --omni \
+    --trust-remote-code \
+    --deploy-config "$DEPLOY_CONFIG"

examples/online_serving/text_to_speech/indextts2/speech_client.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""OpenAI-compatible client for IndexTTS2 TTS via /v1/audio/speech endpoint.
+
+Examples:
+    # With reference audio for voice cloning
+    python speech_client.py --text "你好，世界！" \
+        --ref-audio /path/to/reference.wav
+
+    # With emotion audio
+    python speech_client.py --text "今天心情很好！" \
+        --ref-audio /path/to/ref.wav \
+        --emo-audio /path/to/happy.wav
+
+Server setup:
+    vllm serve IndexTeam/IndexTTS-2 --omni --host 0.0.0.0 --port 8092
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import os
+
+import httpx
+
+DEFAULT_API_BASE = "http://localhost:8092"
+DEFAULT_API_KEY = "sk-empty"
+
+
+def encode_audio_to_base64(audio_path: str) -> str:
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+    ext = audio_path.lower().rsplit(".", 1)[-1]
+    mime = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac"}.get(ext, "audio/wav")
+    with open(audio_path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode("utf-8")
+    return f"data:{mime};base64,{b64}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="IndexTTS2 OpenAI speech client")
+    parser.add_argument("--text", type=str, required=True)
+    parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio for voice cloning")
+    parser.add_argument("--emo-audio", type=str, default=None, help="Emotion reference audio")
+    parser.add_argument("--emo-text", type=str, default=None, help="Emotion description text")
+    parser.add_argument(
+        "--emo-vector",
+        type=float,
+        nargs=8,
+        default=None,
+        help="8-dim emotion vector: happy angry sad afraid disgusted melancholic surprised calm",
+    )
+    parser.add_argument("--emo-alpha", type=float, default=None, help="Emotion weight in [0, 1]")
+    parser.add_argument("--use-emo-text", action="store_true", help="Infer emotion vector from emo-text or text")
+    parser.add_argument("--use-random", action="store_true", help="Use random emotion prototypes")
+    parser.add_argument("--model", type=str, default="IndexTeam/IndexTTS-2")
+    parser.add_argument("--voice", type=str, default=None, help="Uploaded voice name to use instead of --ref-audio")
+    parser.add_argument("--output", type=str, default="output.wav")
+    parser.add_argument("--api-base", type=str, default=DEFAULT_API_BASE)
+    parser.add_argument("--api-key", type=str, default=DEFAULT_API_KEY)
+    parser.add_argument("--response-format", type=str, default="wav")
+    args = parser.parse_args()
+
+    if not args.ref_audio and not args.voice:
+        parser.error("IndexTTS2 requires --ref-audio or --voice for voice cloning")
+
+    payload: dict = {
+        "model": args.model,
+        "input": args.text,
+        "response_format": args.response_format,
+    }
+    if args.voice:
+        payload["voice"] = args.voice
+
+    if args.ref_audio:
+        ref = args.ref_audio
+        if ref.startswith(("http://", "https://", "data:")):
+            payload["ref_audio"] = ref
+        else:
+            payload["ref_audio"] = encode_audio_to_base64(ref)
+
+    extra_params = {}
+    if args.emo_audio:
+        emo = args.emo_audio
+        if emo.startswith(("http://", "https://", "data:")):
+            extra_params["emo_audio"] = emo
+        else:
+            extra_params["emo_audio"] = encode_audio_to_base64(emo)
+    if args.emo_text:
+        extra_params["emo_text"] = args.emo_text
+    if args.emo_vector is not None:
+        extra_params["emo_vector"] = args.emo_vector
+    if args.emo_alpha is not None:
+        extra_params["emo_alpha"] = args.emo_alpha
+    if args.use_emo_text:
+        extra_params["use_emo_text"] = True
+    if args.use_random:
+        extra_params["use_random"] = True
+    if extra_params:
+        payload["extra_params"] = extra_params
+
+    url = f"{args.api_base}/v1/audio/speech"
+    print(f"POST {url}")
+    print(f"  text: {args.text}")
+    if args.ref_audio:
+        print(f"  ref_audio: {args.ref_audio[:80]}...")
+
+    with httpx.Client(timeout=300) as client:
+        resp = client.post(
+            url,
+            json=payload,
+            headers={"Authorization": f"Bearer {args.api_key}"},
+        )
+
+    if resp.status_code != 200:
+        print(f"Error {resp.status_code}: {resp.text[:500]}")
+        return
+
+    with open(args.output, "wb") as f:
+        f.write(resp.content)
+    print(f"Saved: {args.output} ({len(resp.content):,} bytes)")
+
+
+if __name__ == "__main__":
+    main()