diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 7809fcbc7..70ab59356 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -191,13 +191,14 @@ steps: # from downgrading transformers; see # examples/offline_inference/text_to_speech/README.md. - label: "TTS · Gepard-1.0 · L4" - timeout_in_minutes: 60 + timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt - >- pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py + tests/e2e/online_serving/test_gepard_tts.py -m "tts and cuda" --run-level "full_model" mirror_hardwares: l4_1 diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index d4fad4206..b4b2e7f55 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -6,6 +6,7 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz. - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz. - **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz. +- **Gepard-1.0** (`nineninesix/gepard-1.0`) -- single-stage native-AR TTS with a 22.05 kHz NanoCodec. Zero-shot default voice only. See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, MOSS-TTS-Nano, and Breeze-TTS-2. @@ -40,11 +41,16 @@ vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091 # CosyVoice3 (voice cloning only — supply ref_audio + ref_text per request) vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 \ --omni --port 8091 --trust-remote-code + +# Gepard-1.0 (zero-shot default voice; packaged gepard.yaml) +vllm-omni serve nineninesix/gepard-1.0 --omni --port 8091 --trust-remote-code \ + --stage-init-timeout 900 \ + --deploy-config vllm_omni/deploy/gepard.yaml ``` ### Generate Speech -**Using curl:** +**Qwen3-TTS CustomVoice, using curl:** ```bash curl -X POST http://localhost:8091/v1/audio/speech \ @@ -56,7 +62,24 @@ curl -X POST http://localhost:8091/v1/audio/speech \ }' --output output.wav ``` -**Using Python:** +**Gepard-1.0, using curl:** + +Gepard accepts `input` (required), `voice` (`"default"`), `response_format`, +`stream` / `stream_format`, `max_new_tokens`, and `seed`. Unsupported fields +include `speed`, `language`, `instructions`, `task_type`, `ref_audio`, +`ref_text`, `extra_params`, and `word_timestamps`. + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is Gepard speaking.", + "voice": "default", + "seed": 7 + }' --output output.wav +``` + +**Qwen3-TTS CustomVoice, using Python:** ```python import httpx @@ -761,6 +784,28 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne | ------- | ------------- | | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | Voice cloning from `ref_audio` + `ref_text`. No built-in voice presets — upload a voice or pass `ref_audio`/`ref_text` per request. | +### Gepard-1.0 + +| Model | Description | +| ----- | ----------- | +| `nineninesix/gepard-1.0` | Zero-shot native-AR TTS. 22.05 kHz mono. `voice` must be omitted or `"default"`. | + +Gepard request fields: + +| Field | Behavior | +| ----- | -------- | +| `input` | Required. Empty/whitespace-only returns 400. | +| `voice` | Omitted or `"default"` only. Other values 400. | +| `response_format` | `wav` default. Non-streaming: `wav`/`pcm`/`flac`/`mp3`. `opus` returns 400 (22.05 kHz is not an Opus sample rate). Streaming: `pcm`/`wav` only. | +| `stream` / `stream_format` | SSE (`speech.audio.*`) and raw `audio` byte streaming. | +| `speed` | Must be `1.0`. Gepard has no native speed control. | +| `max_new_tokens` | Frame budget (1 token = 1 frame = 1024 samples ≈ 46.4 ms). Default 1000 from deploy YAML; adapter bounds 1..4096. | +| `seed` | Optional. Reaches the in-model 32-head sampler. The packaged YAML currently pins `seed: 42`, so serving is deterministic by default until that pin is removed. | +| `extra_params` | Any key returns 400, including `temperature`/`top_p`/`top_k`. | +| Cloning / style fields | `ref_audio`, `ref_text`, `speaker_embedding`, `instructions`, `language`, `task_type`, `word_timestamps`, and similar declared-but-unsupported fields return 400. | + +See the [Gepard section of the online TTS hub](../user_guide/examples/online_serving/text_to_speech.md#gepard-10) for launch commands. Native-AR recompute preemption is a known limitation under concurrency. + ### OmniVoice | Model | Description | diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index 2b4f617f4..e9499551f 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -18,6 +18,7 @@ For the full list of supported architectures across all modalities, see | --- | --- | --- | --- | --- | --- | | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — | | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ | +| Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — | | higgs-audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✓ (`ref_audio`+`ref_text`) | ✓ (codec_streaming) | — | — | | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ | | OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — | @@ -92,7 +93,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` -Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others). +Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for Gepard, 24 kHz for the others). For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md). @@ -212,6 +213,61 @@ python fish_speech/gradio_demo.py --api-base http://localhost:8091 # if server --- +## Gepard-1.0 + +Single-stage native AR TTS at 22.05 kHz mono. Zero-shot only: omit `voice` or pass `"default"`. Voice cloning from reference audio is not available yet. + +### Prerequisites + +Same NeMo NanoCodec install as the [offline Gepard section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#gepard-10). On a host whose CUDA toolkit cannot build kernels — no `nvcc`/`ninja`, or a consumer Blackwell (`sm_120`) card — also `export VLLM_USE_FLASHINFER_SAMPLER=0` before launch. That env is not a deploy-YAML field. + +### Launch + +```bash +vllm-omni serve nineninesix/gepard-1.0 --omni --port 8091 --trust-remote-code \ + --stage-init-timeout 900 \ + --deploy-config vllm_omni/deploy/gepard.yaml +# or: +./gepard/run_server.sh +``` + +`--stage-init-timeout 900` matches the online e2e fixture and the offline example. Serve defaults to 300s, which is often too short for a cold download of the talker plus NanoCodec. + +The packaged `vllm_omni/deploy/gepard.yaml` must be passed with `--deploy-config`. The checkpoint self-identifies as `qwen3_5_text`, so omitting the YAML launches a diffusion fallback instead of the Gepard pipeline. The YAML sets `async_chunk: false`, `max_num_seqs: 4`, and currently pins `seed: 42`, so serving is deterministic by default until that YAML seed is removed. Pass an explicit per-request `seed` in tests and clients rather than depending on either default. + +### Sending requests + +```bash +python examples/online_serving/text_to_speech/gepard/speech_client.py \ + --text "Hello, this is Gepard speaking." + +python examples/online_serving/text_to_speech/gepard/speech_client.py \ + --text "Hello, this is Gepard speaking." --seed 7 --stream --output output.pcm +``` + +Gepard accepts `input` (required), `voice` (`"default"`), `response_format`, `stream` / `stream_format`, `max_new_tokens`, and `seed`. Unsupported fields include `speed`, `language`, `instructions`, `task_type`, `ref_audio`, `ref_text`, `extra_params`, and `word_timestamps`. + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is Gepard speaking.", + "voice": "default", + "seed": 7 + }' --output output.wav +``` + +### Notes + +- Output: 22.05 kHz mono. `max_new_tokens` is a **frame** budget (1 token = 1 frame = 1024 samples ≈ 46.4 ms at 21.5 fps; adapter bounds 1..4096). +- Supported request fields: `input` (required), `voice` (`default` only), `response_format` (`wav` default; `wav/pcm/flac/mp3` non-streaming; `opus` 400 because 22.05 kHz is not an Opus sample rate; streaming `pcm/wav` only), `stream` / `stream_format`, `max_new_tokens`, `seed`. +- Unsupported: `speed`, `extra_params` (including `temperature`/`top_p`/`top_k`), `ref_audio`, `ref_text`, `speaker_embedding`, `task_type`, `instructions`, `language`, and `word_timestamps`. +- Concurrent requests at `max_num_seqs: 4` are supported. Native-AR recompute preemption is a known limitation of this architecture (a request that is preempted mid-generation can resume incorrectly); keep concurrency at or below `max_num_seqs` and treat preemption as out of scope until the platform fix lands. +- Optional comparison against the upstream Gepard reference server needs Blackwell/Hopper + CUDA 13 + Postgres and is not part of CI. +- For the request-field table, see the [Speech API Gepard section](https://github.com/vllm-project/vllm-omni/blob/main/docs/serving/speech_api.md#gepard-10). + +--- + ## higgs-audio v2 2-stage TTS at 24 kHz: a vLLM-native Llama-3.2-3B talker with a DualFFN audio expert (Stage 0) feeding a HiggsAudio codec decoder (Stage 1) that streams chunks back to the client. diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index 954b96bd3..04ff1a97d 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -19,7 +19,7 @@ list of supported architectures across all modalities, see | Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | 2 (talker + codec) | ✓ | ✓ (async chunk) | voice design / voice direction (`--instruction`) | 24 kHz | | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz | | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz | -| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz | +| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning later) | ✓ (online) | zero-shot | 22.05 kHz | | GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz | | Ming-omni-tts | `inclusionAI/Ming-omni-tts-0.5B` | 2 (AR + audio VAE) | ✓ | ✓ | style / IP / dialect / TTA / podcast | 44.1 kHz | | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz | @@ -167,7 +167,7 @@ python examples/offline_inference/text_to_speech/glm_tts/end2end.py \ ``` ### Architecture -``` +```text Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz) (Llama-based) (32k vocab) (Flow Matching) ``` @@ -491,8 +491,8 @@ Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Ref ### Notes - Output: 22.05 kHz mono WAV. - First run downloads two checkpoints, not one: the model itself, and the NanoCodec decoder it names in `codec_id` (`nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps` by default). Both are fetched automatically; expect the first startup to be correspondingly slower. -- Offline only for now; the `/v1/audio/speech` adapter is a follow-up PR. -- Deploy config: `vllm_omni/deploy/gepard.yaml` (the example's default; copy it and pass `--deploy-config` to change it). +- Offline and online: `examples/offline_inference/text_to_speech/gepard/end2end.py` and `/v1/audio/speech` (see the [online Gepard section](../../online_serving/text_to_speech/README.md#gepard-10)). +- Deploy config: `vllm_omni/deploy/gepard.yaml` (the example's default; copy it and pass `--deploy-config` to change it). Online serving **must** pass `--deploy-config vllm_omni/deploy/gepard.yaml` because the checkpoint self-identifies as `qwen3_5_text`. - Generation length and reproducibility are stage settings, not script flags. One output token is one audio frame, so `max_tokens` in the YAML is the frame budget; `seed` makes the in-model 32-head sampling reproducible. The script deliberately passes no `SamplingParams`: one supplied by a caller replaces the stage defaults wholesale rather than merging, which would drop the pipeline's stop token and run every request to `max_tokens`. - Text length: short texts are repeated internally to match the training layout (the checkpoint's `text_repetition` block); the upper bound is the stage's `max_model_len`, enforced by the engine. Empty text is rejected rather than voiced. - `VLLM_GEPARD_GREEDY=1` swaps the 32-head Gumbel-max sampling for argmax, for reproducible comparisons that do not depend on a seed. diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index 7ad939415..b749731e0 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -17,6 +17,7 @@ For the full list of supported architectures across all modalities, see | Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | speaker tags (`S0`..`S9`, default `S0`) | — | | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | uploaded audio voice only; no presets | ✓ | | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ | +| Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — | | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ | | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — | | IndexTTS-2.5 | native `checkpoints/` bundle | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — | @@ -96,12 +97,52 @@ curl -X POST http://localhost:8091/v1/audio/speech \ }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` -Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for IndexTTS-2, and 24 kHz for many others). +Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for Gepard and IndexTTS-2, and 24 kHz for many others). For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](../../../docs/serving/speech_api.md). --- +## Gepard-1.0 + +Single-stage native AR TTS at 22.05 kHz mono. Zero-shot only: omit `voice` or pass `"default"`. Voice cloning from reference audio is not available yet. + +### Prerequisites + +Same NeMo NanoCodec install as the [offline Gepard section](../../offline_inference/text_to_speech/README.md#gepard-10). On a host whose CUDA toolkit cannot build kernels — no `nvcc`/`ninja`, or a consumer Blackwell (`sm_120`) card — also `export VLLM_USE_FLASHINFER_SAMPLER=0` before launch (same note as offline). That env is not a deploy-YAML field. + +### Launch + +```bash +vllm-omni serve nineninesix/gepard-1.0 --omni --port 8091 --trust-remote-code \ + --stage-init-timeout 900 \ + --deploy-config vllm_omni/deploy/gepard.yaml +# or: +./gepard/run_server.sh +``` + +`--stage-init-timeout 900` matches the online e2e fixture and the offline example. Serve defaults to 300s, which is often too short for a cold download of the talker plus NanoCodec. + +The packaged `vllm_omni/deploy/gepard.yaml` must be passed with `--deploy-config`. The checkpoint self-identifies as `qwen3_5_text`, so omitting the YAML launches a diffusion fallback instead of the Gepard pipeline. The YAML sets `async_chunk: false`, `max_num_seqs: 4`, and currently pins `seed: 42`, so serving is deterministic by default until that YAML seed is removed. Pass an explicit per-request `seed` in tests and clients rather than depending on either default. + +### Sending requests + +```bash +python examples/online_serving/text_to_speech/gepard/speech_client.py \ + --text "Hello, this is Gepard speaking." + +python examples/online_serving/text_to_speech/gepard/speech_client.py \ + --text "Hello, this is Gepard speaking." --seed 7 --stream --output output.pcm +``` + +### Notes + +- Output: 22.05 kHz mono. `max_new_tokens` is a **frame** budget (1 token = 1 frame = 1024 samples ≈ 46.4 ms at 21.5 fps; adapter bounds 1..4096). +- Supported request fields: `input` (required), `voice` (`default` only), `response_format` (`wav` default; `wav/pcm/flac/mp3` non-streaming; `opus` 400 because 22.05 kHz is not an Opus sample rate; streaming `pcm/wav` only), `stream` / `stream_format`, `max_new_tokens`, `seed`. +- Unsupported: `speed`, `extra_params` (including `temperature`/`top_p`/`top_k`), `ref_audio`, `ref_text`, `speaker_embedding`, `task_type`, `instructions`, `language`, and `word_timestamps`. +- Concurrent requests at `max_num_seqs: 4` are supported. Native-AR recompute preemption is a known limitation of this architecture (a request that is preempted mid-generation can resume incorrectly); keep concurrency at or below `max_num_seqs` and treat preemption as out of scope until the platform fix lands. +- Optional comparison against the upstream Gepard reference server needs Blackwell/Hopper + CUDA 13 + Postgres and is not part of CI. + ## dots.tts Single-stage, text-only TTS at 48 kHz. The current integration supports diff --git a/examples/online_serving/text_to_speech/gepard/run_server.sh b/examples/online_serving/text_to_speech/gepard/run_server.sh new file mode 100755 index 000000000..420591288 --- /dev/null +++ b/examples/online_serving/text_to_speech/gepard/run_server.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +# Launch vLLM-Omni server for Gepard-1.0 TTS (zero-shot default voice). +# +# Usage: +# ./run_server.sh +# CUDA_VISIBLE_DEVICES=0 ./run_server.sh +# +# Packaged deploy config: vllm_omni/deploy/gepard.yaml (async_chunk=false, +# 22.05 kHz mono, default seed 42 until the YAML seed is removed). +# Cold start downloads the talker and NeMo NanoCodec; the serve default of +# --stage-init-timeout 300 is too tight (offline end2end.py uses 900). +# +# On a host whose CUDA toolkit cannot JIT FlashInfer (no nvcc/ninja, or a +# consumer Blackwell sm_120 card), also: +# export VLLM_USE_FLASHINFER_SAMPLER=0 + +set -e + +MODEL="${MODEL:-nineninesix/gepard-1.0}" +PORT="${PORT:-8091}" + +echo "Starting Gepard-1.0 server with model: $MODEL" + +vllm-omni serve "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --trust-remote-code \ + --omni \ + --stage-init-timeout 900 \ + --deploy-config vllm_omni/deploy/gepard.yaml diff --git a/examples/online_serving/text_to_speech/gepard/speech_client.py b/examples/online_serving/text_to_speech/gepard/speech_client.py new file mode 100755 index 000000000..4db4f1c7d --- /dev/null +++ b/examples/online_serving/text_to_speech/gepard/speech_client.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Client for Gepard-1.0 TTS via /v1/audio/speech. + +Gepard is zero-shot: omit ``voice`` or pass ``"default"``. Output is 22.05 kHz +mono. ``seed`` is optional; the packaged deploy YAML currently pins ``seed: 42`` +until that default is removed, so requests without an explicit seed are still +deterministic. + +Examples: + python speech_client.py --text "Hello, this is Gepard speaking." + python speech_client.py --text "Hello, this is Gepard speaking." --seed 7 + python speech_client.py --text "Hello, this is Gepard speaking." --stream --output output.pcm +""" + +from __future__ import annotations + +import argparse + +import httpx + +DEFAULT_API_BASE = "http://localhost:8091" +DEFAULT_API_KEY = "EMPTY" +DEFAULT_MODEL = "nineninesix/gepard-1.0" + + +def run_tts(args) -> None: + payload = { + "model": args.model, + "input": args.text, + "voice": args.voice, + "response_format": args.response_format, + } + if args.seed is not None: + payload["seed"] = args.seed + if args.max_new_tokens is not None: + payload["max_new_tokens"] = args.max_new_tokens + if args.stream: + payload["stream"] = True + payload["stream_format"] = "audio" + if args.response_format not in ("pcm", "wav"): + print(f"Note: streaming requires pcm/wav; overriding --response-format {args.response_format} to 'pcm'") + payload["response_format"] = "pcm" + + print(f"Model: {args.model}") + print(f"Text: {args.text}") + print(f"Voice: {args.voice}") + if args.seed is not None: + print(f"Seed: {args.seed}") + print("Generating audio...") + + api_url = f"{args.api_base}/v1/audio/speech" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {args.api_key}", + } + + if args.stream: + output_path = args.output or ("output.wav" if payload["response_format"] == "wav" else "output.pcm") + with httpx.Client(timeout=300.0) as client: + with client.stream("POST", api_url, json=payload, headers=headers) as resp: + if resp.status_code != 200: + print(f"Error: {resp.status_code}") + print(resp.read().decode()) + return + total_bytes = 0 + with open(output_path, "wb") as f: + for chunk in resp.iter_bytes(): + f.write(chunk) + total_bytes += len(chunk) + print(f"Streamed {total_bytes} bytes to: {output_path}") + return + + with httpx.Client(timeout=300.0) as client: + response = client.post(api_url, json=payload, headers=headers) + + if response.status_code != 200: + print(f"Error: {response.status_code}") + print(response.text) + return + + output_path = args.output or "output.wav" + with open(output_path, "wb") as f: + f.write(response.content) + print(f"Audio saved to: {output_path}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Gepard-1.0 TTS client") + parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL") + parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key") + parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name") + parser.add_argument("--text", required=True, help="Text to synthesize") + parser.add_argument("--voice", default="default", help="Voice name (only 'default' is supported)") + parser.add_argument("--seed", type=int, default=None, help="Sampling seed") + parser.add_argument( + "--max-new-tokens", + type=int, + default=None, + help="Frame budget (1 token = 1 frame ≈ 46.4 ms at 21.5 fps)", + ) + parser.add_argument("--stream", action="store_true", help="Enable streaming (PCM output)") + parser.add_argument( + "--response-format", + default="wav", + choices=["wav", "mp3", "flac", "pcm"], + help="Audio format (default: wav). Streaming is pcm/wav only. Opus is not supported at 22.05 kHz.", + ) + parser.add_argument("--output", "-o", default=None, help="Output file path") + args = parser.parse_args() + run_tts(args) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/online_serving/test_gepard_tts.py b/tests/e2e/online_serving/test_gepard_tts.py new file mode 100644 index 000000000..e6887bbbc --- /dev/null +++ b/tests/e2e/online_serving/test_gepard_tts.py @@ -0,0 +1,524 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""E2E online serving tests for Gepard-1.0 via ``/v1/audio/speech``. + +Zero-shot default voice. Needs a GPU and the NeMo NanoCodec; the weekly +``TTS · Gepard-1.0 · L4`` step installs NeMo then runs this file. +""" + +from __future__ import annotations + +import array +import os +import statistics +import struct +import time +import wave +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from io import BytesIO + +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +import pytest +import requests +import soundfile as sf + +from tests.helpers.mark import hardware_test +from tests.helpers.media import convert_audio_bytes_to_text +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +pytest.importorskip("nemo.collections.tts.models") + +pytestmark = [pytest.mark.slow, pytest.mark.tts] + +MODEL = "nineninesix/gepard-1.0" +SAMPLE_RATE = 22050 +SAMPLES_PER_FRAME = 1024 +PCM16_BYTES_PER_FRAME = SAMPLES_PER_FRAME * 2 +DEFAULT_TIMEOUT_S = 180.0 + +_DISTINGUISHABLE_PROMPTS = { + "He drinks coffee every morning.": "coffee", + "Machine learning is interesting.": "learning", + "Please close the window before leaving.": "window", + "My favorite color is purple.": "purple", +} + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_deploy_config_path("gepard.yaml"), + server_args=["--trust-remote-code", "--disable-log-stats"], + stage_init_timeout=900, + ), + id="gepard", + ) +] + + +def _base_config(omni_server, text: str, **extra) -> dict: + cfg = { + "model": omni_server.model, + "input": text, + "voice": "default", + "timeout": DEFAULT_TIMEOUT_S, + "seed": 7, + # NanoCodec is quieter than the 24 kHz models the PCM HNR helper was + # calibrated on; keep the catastrophic-failure check, not a quality gate. + "min_hnr_db": -2.0, + "transcript_escalation_model": "large-v3", + } + cfg.update(extra) + return cfg + + +def _wav_sample_rate(wav_bytes: bytes) -> int: + return struct.unpack_from(" int: + data, _sr = sf.read(BytesIO(wav_bytes), dtype="int16") + return int(data.size) * 2 + + +def _pcm16le_to_wav(pcm: bytes, sample_rate: int = SAMPLE_RATE) -> bytes: + """Whisper helpers expect a container; streamed speech is raw s16le.""" + buf = BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(pcm) + return buf.getvalue() + + +def _pcm16_samples(pcm: bytes) -> array.array: + samples = array.array("h") + samples.frombytes(pcm) + return samples + + +def _pcm16_diag(label: str, left: bytes, right: bytes) -> None: + extra_frames = (len(left) - len(right)) // PCM16_BYTES_PER_FRAME + overlap = min(len(left), len(right)) + first_frame_match = ( + overlap >= PCM16_BYTES_PER_FRAME and left[:PCM16_BYTES_PER_FRAME] == right[:PCM16_BYTES_PER_FRAME] + ) + worst = None + corr = None + first_diff_sample = None + if overlap >= 2 and overlap % 2 == 0: + a = _pcm16_samples(left[:overlap]) + b = _pcm16_samples(right[:overlap]) + worst = max(abs(x - y) for x, y in zip(a, b, strict=True)) + for i, (x, y) in enumerate(zip(a, b, strict=True)): + if x != y: + first_diff_sample = i + break + try: + corr = round(statistics.correlation(a, b), 6) + except statistics.StatisticsError: + corr = None + print( + f"[Gepard pcm-diag] {label} left={len(left)} right={len(right)} " + f"extra_frames={extra_frames} first_frame_match={first_frame_match} " + f"first_diff_sample={first_diff_sample} max_abs={worst} corr={corr}" + ) + + +def _scan_logs_for_preemption(omni_server) -> None: + log_paths = getattr(omni_server, "_stage_log_paths", {}) or {} + hits: list[str] = [] + for path in log_paths.values(): + if path is None or not path.exists(): + continue + text = path.read_text(encoding="utf-8", errors="replace").lower() + if "preempt" in text: + hits.append(str(path)) + assert not hits, f"preemption mentioned in stage logs: {hits}" + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +@pytest.mark.parametrize("response_format", ["wav", "pcm"]) +def test_text_to_audio_basic(omni_server, online_client, response_format: str, run_level: str) -> None: + text = "Hello, this is Gepard speaking." + # whisper-small/large-v3 mishear 22.05 kHz WAV of this prompt as + # "Jeb Ard" / "Jeff Bard" (cosine 0.80 / 0.75). Offline uses keyword + # containment; the PCM twin below still goes through the 0.9 ASR gate. + if response_format == "wav": + url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech" + payload = { + "model": omni_server.model, + "input": text, + "voice": "default", + "seed": 7, + "stream": False, + "response_format": "wav", + } + r = requests.post(url, json=payload, timeout=DEFAULT_TIMEOUT_S) + r.raise_for_status() + audio = r.content + assert _wav_sample_rate(audio) == SAMPLE_RATE + pcm_len = _wav_pcm_payload_len(audio) + else: + [resp] = online_client.send_audio_speech_request( + _base_config(omni_server, text, stream=False, response_format=response_format) + ) + assert resp.audio_bytes + pcm_len = len(resp.audio_bytes) + assert pcm_len % PCM16_BYTES_PER_FRAME == 0 + if run_level in {"advanced_model", "full_model"}: + duration = pcm_len / 2 / SAMPLE_RATE + assert 0.5 < duration < 30.0, f"implausible duration {duration:.2f}s" + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_seed_determinism(omni_server, online_client) -> None: + text = "Hello, this is Gepard speaking." + [a] = online_client.send_audio_speech_request(_base_config(omni_server, text, seed=7, response_format="pcm")) + [b] = online_client.send_audio_speech_request(_base_config(omni_server, text, seed=7, response_format="pcm")) + [c] = online_client.send_audio_speech_request(_base_config(omni_server, text, seed=11, response_format="pcm")) + assert a.audio_bytes == b.audio_bytes + assert a.audio_bytes != c.audio_bytes + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_streaming_pcm_incremental(omni_server, online_client) -> None: + text = "Hello, this is Gepard speaking." + url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech" + payload = { + "model": omni_server.model, + "input": text, + "voice": "default", + "seed": 7, + "stream": True, + "stream_format": "audio", + "response_format": "pcm", + } + start = time.perf_counter() + arrivals: list[tuple[float, int]] = [] + streamed = bytearray() + with requests.post(url, json=payload, stream=True, timeout=DEFAULT_TIMEOUT_S) as resp: + resp.raise_for_status() + content_type = resp.headers.get("content-type") or "" + assert "pcm" in content_type or "octet-stream" in content_type + for chunk in resp.iter_content(chunk_size=None): + if not chunk: + continue + arrivals.append((time.perf_counter(), len(chunk))) + streamed.extend(chunk) + assert len(arrivals) >= 2, f"expected incremental chunks, got {len(arrivals)}" + assert arrivals[1][0] > arrivals[0][0] + ttfa_ms = (arrivals[0][0] - start) * 1000.0 + cadence_ms = [(arrivals[i][0] - arrivals[i - 1][0]) * 1000.0 for i in range(1, len(arrivals))] + print(f"[Gepard streaming] TTFA_ms={ttfa_ms:.1f} chunk_cadence_ms={cadence_ms}") + + [non_stream] = online_client.send_audio_speech_request( + _base_config(omni_server, text, seed=7, stream=False, response_format="pcm") + ) + assert bytes(streamed) == non_stream.audio_bytes + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_seeded_streaming_stops(omni_server) -> None: + url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech" + payload = { + "model": omni_server.model, + "input": "Hello, this is Gepard speaking.", + "voice": "default", + "seed": 7, + "stream": True, + "stream_format": "audio", + "response_format": "pcm", + } + with requests.post(url, json=payload, stream=True, timeout=DEFAULT_TIMEOUT_S) as resp: + resp.raise_for_status() + audio = b"".join(chunk for chunk in resp.iter_content(chunk_size=None) if chunk) + duration = len(audio) / 2 / SAMPLE_RATE + assert duration < 30.0, f"seeded stream ran {duration:.1f}s — stop constraint dropped?" + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_sse_stream_emits_delta_then_done(omni_server) -> None: + url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech" + payload = { + "model": omni_server.model, + "input": "Hello, this is Gepard speaking.", + "voice": "default", + "seed": 7, + "stream": True, + "stream_format": "sse", + "response_format": "pcm", + } + with requests.post(url, json=payload, stream=True, timeout=DEFAULT_TIMEOUT_S) as resp: + resp.raise_for_status() + assert resp.headers.get("content-type", "").startswith("text/event-stream") + body = b"".join(resp.iter_content(chunk_size=None)).decode("utf-8", errors="replace") + assert "speech.audio.delta" in body + assert "speech.audio.done" in body + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_concurrent_requests_stay_isolated(omni_server, online_client, run_level: str) -> None: + def _one(text: str): + return online_client.send_audio_speech_request(_base_config(omni_server, text, seed=7, response_format="wav"))[ + 0 + ] + + texts = list(_DISTINGUISHABLE_PROMPTS) + with ThreadPoolExecutor(max_workers=4) as pool: + responses = list(pool.map(_one, texts)) + + assert all(r.success for r in responses) + for i in range(len(responses)): + for j in range(i + 1, len(responses)): + assert responses[i].audio_bytes != responses[j].audio_bytes + + if run_level in {"advanced_model", "full_model"}: + for text, resp in zip(texts, responses, strict=True): + keyword = _DISTINGUISHABLE_PROMPTS[text] + transcript = convert_audio_bytes_to_text(resp.audio_bytes, language="en").lower() + assert keyword in transcript, f"expected {keyword!r} in {transcript!r} for {text!r}" + + _scan_logs_for_preemption(omni_server) + + +def _speech_url(omni_server) -> str: + return f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech" + + +def _stream_payload(omni_server, text: str, **extra) -> dict: + payload = { + "model": omni_server.model, + "input": text, + "voice": "default", + "seed": 7, + "stream": True, + "stream_format": "audio", + "response_format": "pcm", + } + payload.update(extra) + return payload + + +@dataclass +class _StreamCapture: + pcm: bytes + arrivals: list[tuple[float, int]] + + @property + def duration_s(self) -> float: + return len(self.pcm) / 2 / SAMPLE_RATE + + +def _capture_stream( + url: str, + payload: dict, + timeout_s: float, + *, + sleep_after_first_s: float = 0.0, +) -> _StreamCapture: + start = time.perf_counter() + arrivals: list[tuple[float, int]] = [] + pcm = bytearray() + with requests.post(url, json=payload, stream=True, timeout=timeout_s) as resp: + resp.raise_for_status() + first = True + for chunk in resp.iter_content(chunk_size=None): + if not chunk: + continue + arrivals.append((time.perf_counter() - start, len(chunk))) + pcm.extend(chunk) + if first and sleep_after_first_s > 0: + time.sleep(sleep_after_first_s) + first = False + return _StreamCapture(bytes(pcm), arrivals) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_concurrent_streaming_stays_isolated(omni_server, run_level: str) -> None: + url = _speech_url(omni_server) + texts = list(_DISTINGUISHABLE_PROMPTS) + + def _one(text: str) -> tuple[str, _StreamCapture]: + return text, _capture_stream(url, _stream_payload(omni_server, text), DEFAULT_TIMEOUT_S) + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(_one, texts)) + + captures = [cap for _, cap in results] + assert all(len(c.arrivals) >= 2 for c in captures) + for i in range(len(captures)): + for j in range(i + 1, len(captures)): + assert captures[i].pcm != captures[j].pcm + + if run_level in {"advanced_model", "full_model"}: + for text, cap in results: + keyword = _DISTINGUISHABLE_PROMPTS[text] + transcript = convert_audio_bytes_to_text(_pcm16le_to_wav(cap.pcm), language="en").lower() + assert keyword in transcript, f"expected {keyword!r} in {transcript!r} for {text!r}" + + _scan_logs_for_preemption(omni_server) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_slow_client_still_completes(omni_server, online_client) -> None: + """Slow reader must finish, and seed=7 must match a prior non-stream. + + Runs before ``test_client_disconnect_releases_capacity`` so an abort + storm cannot contaminate this comparison. Prints overlap diagnostics + before the bit-exact asserts. + """ + text = "Hello, this is Gepard speaking." + [baseline] = online_client.send_audio_speech_request( + _base_config(omni_server, text, seed=7, stream=False, response_format="pcm") + ) + cap = _capture_stream( + url=_speech_url(omni_server), + payload=_stream_payload(omni_server, text), + timeout_s=DEFAULT_TIMEOUT_S, + sleep_after_first_s=3.0, + ) + assert len(cap.arrivals) >= 2 + assert len(cap.pcm) % PCM16_BYTES_PER_FRAME == 0 + assert 0.5 < cap.duration_s < 30.0 + print(f"[Gepard slow-client] chunks={len(cap.arrivals)} duration_s={cap.duration_s:.3f}") + _pcm16_diag("slow-vs-prior-nonstream", cap.pcm, baseline.audio_bytes) + [after] = online_client.send_audio_speech_request( + _base_config(omni_server, text, seed=7, stream=False, response_format="pcm") + ) + _pcm16_diag("after-nonstream-vs-prior", after.audio_bytes, baseline.audio_bytes) + assert cap.pcm == baseline.audio_bytes + assert after.audio_bytes == baseline.audio_bytes + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_client_disconnect_releases_capacity(omni_server, online_client) -> None: + """A follow-up request must complete after in-flight streams are closed. + + Logs PCM diagnostics against a pre-abort ``seed=7`` baseline. Bit-exact + equality after abort is not asserted here: a dying stream can still share + a GPU batch with the next generate after ``abort()`` returns, and Gepard's + in-model stop head / bf16 GEMM are not row-stable in that batch. + """ + url = _speech_url(omni_server) + text = "Hello, this is Gepard speaking." + [baseline] = online_client.send_audio_speech_request( + _base_config(omni_server, text, seed=7, stream=False, response_format="pcm") + ) + long_payload = _stream_payload( + omni_server, + "Please keep talking about the weather, the window, coffee, and purple flowers.", + max_new_tokens=80, + ) + for _ in range(4): + resp = requests.post(url, json=long_payload, stream=True, timeout=DEFAULT_TIMEOUT_S) + resp.raise_for_status() + try: + chunk = next(c for c in resp.iter_content(chunk_size=None) if c) + assert chunk + finally: + resp.close() + + t0 = time.perf_counter() + follow = requests.post( + url, + json={ + "model": omni_server.model, + "input": text, + "voice": "default", + "seed": 7, + "stream": False, + "response_format": "pcm", + }, + timeout=DEFAULT_TIMEOUT_S, + ) + follow.raise_for_status() + elapsed = time.perf_counter() - t0 + pcm_len = len(follow.content) + assert pcm_len % PCM16_BYTES_PER_FRAME == 0 + print(f"[Gepard disconnect] followup_s={elapsed:.2f} pcm_bytes={pcm_len}") + _pcm16_diag("disconnect-followup-vs-prior", follow.content, baseline.audio_bytes) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_mixed_stream_and_nonstream_isolated(omni_server, online_client, run_level: str) -> None: + url = _speech_url(omni_server) + stream_text = "He drinks coffee every morning." + nonstream_text = "My favorite color is purple." + + def _stream(): + return _capture_stream(url, _stream_payload(omni_server, stream_text), DEFAULT_TIMEOUT_S) + + def _nonstream(): + return online_client.send_audio_speech_request( + _base_config(omni_server, nonstream_text, seed=7, response_format="wav") + )[0] + + with ThreadPoolExecutor(max_workers=2) as pool: + stream_future = pool.submit(_stream) + nonstream_future = pool.submit(_nonstream) + streamed = stream_future.result() + nonstream = nonstream_future.result() + + assert streamed.pcm + assert nonstream.success + assert streamed.pcm != nonstream.audio_bytes + if run_level in {"advanced_model", "full_model"}: + stream_tr = convert_audio_bytes_to_text(_pcm16le_to_wav(streamed.pcm), language="en").lower() + nonstream_tr = convert_audio_bytes_to_text(nonstream.audio_bytes, language="en").lower() + assert "coffee" in stream_tr, stream_tr + assert "purple" in nonstream_tr, nonstream_tr + _scan_logs_for_preemption(omni_server) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_max_new_tokens_caps_whole_frames(omni_server) -> None: + frames = 16 + url = _speech_url(omni_server) + r = requests.post( + url, + json={ + "model": omni_server.model, + "input": "Hello, this is Gepard speaking.", + "voice": "default", + "seed": 7, + "stream": False, + "response_format": "pcm", + "max_new_tokens": frames, + }, + timeout=DEFAULT_TIMEOUT_S, + ) + assert r.status_code == 200, r.text + pcm_len = len(r.content) + assert pcm_len % PCM16_BYTES_PER_FRAME == 0 + assert pcm_len == frames * PCM16_BYTES_PER_FRAME, f"got {pcm_len} bytes, want {frames} frames" + duration = pcm_len / 2 / SAMPLE_RATE + assert duration < 1.0, f"budget cap should stop near 16 frames, got {duration:.2f}s" + + over = requests.post( + url, + json={ + "model": omni_server.model, + "input": "Hello, this is Gepard speaking.", + "voice": "default", + "max_new_tokens": 4097, + }, + timeout=DEFAULT_TIMEOUT_S, + ) + assert over.status_code == 400 + assert "max_new_tokens" in over.text diff --git a/tests/entrypoints/openai_api/test_gepard_tts_adapter.py b/tests/entrypoints/openai_api/test_gepard_tts_adapter.py new file mode 100644 index 000000000..f9e1e3dd2 --- /dev/null +++ b/tests/entrypoints/openai_api/test_gepard_tts_adapter.py @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""CPU tests for the Gepard-1.0 speech adapter. + +Covers detection, the reject matrix, voice default, empty input, extra_params, +max_new_tokens frame bounds, and build() parity with the offline prompt helper. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest +import torch +from pydantic import ValidationError +from pytest_mock import MockerFixture + +from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.entrypoints.openai.tts_adapters import detect_tts_model_type, resolve_adapter +from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext +from vllm_omni.entrypoints.openai.tts_adapters.gepard import GepardAdapter +from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig +from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids +from vllm_omni.outputs import OmniRequestOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_MODEL = "nineninesix/gepard-1.0" + + +def _adapter(*, revision: str | None = None) -> GepardAdapter: + engine = SimpleNamespace(model_config=SimpleNamespace(model=_MODEL, revision=revision)) + return GepardAdapter(SpeechServingContext(server=SimpleNamespace(engine_client=engine), engine_client=engine)) + + +def test_gepard_detection_and_registration() -> None: + assert resolve_adapter("gepard") is GepardAdapter + assert GepardAdapter.stage_keys == {"gepard"} + assert GepardAdapter.model_archs == {"GepardTalkerForConditionalGeneration"} + assert detect_tts_model_type("gepard", None) == "gepard" + assert detect_tts_model_type("gepard", "GepardTalkerForConditionalGeneration") == "gepard" + assert detect_tts_model_type(None, "GepardTalkerForConditionalGeneration") == "gepard" + + +def test_gepard_accepts_omitted_and_default_voice() -> None: + adapter = _adapter() + assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.")) is None + assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", voice="default")) is None + assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", voice="Default")) is None + assert ( + adapter.validate( + OpenAICreateSpeechRequest.model_validate({"input": "Hello from Gepard.", "speaker": "default"}) + ) + is None + ) + + +@pytest.mark.parametrize( + ("kwargs", "field"), + [ + ({"voice": "vivian"}, "voice"), + ({"speed": 1.2}, "speed"), + ({"extra_params": {"temperature": 0.3}}, "extra_params"), + ({"extra_params": {"top_p": 0.9}}, "extra_params"), + ({"extra_params": {"top_k": 20}}, "extra_params"), + ({"extra_params": {"foo": 1}}, "extra_params"), + ({"ref_audio": "https://example.com/ref.wav"}, "ref_audio"), + ({"ref_text": "transcript"}, "ref_text"), + ({"ref_audio_2": "https://example.com/ref2.wav"}, "ref_audio_2"), + ({"speaker_embedding": [0.1, 0.2]}, "speaker_embedding"), + ({"x_vector_only_mode": True}, "x_vector_only_mode"), + ({"task_type": "Base"}, "task_type"), + ({"non_streaming_mode": True}, "non_streaming_mode"), + ({"instructions": "speak happily"}, "instructions"), + ({"language": "English"}, "language"), + ({"ambient_sound": "rain"}, "ambient_sound"), + ({"duration_seconds": 2.0}, "duration_seconds"), + ({"initial_codec_chunk_frames": 4}, "initial_codec_chunk_frames"), + ({"word_timestamps": True}, "word_timestamps"), + ({"response_format": "opus"}, "opus"), + ], +) +def test_gepard_rejects_unsupported_fields(kwargs: dict, field: str) -> None: + adapter = _adapter() + err = adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", **kwargs)) + assert err is not None + assert field in err + + +@pytest.mark.parametrize("text", ["", " ", "\n\t"]) +def test_gepard_rejects_empty_input(text: str) -> None: + adapter = _adapter() + err = adapter.validate(OpenAICreateSpeechRequest(input=text)) + assert err is not None + assert "empty" in err.lower() + + +@pytest.mark.parametrize("max_new_tokens", [0, -1]) +def test_gepard_schema_rejects_non_positive_max_new_tokens(max_new_tokens: int) -> None: + with pytest.raises(ValidationError, match="greater than or equal to 1"): + OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=max_new_tokens) + + +def test_gepard_rejects_max_new_tokens_above_frame_budget() -> None: + adapter = _adapter() + err = adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=4097)) + assert err is not None + assert "max_new_tokens" in err + assert "frames" in err + + +def test_gepard_accepts_max_new_tokens_bounds() -> None: + adapter = _adapter() + assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=1)) is None + assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=4096)) is None + + +def test_gepard_apply_sampling_overrides_uses_frames() -> None: + adapter = _adapter() + stage_defaults = [SimpleNamespace(max_tokens=1000)] + overridden = adapter.apply_sampling_overrides( + stage_defaults, + OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=32), + ) + assert overridden[0].max_tokens == 32 + assert stage_defaults[0].max_tokens == 1000 + + +def test_gepard_build_matches_offline_prompt_builder(monkeypatch) -> None: + adapter = _adapter() + cfg = GepardConfig() + monkeypatch.setattr( + "vllm_omni.model_executor.models.gepard.configuration_gepard.GepardConfig.from_checkpoint", + staticmethod(lambda *_a, **_k: cfg), + ) + + class _Tok: + def __call__(self, text, add_special_tokens=False): + return {"input_ids": [11, 12, 13]} + + adapter._tokenizer = _Tok() + adapter._gepard_config = cfg + text = "Hello from Gepard." + prepared = asyncio.run(adapter.build(OpenAICreateSpeechRequest(input=text, seed=7), [], False)) + + expected = build_gepard_prompt_ids([11, 12, 13], config=cfg) + assert prepared.prompt["prompt_token_ids"] == expected + assert prepared.prompt["additional_information"] == {"text": [text]} + assert prepared.model_type == "gepard" + assert prepared.output_policy.accumulate_nonstreaming is True + + +def test_gepard_loads_tokenizer_and_config_at_served_revision(monkeypatch) -> None: + adapter = _adapter(revision="abc123") + seen: dict[str, object] = {} + + def fake_from_checkpoint(model, backbone_config=None, revision=None): + seen["config"] = (model, revision) + return GepardConfig() + + class _Tok: + def __init__(self, *args, **kwargs): + seen["tok"] = (args[0] if args else None, kwargs.get("revision")) + + def __call__(self, text, add_special_tokens=False): + return {"input_ids": [11, 12, 13]} + + monkeypatch.setattr( + "vllm_omni.model_executor.models.gepard.configuration_gepard.GepardConfig.from_checkpoint", + staticmethod(fake_from_checkpoint), + ) + monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", _Tok) + prepared = asyncio.run(adapter.build(OpenAICreateSpeechRequest(input="Hello from Gepard."), [], False)) + assert seen["config"] == (_MODEL, "abc123") + assert seen["tok"] == (_MODEL, "abc123") + assert prepared.prompt["prompt_token_ids"] == build_gepard_prompt_ids([11, 12, 13], config=GepardConfig()) + + +def _gepard_mock_output(sample_rate: int = 22050, num_samples: int = 2048) -> OmniRequestOutput: + class MockCompletionOutput: + def __init__(self) -> None: + self.index = 0 + self.text = "" + self.token_ids: list[int] = [] + self.finish_reason = "stop" + self.stop_reason = None + self.logprobs = None + + class MockRequestOutput: + def __init__(self) -> None: + self.request_id = "speech-gepard-mock" + self.outputs = [MockCompletionOutput()] + self.multimodal_output = { + "audio": torch.zeros(num_samples), + "sr": torch.tensor(sample_rate), + } + self.finished = True + self.prompt_token_ids = None + self.encoder_prompt_token_ids = None + self.num_cached_tokens = None + self.prompt_logprobs = None + self.kv_transfer_params = None + + output = OmniRequestOutput.from_stage_output(MockRequestOutput(), stage_id=0, final_output_type="audio") + output.metrics = {"stage_metrics": {"0": {"num_tokens_in": 0, "num_tokens_out": 2}}} + return output + + +@pytest.fixture +def gepard_server(mocker: MockerFixture): + mocker.patch( + "vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers", + return_value={"default"}, + ) + mocker.patch( + "vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate", + return_value=22050.0, + ) + + async def _gen(*_args, **_kwargs): + yield _gepard_mock_output() + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock(model=_MODEL, async_chunk=False, revision=None) + mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=1000, seed=42, extra_args=None)] + mock_engine_client.tts_batch_max_items = 32 + mock_engine_client.generate = mocker.MagicMock(side_effect=lambda **_k: _gen()) + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace(model_stage="gepard", model_arch="GepardTalkerForConditionalGeneration"), + tts_args={}, + ) + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + server = OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + cfg = GepardConfig() + server._adapter._gepard_config = cfg + server._adapter._tokenizer = lambda text, add_special_tokens=False: {"input_ids": [11, 12, 13]} + yield server + server.shutdown() + + +def test_gepard_prepare_uses_prompt_builder(gepard_server) -> None: + request_id, _generator, _ = asyncio.run( + gepard_server._prepare_speech_generation(OpenAICreateSpeechRequest(input="Hello from Gepard.")) + ) + assert request_id.startswith("speech-") + prompt = gepard_server.engine_client.generate.call_args.kwargs["prompt"] + assert prompt["prompt_token_ids"] == build_gepard_prompt_ids([11, 12, 13], config=GepardConfig()) + + +@pytest.mark.parametrize("response_format", ["wav", "pcm", "flac", "mp3"]) +def test_gepard_mocked_engine_encodes_response_formats(gepard_server, response_format: str) -> None: + audio, media_type = asyncio.run( + gepard_server._generate_audio_bytes( + OpenAICreateSpeechRequest(input="Hello from Gepard.", response_format=response_format) + ) + ) + assert isinstance(audio, (bytes, str)) + assert len(audio) > 0 + expected_media_type = {"wav": "audio/wav", "pcm": "audio/pcm", "flac": "audio/flac", "mp3": "audio/mpeg"} + assert media_type == expected_media_type[response_format] + + +def test_gepard_streaming_rejects_non_pcm_wav() -> None: + with pytest.raises(ValidationError, match="requires response_format='pcm' or 'wav'"): + OpenAICreateSpeechRequest(input="Hello from Gepard.", stream=True, response_format="mp3") diff --git a/tests/entrypoints/openai_api/test_serving_speech_metrics.py b/tests/entrypoints/openai_api/test_serving_speech_metrics.py index a5e284a01..a1fb74a46 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_metrics.py +++ b/tests/entrypoints/openai_api/test_serving_speech_metrics.py @@ -46,6 +46,7 @@ def _serving(metrics: _MetricsStub, *, adapter=None) -> OmniOpenAIServingSpeech: serving._tts_model_type = "qwen3_tts" serving.engine_client = SimpleNamespace(mod_metrics=metrics, request_states={}) serving._get_tts_adapter = lambda: adapter + serving._speech_output_policies = {} serving.create_audio = lambda audio_obj: SimpleNamespace( audio_data=b"\0\0" * int(audio_obj.audio_tensor.size), media_type="audio/pcm", diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index edd2b4cd4..e0a165a55 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -69,6 +69,7 @@ EXPECTED_MODEL_TYPES = { "step_audio2", "indextts2", "indextts2_5", + "gepard", "dots_tts", } @@ -143,6 +144,17 @@ def _build_moss_tts_request(adapter_cls, mocker, *, request_seed): ) +@pytest.mark.parametrize( + ("adapter_cls", "expect_accumulate"), + [(MossTTSAdapter, False), (MossTTSNanoAdapter, True)], +) +def test_moss_tts_accumulate_nonstreaming_follows_adapter_flag(adapter_cls, expect_accumulate, mocker): + prepared = _build_moss_tts_request(adapter_cls, mocker, request_seed=7) + + assert adapter_cls.accumulate_nonstreaming is expect_accumulate + assert prepared.output_policy.accumulate_nonstreaming is expect_accumulate + + # Full-family coverage pins the adapter contract; only Nano consumes this seed end to end today. @pytest.mark.parametrize("adapter_cls", [MossTTSAdapter, MossTTSNanoAdapter]) @pytest.mark.parametrize("request_seed", [0, 1234]) diff --git a/tests/entrypoints/openai_api/test_tts_detection.py b/tests/entrypoints/openai_api/test_tts_detection.py index 6c7671659..90e7e0fee 100644 --- a/tests/entrypoints/openai_api/test_tts_detection.py +++ b/tests/entrypoints/openai_api/test_tts_detection.py @@ -148,6 +148,7 @@ _PIPELINE_STAGES = [ "dit", "fish_speech_slow_ar", "fused_thinker_talker", + "gepard", "glm_tts", "glm_tts_dit", "higgs_audio_v2", diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index be282da81..33abbb618 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -231,6 +231,7 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc serving._tts_model_type = "higgs_audio_v3" serving._get_tts_adapter = lambda: Adapter() serving._track_ref_audio_artifact_warmup = lambda *args, **kwargs: None + serving._speech_output_policies = {} _, stage_sampling_params, _ = asyncio.run(serving._prepare_speech_generation(request, request_id="speech-test")) stage0_params = stage_sampling_params[0] diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index d4aa72818..f54ca3e28 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -53,6 +53,7 @@ from vllm_omni.entrypoints.openai.speech_usage import ( qwen3_tts_input_token_details, ) from vllm_omni.entrypoints.openai.tts_adapters import ( + OutputPolicy, SpeechServingContext, TTSGenerationError, all_tts_stage_keys, @@ -396,6 +397,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._max_instructions_length = self._compute_max_instructions_length() self._tts_tokenizer = None + # Per-request output policy from the adapter's PreparedRequest. Keyed + # by request_id so concurrent speech requests cannot overwrite each + # other. Stored only for non-streaming requests and consumed (popped) + # by the non-streaming accumulator in `_generate_audio_bytes`. + self._speech_output_policies: dict[str, OutputPolicy] = {} # Batch configuration self._batch_max_items: int = getattr(self.engine_client, "tts_batch_max_items", 32) @@ -1872,6 +1878,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): tts_params = prepared.tts_params model_type = prepared.model_type qwen3_ref_audio_warmup_artifact_key = prepared.warmup_artifact_key + output_policy = prepared.output_policy else: # Qwen omni models (Qwen3-Omni, Qwen2.5-Omni) use a "talker" # stage whose preprocess requires chat-templated tokens. The @@ -1893,6 +1900,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ) tts_params = {} prompt = {"prompt": request.input} + output_policy = OutputPolicy() if model_type is None: if self._is_tts: @@ -1962,6 +1970,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): qwen3_ref_audio_warmup_artifact_key, x_vector_only=self._tts_x_vector_only(tts_params), ) + # Only the non-streaming path consumes output policies (in + # `_generate_audio_bytes`); streaming never reads them. Store late — + # after every fallible step above — and skip streaming requests + # entirely, so no caller has to discard entries (request ids are + # unique per request, so a stranded entry would never be reclaimed). + if not request.is_streaming(): + self._speech_output_policies[request_id] = output_policy return request_id, generator, tts_params async def _generate_pcm_chunks( @@ -2041,13 +2056,17 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): artifact_ready = False try: - # MOSS-TTS-Nano emits delta chunks per yield (single-stage, - # async_chunk=false). The engine surfaces each yield as its own - # RequestOutput, so we need to accumulate across the async-for loop — - # final_output alone only carries the last (often empty) sentinel. - is_moss = self._tts_model_type == "moss_tts_nano" - moss_chunks: list[Any] = [] - moss_sample_rate: int | None = None + # Sparse-audio models (MOSS-TTS-Nano, Gepard) emit delta chunks per + # yield with async_chunk=false. The engine surfaces each yield as its + # own RequestOutput, so we accumulate across the async-for loop when + # the adapter asks for it — final_output alone may only carry the + # last (often empty) sentinel. Prefer the engine's own concatenated + # waveform when FINAL_ONLY already produced one. + accumulate_nonstreaming = self._speech_output_policies.pop( + request_id, OutputPolicy() + ).accumulate_nonstreaming + delta_chunks: list[Any] = [] + delta_sample_rate: int | None = None final_output: OmniRequestOutput | None = None # Non-streaming is FINAL_ONLY, so the stage-0 output carries the full @@ -2067,7 +2086,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): _, audio_key = self._extract_audio_output(res) if audio_key is not None: audio_res = res - if not is_moss: + if not accumulate_nonstreaming: continue try: step_audio, step_key = self._extract_audio_output(res) @@ -2080,11 +2099,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): candidates = chunk if isinstance(chunk, list) else [chunk] for cand in candidates: if hasattr(cand, "numel") and cand.numel() > 0: - moss_chunks.append(cand) + delta_chunks.append(cand) sr_step = step_audio.get("sr") if sr_step is not None: sr_val_step = sr_step[-1] if isinstance(sr_step, list) and sr_step else sr_step - moss_sample_rate = int(sr_val_step.item()) if hasattr(sr_val_step, "item") else int(sr_val_step) + delta_sample_rate = int(sr_val_step.item()) if hasattr(sr_val_step, "item") else int(sr_val_step) if final_output is None: raise ValueError("No output generated from the model.") @@ -2117,7 +2136,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) - if is_moss: + if accumulate_nonstreaming: # Prefer the engine's own consolidated audio when present. After the # vllm 0.20 rebase non-stream requests resolve to FINAL_ONLY, so # final_output already carries the full concatenated waveform; the @@ -2133,12 +2152,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if final_audio is not None: audio_tensor = final_audio - elif moss_chunks: - audio_tensor = torch.cat(moss_chunks, dim=-1) + elif delta_chunks: + audio_tensor = torch.cat(delta_chunks, dim=-1) else: audio_tensor = np.zeros((0,), dtype=np.float32) - if moss_sample_rate is not None: - sample_rate = moss_sample_rate + if delta_sample_rate is not None: + sample_rate = delta_sample_rate elif isinstance(audio_tensor, list): async_chunk = bool(getattr(self.engine_client.model_config, "async_chunk", False)) if async_chunk: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py index 8ed54afd8..9f5387a7f 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py @@ -110,6 +110,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( # noqa: E402,F401 covo_audio, dots_tts, fish_speech, + gepard, glm_tts, higgs_audio_v2, higgs_audio_v3, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/gepard.py b/vllm_omni/entrypoints/openai/tts_adapters/gepard.py new file mode 100644 index 000000000..255525a0d --- /dev/null +++ b/vllm_omni/entrypoints/openai/tts_adapters/gepard.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Gepard-1.0 serving adapter for ``/v1/audio/speech``. + +Zero-shot only: the trained ``null_prefix`` is the default voice. Prompt IDs +are assembled with the same layout helper the offline path uses. Frame budget +rides ``SamplingParams.max_tokens`` (one token = one 1024-sample frame). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from vllm.utils.async_utils import make_async + +from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter +from vllm_omni.entrypoints.openai.tts_adapters.base import ( + ARTTSAdapter, + OutputPolicy, + PreparedRequest, + apply_max_new_tokens, + conditioning_cache_salt, +) + +if TYPE_CHECKING: + from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest + from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig + +_DEFAULT_VOICE = "default" + +# Declared schema fields Gepard does not consume. Rejecting them here is +# load-bearing: the pydantic model silently drops undeclared keys, so a +# declared-but-ignored field would otherwise succeed and mislead. +_UNSUPPORTED_OPTIONAL_FIELDS: tuple[tuple[str, str], ...] = ( + ("ref_audio", "ref_audio"), + ("ref_text", "ref_text"), + ("ref_audio_2", "ref_audio_2"), + ("speaker_embedding", "speaker_embedding"), + ("x_vector_only_mode", "x_vector_only_mode"), + ("task_type", "task_type"), + ("non_streaming_mode", "non_streaming_mode"), + ("instructions", "instructions"), + ("language", "language"), + ("ambient_sound", "ambient_sound"), + ("duration_seconds", "duration_seconds"), + ("initial_codec_chunk_frames", "initial_codec_chunk_frames"), +) + + +@register_tts_adapter +class GepardAdapter(ARTTSAdapter): + name = "gepard" + stage_keys = frozenset({"gepard"}) + model_archs = frozenset({"GepardTalkerForConditionalGeneration"}) + + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._tokenizer: Any = None + self._gepard_config: GepardConfig | None = None + self._build_prompt_async = make_async(self._build_prompt, executor=getattr(ctx.server, "_tts_executor", None)) + + def _load_supported_speakers(self) -> set[str]: + return {_DEFAULT_VOICE} + + def validate(self, request: OpenAICreateSpeechRequest) -> str | None: + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.voice is not None: + voice = request.voice.lower() + if voice != _DEFAULT_VOICE: + return ( + f"Invalid voice '{request.voice}'. Supported: {_DEFAULT_VOICE}. " + "Voice cloning from reference audio is not available yet." + ) + + if request.speed is not None and request.speed != 1.0: + return "Gepard does not support 'speed' adjustments; audio is rendered at the native 22.05 kHz rate" + + if request.extra_params: + keys = ", ".join(sorted(request.extra_params)) + return ( + f"Gepard does not accept extra_params (got: {keys}). " + "Sampling is fixed in-model; temperature/top_p/top_k cannot be overridden." + ) + + if request.word_timestamps: + return "Gepard does not support 'word_timestamps'; there is no forced-aligner stage in this pipeline" + + fmt = (request.response_format or "").lower() + if fmt == "opus": + return ( + "Gepard does not support 'response_format'='opus'; " + "native 22.05 kHz is not an Opus sample rate (8000, 12000, 16000, 24000, 48000)" + ) + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min or request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens must be between {self.max_new_tokens_min} and {self.max_new_tokens_max} frames" + + for attr, field in _UNSUPPORTED_OPTIONAL_FIELDS: + if getattr(request, attr, None) is not None: + return f"Gepard does not support '{field}'" + + return None + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: OpenAICreateSpeechRequest, + prompt: dict | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) + + async def build( + self, + request: OpenAICreateSpeechRequest, + sampling_params_list: list, + has_inline_ref_audio: bool, + ) -> PreparedRequest: + tts_params: dict = {} + prompt = await self._build_prompt_async(request.input) + prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) + return PreparedRequest( + prompt=prompt, + tts_params=tts_params, + model_type=self.name, + output_policy=OutputPolicy(accumulate_nonstreaming=True), + ) + + def _build_prompt(self, text: str) -> dict[str, Any]: + from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids + + prompt_token_ids = build_gepard_prompt_ids(self._tokenize(text), config=self._config()) + return { + "prompt_token_ids": prompt_token_ids, + "additional_information": {"text": [text]}, + } + + def _model_config(self) -> Any: + engine = self.ctx.engine_client + assert engine is not None + return engine.model_config + + def _config(self) -> GepardConfig: + if self._gepard_config is None: + from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig + + model_config = self._model_config() + self._gepard_config = GepardConfig.from_checkpoint( + model_config.model, revision=getattr(model_config, "revision", None) + ) + return self._gepard_config + + def _tokenize(self, text: str) -> list[int]: + tokenizer = self._tokenizer + if tokenizer is None: + from transformers import AutoTokenizer + + model_config = self._model_config() + tokenizer = AutoTokenizer.from_pretrained( + model_config.model, + revision=getattr(model_config, "revision", None), + trust_remote_code=True, + ) + self._tokenizer = tokenizer + return tokenizer(text, add_special_tokens=False)["input_ids"] diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index 07e4bb197..bed20efc2 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -13,6 +13,7 @@ from vllm.inputs import tokens_input from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ( ARTTSAdapter, + OutputPolicy, PreparedRequest, apply_max_new_tokens, conditioning_cache_salt, @@ -23,6 +24,8 @@ if TYPE_CHECKING: class _MossTTSAdapterBase(ARTTSAdapter): + accumulate_nonstreaming: bool = False + def __init__(self, ctx) -> None: super().__init__(ctx) self._moss_variant = None if self.name == "moss_tts_nano" else self._detect_moss_variant() @@ -369,7 +372,12 @@ class _MossTTSAdapterBase(ARTTSAdapter): prompt = tokens_input(prompt_token_ids=[1]) prompt["additional_information"] = tts_params prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) - return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type=self.name) + return PreparedRequest( + prompt=prompt, + tts_params=tts_params, + model_type=self.name, + output_policy=OutputPolicy(accumulate_nonstreaming=self.accumulate_nonstreaming), + ) def apply_sampling_overrides( self, @@ -385,6 +393,7 @@ class _MossTTSAdapterBase(ARTTSAdapter): class MossTTSNanoAdapter(_MossTTSAdapterBase): stage_keys = frozenset({"moss_tts_nano"}) name = "moss_tts_nano" + accumulate_nonstreaming = True @register_tts_adapter