From 11f6a0b72e547cd960fd3b09f8d6cec502a42828 Mon Sep 17 00:00:00 2001 From: faceless void <100186390+syd520zy@users.noreply.github.com> Date: Tue, 30 Jun 2026 10:17:24 +0800 Subject: [PATCH] Make speech streaming default to SSE (#4679) Signed-off-by: syd520zy <529477025@qq.com> --- benchmarks/fish-speech/bench_speaker_cache.py | 2 + docs/serving/speech_api.md | 17 ++-- .../examples/online_serving/text_to_speech.md | 8 +- .../online_serving/text_to_speech/README.md | 9 ++- .../cosyvoice3/speech_client.py | 1 + .../text_to_speech/fish_speech/gradio_demo.py | 2 + .../fish_speech/speech_client.py | 1 + .../text_to_speech/glm_tts/gradio_demo.py | 2 + .../glm_tts/openai_speech_client.py | 3 + .../ming_tts/openai_speech_client.py | 1 + .../text_to_speech/ming_tts/run_curl.sh | 1 + .../moss_tts_nano/gradio_demo.py | 1 + .../text_to_speech/qwen3_tts/tts_common.py | 2 + .../text_to_speech/voxcpm2/gradio_demo.py | 2 + recipes/OpenBMB/VoxCPM2.md | 1 + recipes/OpenMOSS/MOSS-TTS.md | 3 +- recipes/Qwen/Qwen3-TTS.md | 3 +- recipes/mistralai/Voxtral-TTS.md | 1 + .../test_cosyvoice3_tts_expansion.py | 1 + .../online_serving/test_glm_tts_expansion.py | 1 + .../test_higgs_audio_v2_expansion.py | 2 + .../e2e/online_serving/test_higgs_audio_v3.py | 2 + tests/e2e/online_serving/test_indextts2.py | 1 + .../online_serving/test_ming_tts_expansion.py | 1 + .../online_serving/test_moss_tts_expansion.py | 1 + .../test_moss_tts_nano_expansion.py | 1 + .../e2e/online_serving/test_qwen3_tts_base.py | 1 + .../test_qwen3_tts_base_expansion.py | 1 + .../test_qwen3_tts_customvoice.py | 1 + .../test_qwen3_tts_customvoice_expansion.py | 1 + tests/e2e/online_serving/test_voxcpm2_tts.py | 1 + .../test_voxcpm2_tts_expansion.py | 1 + .../test_voxtral_tts_expansion.py | 1 + .../test_qwen3_tts_speaker_embedding.py | 1 + .../openai_api/test_serving_speech.py | 79 ++++++++++++++----- tests/helpers/runtime.py | 1 + vllm_omni/benchmarks/patch/patch.py | 10 ++- .../entrypoints/openai/protocol/audio.py | 12 +-- .../entrypoints/openai/serving_speech.py | 24 +++--- 39 files changed, 146 insertions(+), 58 deletions(-) diff --git a/benchmarks/fish-speech/bench_speaker_cache.py b/benchmarks/fish-speech/bench_speaker_cache.py index a2ac460af..764e4e9df 100644 --- a/benchmarks/fish-speech/bench_speaker_cache.py +++ b/benchmarks/fish-speech/bench_speaker_cache.py @@ -201,6 +201,7 @@ async def main(): "input": prompt, "voice": "default", "stream": True, + "stream_format": "audio", "response_format": "pcm", "ref_audio": ref_audio_b64, "ref_text": args.ref_text, @@ -240,6 +241,7 @@ async def main(): "input": prompt, "voice": args.voice_name, "stream": True, + "stream_format": "audio", "response_format": "pcm", "ref_text": args.ref_text, "max_new_tokens": 2048, diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index c11bbd1e9..27dfda95f 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -122,8 +122,8 @@ Content-Type: application/json | `max_new_tokens` | integer | 2048 | Maximum tokens to generate | | `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. | | `non_streaming_mode` | bool | null | Qwen3-TTS prompt construction mode override. Does not affect HTTP response streaming or async-chunk pipelining. When null, Qwen3-TTS uses model defaults: Base=false, CustomVoice/VoiceDesign=true. | -| `stream_format` | string | null | Streaming output format. `"audio"` streams raw audio bytes as they are decoded; `"sse"` streams OpenAI `speech.audio.*` Server-Sent Events. Omit (null) for a single non-streaming binary response. See [Response Format](#response-format). | -| `stream` | bool | false | Legacy streaming switch, equivalent to `stream_format="audio"`. Requires `response_format="pcm"` or `"wav"`. Speed adjustment is not supported when streaming. | +| `stream` | bool | false | When true, stream OpenAI `speech.audio.*` SSE events (requires `response_format="pcm"` or `"wav"`). For raw PCM/WAV byte streaming, set `stream_format="audio"`. | +| `stream_format` | string | null | Streaming output format. `"audio"` streams raw audio bytes as they are decoded; `"sse"` streams OpenAI `speech.audio.*` Server-Sent Events. If omitted, `stream=true` selects SSE and `stream=false` remains non-streaming. See [Response Format](#response-format). | **Supported languages:** Only applicable to Qwen3-TTS. Derived from the model configuration (`talker_config.codec_language_id` in the checkpoint's `config.json`), plus `Auto`, which is always accepted. Official Qwen3-TTS checkpoints support: Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian. @@ -143,18 +143,15 @@ The response shape depends on the streaming parameters: complete clip as binary audio data with an appropriate `Content-Type` header (e.g. `audio/wav`). The raw-bytes body has no JSON carrier, so no `usage` is reported. -**Raw audio stream** (`stream=true` or `stream_format="audio"`). Streams raw -audio bytes (PCM or WAV) as they are decoded. +**Raw audio stream** (`stream_format="audio"`). Streams raw audio bytes (PCM or +WAV) as they are decoded. Both streaming modes (`stream_format="audio"` and `"sse"`) require `response_format="pcm"` or `"wav"`, and `speed` must be `1.0` (or omitted). -**SSE stream** (`stream_format="sse"`). Streams [OpenAI `speech.audio.*` Server-Sent -Events](https://platform.openai.com/docs/api-reference/audio-streaming). Set -`stream_format="sse"` and leave the legacy `stream` unset: `stream=true` takes -precedence and forces the raw-audio path, so the response would be audio bytes with -no `speech.audio.*` events (and no terminal `usage`). Each event has an `event:` -line and a JSON `data:` line: +**SSE stream** (`stream=true` or `stream_format="sse"`). Streams [OpenAI +`speech.audio.*` Server-Sent Events](https://platform.openai.com/docs/api-reference/audio-streaming). +Each event has an `event:` line and a JSON `data:` line: - `speech.audio.delta` — a base64 audio chunk: diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index 9ee6b33fb..5640845d8 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -79,7 +79,7 @@ response = client.audio.speech.create( response.stream_to_file("output.wav") ``` -Streaming PCM output (where supported) — set `stream=true` with `response_format="pcm"`: +Streaming PCM output (where supported) — set `stream=true`, `stream_format="audio"`, and `response_format="pcm"`: ```bash curl -X POST http://localhost:8091/v1/audio/speech \ @@ -88,6 +88,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "input": "Hello, how are you?", "voice": "default", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` @@ -414,10 +415,11 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "voice": "vivian", "language": "English", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` -Streaming requires `response_format="pcm"` and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. +Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. ### Streaming WebSocket The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence: @@ -496,7 +498,7 @@ After startup, `/v1/audio/voices` lists `alice`, and `/v1/audio/speech` can use ```bash python voxcpm2/gradio_demo.py ``` -Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Audio is streamed from the OpenAI Speech endpoint with `stream=true`. +Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Raw PCM audio is streamed from the OpenAI Speech endpoint with `stream=true` and `stream_format="audio"`. --- diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index e8f862654..201cb5d78 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -79,7 +79,7 @@ response = client.audio.speech.create( response.stream_to_file("output.wav") ``` -Streaming PCM output (where supported) — set `stream=true` with `response_format="pcm"`: +Streaming PCM output (where supported) — set `stream=true`, `stream_format="audio"`, and `response_format="pcm"`: ```bash curl -X POST http://localhost:8091/v1/audio/speech \ @@ -88,6 +88,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "input": "Hello, how are you?", "voice": "default", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` @@ -363,6 +364,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ \"input\": \"Hello, streaming output from MOSS-TTS-Nano.\", \"ref_audio\": \"data:audio/wav;base64,${REF_AUDIO}\", \"stream\": true, + \"stream_format\": \"audio\", \"response_format\": \"pcm\" }" --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 - ``` @@ -528,10 +530,11 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "voice": "vivian", "language": "English", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` -Streaming requires `response_format="pcm"` and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. +Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. ### Streaming WebSocket The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence: @@ -632,7 +635,7 @@ After startup, `/v1/audio/voices` lists `alice`, and `/v1/audio/speech` can use ```bash python voxcpm2/gradio_demo.py ``` -Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Audio is streamed from the OpenAI Speech endpoint with `stream=true`. +Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Raw PCM audio is streamed from the OpenAI Speech endpoint with `stream=true` and `stream_format="audio"`. --- diff --git a/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py b/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py index a837d9243..5ac11cb97 100644 --- a/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py +++ b/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py @@ -60,6 +60,7 @@ def run_tts(args) -> None: if args.stream: payload["stream"] = True + payload["stream_format"] = "audio" payload["response_format"] = "pcm" print(f"Model: {args.model}") diff --git a/examples/online_serving/text_to_speech/fish_speech/gradio_demo.py b/examples/online_serving/text_to_speech/fish_speech/gradio_demo.py index cd2279476..ed22aa007 100644 --- a/examples/online_serving/text_to_speech/fish_speech/gradio_demo.py +++ b/examples/online_serving/text_to_speech/fish_speech/gradio_demo.py @@ -66,6 +66,8 @@ def build_payload( "response_format": "pcm" if stream else response_format, "stream": stream, } + if stream: + payload["stream_format"] = "audio" # Voice cloning: ref_audio takes priority over URL. ref_url_stripped = ref_audio_url.strip() if ref_audio_url else "" diff --git a/examples/online_serving/text_to_speech/fish_speech/speech_client.py b/examples/online_serving/text_to_speech/fish_speech/speech_client.py index 43acffdc7..726ccb0a6 100644 --- a/examples/online_serving/text_to_speech/fish_speech/speech_client.py +++ b/examples/online_serving/text_to_speech/fish_speech/speech_client.py @@ -53,6 +53,7 @@ def run_tts(args) -> None: if args.stream: payload["stream"] = True + payload["stream_format"] = "audio" payload["response_format"] = "pcm" print(f"Model: {args.model}") diff --git a/examples/online_serving/text_to_speech/glm_tts/gradio_demo.py b/examples/online_serving/text_to_speech/glm_tts/gradio_demo.py index 12e5d4b11..3c5d3e206 100644 --- a/examples/online_serving/text_to_speech/glm_tts/gradio_demo.py +++ b/examples/online_serving/text_to_speech/glm_tts/gradio_demo.py @@ -64,6 +64,8 @@ def build_payload( "stream": stream, "max_new_tokens": max_new_tokens, } + if stream: + payload["stream_format"] = "audio" # Voice cloning: ref_audio takes priority over URL. ref_url_stripped = ref_audio_url.strip() if ref_audio_url else "" diff --git a/examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py b/examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py index a0ff181c3..904d01909 100644 --- a/examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py +++ b/examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py @@ -40,6 +40,9 @@ def run_tts_generation(args) -> None: "ref_audio": args.ref_audio, "ref_text": args.ref_text, } + if args.stream: + payload["stream_format"] = "audio" + payload["response_format"] = "pcm" if args.max_new_tokens: payload["max_new_tokens"] = args.max_new_tokens diff --git a/examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py b/examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py index 536bfb048..cd7bfafbd 100644 --- a/examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py +++ b/examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py @@ -123,6 +123,7 @@ def run_tts(args) -> None: payload["max_new_tokens"] = args.max_new_tokens if args.stream: payload["stream"] = True + payload["stream_format"] = "audio" payload["response_format"] = "pcm" api_url = f"{args.api_base}/v1/audio/speech" diff --git a/examples/online_serving/text_to_speech/ming_tts/run_curl.sh b/examples/online_serving/text_to_speech/ming_tts/run_curl.sh index 10fcaf59a..35f7956f2 100755 --- a/examples/online_serving/text_to_speech/ming_tts/run_curl.sh +++ b/examples/online_serving/text_to_speech/ming_tts/run_curl.sh @@ -65,6 +65,7 @@ PY \"model\": \"${MODEL}\", \"input\": \"${TEXT}\", \"stream\": true, + \"stream_format\": \"audio\", \"response_format\": \"pcm\" }" "$STREAM_OUTPUT" ;; diff --git a/examples/online_serving/text_to_speech/moss_tts_nano/gradio_demo.py b/examples/online_serving/text_to_speech/moss_tts_nano/gradio_demo.py index 4a8e4896d..0278db09f 100644 --- a/examples/online_serving/text_to_speech/moss_tts_nano/gradio_demo.py +++ b/examples/online_serving/text_to_speech/moss_tts_nano/gradio_demo.py @@ -335,6 +335,7 @@ def build_payload( "ref_audio": encode_audio_to_base64(ref_audio), "response_format": "pcm" if stream else response_format, "stream": stream, + **({"stream_format": "audio"} if stream else {}), } diff --git a/examples/online_serving/text_to_speech/qwen3_tts/tts_common.py b/examples/online_serving/text_to_speech/qwen3_tts/tts_common.py index 44ba7c3ab..df66d452b 100644 --- a/examples/online_serving/text_to_speech/qwen3_tts/tts_common.py +++ b/examples/online_serving/text_to_speech/qwen3_tts/tts_common.py @@ -93,6 +93,8 @@ def build_payload( "response_format": "pcm" if stream else response_format, "stream": stream, } + if stream: + payload["stream_format"] = "audio" if not stream: payload["speed"] = speed diff --git a/examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py b/examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py index c6706198a..f9df10d52 100644 --- a/examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py +++ b/examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py @@ -515,6 +515,8 @@ def create_app(api_base: str): "response_format": "pcm" if stream_enabled else "wav", "stream": stream_enabled, } + if stream_enabled: + payload["stream_format"] = "audio" # Reference audio for cloning ref_url_s = ref_url.strip() if ref_url else "" diff --git a/recipes/OpenBMB/VoxCPM2.md b/recipes/OpenBMB/VoxCPM2.md index 9414d8a48..0fd8e210e 100644 --- a/recipes/OpenBMB/VoxCPM2.md +++ b/recipes/OpenBMB/VoxCPM2.md @@ -149,6 +149,7 @@ curl -X POST http://127.0.0.1:8000/v1/audio/speech \ "input": "Streaming PCM test on a single RTX 4090 with VoxCPM2.", "voice": "default", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 - ``` diff --git a/recipes/OpenMOSS/MOSS-TTS.md b/recipes/OpenMOSS/MOSS-TTS.md index c88bf7281..e9cb798fe 100644 --- a/recipes/OpenMOSS/MOSS-TTS.md +++ b/recipes/OpenMOSS/MOSS-TTS.md @@ -104,7 +104,8 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "voice": "default", "ref_audio": "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/zh_1.wav", "response_format": "wav", - "stream": true + "stream": true, + "stream_format": "audio" }' --output output.wav ``` diff --git a/recipes/Qwen/Qwen3-TTS.md b/recipes/Qwen/Qwen3-TTS.md index 164fba2c9..b6d2c9293 100644 --- a/recipes/Qwen/Qwen3-TTS.md +++ b/recipes/Qwen/Qwen3-TTS.md @@ -148,6 +148,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "voice": "vivian", "language": "English", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` @@ -163,7 +164,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-ty - Memory usage: The deploy config allocates `gpu_memory_utilization: 0.3` per stage (talker + code2wav share a single GPU). For the 0.6B variants or constrained GPUs, adjust via `--gpu-memory-utilization`. - Key flags: `--omni` is required. `--deploy-config` points to the bundled two-stage pipeline config. -- Async chunking: Enabled by default in `qwen3_tts.yaml` for streaming-friendly first-audio latency. Streaming requires `stream=true` with `response_format="pcm"`. +- Async chunking: Enabled by default in `qwen3_tts.yaml` for streaming-friendly first-audio latency. Raw audio streaming requires `stream=true`, `stream_format="audio"`, and `response_format="pcm"`. - Task/model matching: Each task type requires its matching model checkpoint. Using a CustomVoice model for a Base (voice clone) request will fail. - Known limitations: The server serves one model variant at a time. To switch task types (e.g., CustomVoice to Base), restart the server with the corresponding model. diff --git a/recipes/mistralai/Voxtral-TTS.md b/recipes/mistralai/Voxtral-TTS.md index aae48a689..40d5ff964 100644 --- a/recipes/mistralai/Voxtral-TTS.md +++ b/recipes/mistralai/Voxtral-TTS.md @@ -80,6 +80,7 @@ curl -X POST http://127.0.0.1:8091/v1/audio/speech \ "voice": "casual_female", "language": "English", "stream": true, + "stream_format": "audio", "response_format": "pcm" }' \ --output voxtral_stream.pcm diff --git a/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py b/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py index c9e7362fc..4b21f956e 100644 --- a/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py +++ b/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py @@ -104,6 +104,7 @@ def test_voice_clone_zh_002(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt("zh"), "stream": True, + "stream_format": "audio", "response_format": "wav", "ref_audio": REF_AUDIO_URL, "ref_text": REF_TEXT, diff --git a/tests/e2e/online_serving/test_glm_tts_expansion.py b/tests/e2e/online_serving/test_glm_tts_expansion.py index db3423732..c558e0c40 100644 --- a/tests/e2e/online_serving/test_glm_tts_expansion.py +++ b/tests/e2e/online_serving/test_glm_tts_expansion.py @@ -97,6 +97,7 @@ def test_voice_clone_zh_async_chunk(omni_server, openai_client) -> None: "model": omni_server.model, "input": "今天天气真不错，适合出去散散步。", "stream": True, + "stream_format": "audio", "response_format": "wav", "ref_audio": REF_AUDIO_URL, "ref_text": REF_TEXT, diff --git a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py index 41f2b6a6a..e7371fe17 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py +++ b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py @@ -98,6 +98,7 @@ class TestHiggsAudioV2OnlineHappyPath: "model": omni_server.model, "input": "Streaming the quick brown fox over the lazy dog.", "stream": True, + "stream_format": "audio", "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, "min_audio_bytes": _MIN_AUDIO_BYTES, @@ -270,6 +271,7 @@ class TestHiggsAudioV2OnlineVoiceClone: "ref_audio": _REF_AUDIO_URL, "ref_text": _REF_TEXT, "stream": True, + "stream_format": "audio", "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, "min_audio_bytes": _MIN_AUDIO_BYTES, diff --git a/tests/e2e/online_serving/test_higgs_audio_v3.py b/tests/e2e/online_serving/test_higgs_audio_v3.py index b8c37a211..c13e90bf5 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v3.py +++ b/tests/e2e/online_serving/test_higgs_audio_v3.py @@ -135,6 +135,7 @@ class TestHiggsAudioV3OnlineHappyPath: "model": omni_server.model, "input": "The quick brown fox jumps over the lazy dog.", "stream": True, + "stream_format": "audio", "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, "min_hnr_db": 0.0, @@ -160,6 +161,7 @@ class TestHiggsAudioV3OnlineHappyPath: "model": omni_server.model, "input": "She sells seashells by the seashore.", "stream": True, + "stream_format": "audio", "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, "min_hnr_db": -2.0, diff --git a/tests/e2e/online_serving/test_indextts2.py b/tests/e2e/online_serving/test_indextts2.py index c0fceb1ce..5ab0d1f8d 100644 --- a/tests/e2e/online_serving/test_indextts2.py +++ b/tests/e2e/online_serving/test_indextts2.py @@ -75,6 +75,7 @@ def test_streaming(omni_server, openai_client) -> None: "model": omni_server.model, "input": "Hello, this is a voice cloning demo.", "stream": True, + "stream_format": "audio", "response_format": "pcm", "ref_audio": REF_AUDIO_URL, "min_audio_bytes": 1024, diff --git a/tests/e2e/online_serving/test_ming_tts_expansion.py b/tests/e2e/online_serving/test_ming_tts_expansion.py index 40619617b..d4fbb7026 100644 --- a/tests/e2e/online_serving/test_ming_tts_expansion.py +++ b/tests/e2e/online_serving/test_ming_tts_expansion.py @@ -90,6 +90,7 @@ def test_text_to_audio_streaming_001(omni_server, openai_client) -> None: "input": get_prompt("zh_short"), "voice": "灵小甄", "stream": True, + "stream_format": "audio", "response_format": "wav", "timeout": 300.0, } diff --git a/tests/e2e/online_serving/test_moss_tts_expansion.py b/tests/e2e/online_serving/test_moss_tts_expansion.py index 4aae8c7c0..2d048fe52 100644 --- a/tests/e2e/online_serving/test_moss_tts_expansion.py +++ b/tests/e2e/online_serving/test_moss_tts_expansion.py @@ -137,6 +137,7 @@ def test_text_to_audio_002_streaming(omni_server, openai_client, ref_audio_data_ "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "response_format": "pcm", "ref_audio": ref_audio_data_url, "min_hnr_db": -5.0, diff --git a/tests/e2e/online_serving/test_moss_tts_nano_expansion.py b/tests/e2e/online_serving/test_moss_tts_nano_expansion.py index 4b1a99b2c..3468a9656 100644 --- a/tests/e2e/online_serving/test_moss_tts_nano_expansion.py +++ b/tests/e2e/online_serving/test_moss_tts_nano_expansion.py @@ -129,6 +129,7 @@ def test_text_to_audio_002(omni_server, openai_client, ref_audio_data_url) -> No "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "response_format": "pcm", "ref_audio": ref_audio_data_url, "min_hnr_db": -5.0, diff --git a/tests/e2e/online_serving/test_qwen3_tts_base.py b/tests/e2e/online_serving/test_qwen3_tts_base.py index 5bacd7ced..60caf8f82 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_base.py +++ b/tests/e2e/online_serving/test_qwen3_tts_base.py @@ -100,6 +100,7 @@ def test_text_to_audio_002(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, "response_format": "wav", "task_type": "Base", diff --git a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py index 866d7483c..0edaa4668 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py @@ -80,6 +80,7 @@ def test_voice_clone_streaming_001(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "response_format": "wav", "task_type": "Base", "voice": "clone", diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py index 50494fb72..2aab84840 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py @@ -95,6 +95,7 @@ def test_text_to_audio_002(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "response_format": "wav", "task_type": "CustomVoice", "voice": "vivian", diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py index ce3bec928..9c941cad8 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py @@ -75,6 +75,7 @@ def test_voice_001(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "response_format": "wav", "task_type": "CustomVoice", "voice": "uncle_fu", diff --git a/tests/e2e/online_serving/test_voxcpm2_tts.py b/tests/e2e/online_serving/test_voxcpm2_tts.py index cf493bc14..19ddc0401 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts.py @@ -89,6 +89,7 @@ def test_text_to_audio_002(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, "response_format": "wav", "voice": "default", diff --git a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py index d09570811..7419205c5 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py @@ -62,6 +62,7 @@ def test_voice_clone_streaming_001(omni_server, openai_client) -> None: "model": omni_server.model, "input": get_prompt(), "stream": True, + "stream_format": "audio", "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, "response_format": "wav", "voice": "default", diff --git a/tests/e2e/online_serving/test_voxtral_tts_expansion.py b/tests/e2e/online_serving/test_voxtral_tts_expansion.py index 522d18f37..1448567d3 100644 --- a/tests/e2e/online_serving/test_voxtral_tts_expansion.py +++ b/tests/e2e/online_serving/test_voxtral_tts_expansion.py @@ -56,6 +56,7 @@ def test_speech_english_streaming(omni_server, openai_client) -> None: "voice": "casual_female", "language": "English", "stream": True, + "stream_format": "audio", "response_format": "pcm", "timeout": 120.0, "min_audio_bytes": _MIN_AUDIO_BYTES_BASIC, diff --git a/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py b/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py index d82e1a68b..d51ac653a 100644 --- a/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py +++ b/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py @@ -141,6 +141,7 @@ class TestSpeakerEmbeddingBase: "x_vector_only_mode": True, "response_format": "pcm", "stream": True, + "stream_format": "audio", "max_new_tokens": MAX_NEW_TOKENS, } with httpx.Client(timeout=120.0) as client: diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index a2e31dcd7..c3e4bd960 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -1849,17 +1849,22 @@ class TestStreamingProtocolValidation: OpenAICreateSpeechRequest(input="Hello", stream_format="audio", response_format="pcm", speed=2.0) def test_stream_valid(self): - """stream=True + response_format in ('pcm', 'wav') + speed=1.0 is accepted.""" + """stream=True + response_format in ('pcm', 'wav') + speed=1.0 is accepted as SSE.""" req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm") assert req.stream is True + assert req.is_sse_stream() is True + assert req.is_raw_audio_stream() is False req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="wav") assert req.stream is True + assert req.is_sse_stream() is True + assert req.is_raw_audio_stream() is False def test_stream_format_audio_is_valid(self): req = OpenAICreateSpeechRequest(input="Hello", stream_format="audio", response_format="pcm") assert req.stream_format == "audio" assert req.is_raw_audio_stream() is True + assert req.is_sse_stream() is False def test_sse_stream_format_is_valid(self): """stream_format='sse' is accepted for /audio/speech.""" @@ -1867,6 +1872,7 @@ class TestStreamingProtocolValidation: assert req.stream_format == "sse" assert req.is_sse_stream() is True + assert req.is_raw_audio_stream() is False class TestStreamingResponse: @@ -1939,13 +1945,24 @@ class TestStreamingResponse: app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None) return app + @staticmethod + def _assert_sse_audio_response(response, response_format: str = "pcm"): + assert response.status_code == 200 + assert "text/event-stream" in response.headers["content-type"] + body = response.text + assert "event: speech.audio.delta" in body + assert "event: speech.audio.done" in body + data_line = next(line for line in body.splitlines() if line.startswith("data: ")) + payload = json.loads(data_line.removeprefix("data: ")) + assert payload["type"] == "speech.audio.delta" + assert payload["response_format"] == response_format + assert base64.b64decode(payload["audio"]) + def test_streaming(self, streaming_app): - """stream=True must return audio/pcm with non-empty body.""" + """stream=True defaults to OpenAI speech.audio.* SSE events.""" client = TestClient(streaming_app) response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": "pcm"}) - assert response.status_code == 200 - assert "audio/pcm" in response.headers["content-type"] - assert len(response.content) > 0 + self._assert_sse_audio_response(response) def test_stream_format_audio_streaming(self, streaming_app): """stream_format=audio without stream=True returns raw audio/pcm chunks.""" @@ -1967,25 +1984,26 @@ class TestStreamingResponse: json={"input": "Hello", "stream_format": "sse", "response_format": "pcm"}, ) - assert response.status_code == 200 - assert "text/event-stream" in response.headers["content-type"] - body = response.text - assert "event: speech.audio.delta" in body - assert "event: speech.audio.done" in body - data_line = next(line for line in body.splitlines() if line.startswith("data: ")) - payload = json.loads(data_line.removeprefix("data: ")) - assert payload["type"] == "speech.audio.delta" - assert payload["response_format"] == "pcm" - assert base64.b64decode(payload["audio"]) + self._assert_sse_audio_response(response) - def test_stream_true_prefers_raw_audio_over_sse(self, streaming_app): - """stream=True keeps the existing raw audio stream behavior even with stream_format=sse.""" + def test_stream_true_with_stream_format_sse_uses_sse(self, streaming_app): + """stream=True and stream_format=sse both select SSE streaming.""" client = TestClient(streaming_app) response = client.post( "/v1/audio/speech", json={"input": "Hello", "stream": True, "stream_format": "sse", "response_format": "pcm"}, ) + self._assert_sse_audio_response(response) + + def test_stream_format_audio_with_stream_true_opts_into_raw_audio(self, streaming_app): + """stream_format=audio remains an explicit raw audio opt-in.""" + client = TestClient(streaming_app) + response = client.post( + "/v1/audio/speech", + json={"input": "Hello", "stream": True, "stream_format": "audio", "response_format": "pcm"}, + ) + assert response.status_code == 200 assert "audio/pcm" in response.headers["content-type"] assert "text/event-stream" not in response.headers["content-type"] @@ -3021,7 +3039,10 @@ class TestWAVStreaming: def test_wav_streaming_success(self, wav_streaming_app): """Test WAV format streaming returns correct content type and includes WAV header.""" client = TestClient(wav_streaming_app) - response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": "wav"}) + response = client.post( + "/v1/audio/speech", + json={"input": "Hello", "stream": True, "stream_format": "audio", "response_format": "wav"}, + ) assert response.status_code == 200 assert "audio/wav" in response.headers["content-type"] @@ -3040,7 +3061,10 @@ class TestWAVStreaming: unsupported_formats = ["mp3"] for fmt in unsupported_formats: - response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": fmt}) + response = client.post( + "/v1/audio/speech", + json={"input": "Hello", "stream": True, "stream_format": "audio", "response_format": fmt}, + ) assert response.status_code == 422 @@ -3336,6 +3360,23 @@ class TestTTSAsyncOffloading: mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, True) + def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): + """stream=True should request delta-style multimodal outputs for SSE streaming.""" + qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._build_tts_params = mocker.MagicMock( + return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} + ) + qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + mock_coerce = mocker.patch( + "vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", + return_value=qwen3_tts_server.engine_client.default_sampling_params_list, + ) + request = OpenAICreateSpeechRequest(input="hello", stream=True, response_format="pcm") + + asyncio.run(qwen3_tts_server._prepare_speech_generation(request)) + + mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, True) + def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=audio should request delta-style multimodal outputs.""" qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 89ed91e5f..51ee981cb 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -1770,6 +1770,7 @@ class OpenAIClientHandler: "seed", "instructions", "speed", + "stream_format", ): if key in request_config: extra_body[key] = request_config[key] diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index 091240690..0480e2dc9 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -1133,9 +1133,9 @@ async def async_request_openai_audio_speech( ) -> MixRequestFuncOutput: """Streaming request to /v1/audio/speech endpoint. - Sends ``stream=true`` with ``response_format=pcm`` so the server returns - raw PCM chunks as they are decoded. This allows measuring TTFP (time to - first audio packet) separately from E2EL. + Sends ``stream=true`` with ``stream_format=audio`` and ``response_format=pcm`` + so the server returns raw PCM chunks as they are decoded. This allows measuring + TTFP (time to first audio packet) separately from E2EL. """ api_url = request_func_input.api_url _validate_api_url(api_url, "OpenAI Audio Speech API", "audio/speech") @@ -1144,12 +1144,14 @@ async def async_request_openai_audio_speech( "model": request_func_input.model_name if request_func_input.model_name else request_func_input.model, "input": request_func_input.prompt, "stream": True, + "stream_format": "audio", "response_format": "pcm", } _update_payload_common(payload, request_func_input) # Seed-TTS + WER: ``--extra-body`` may set stream=false / other formats; speech must stream PCM. if getattr(request_func_input, "seed_tts_row", False) and _seed_tts_capture_pcm_for_wer(): payload["stream"] = True + payload["stream_format"] = "audio" payload["response_format"] = "pcm" headers = { @@ -1219,7 +1221,7 @@ async def async_request_openai_audio_speech( ct = response.headers.get("Content-Type", "") logger.warning( "Seed-TTS WER: HTTP 200 but no PCM bytes (Content-Type=%r, url=%s). " - "Check stream=true and response_format=pcm on the server.", + "Check stream=true, stream_format=audio, and response_format=pcm on the server.", ct, api_url, ) diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 48a646edd..ad7bf8fae 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -69,13 +69,15 @@ class OpenAICreateSpeechRequest(BaseModel): default=None, description=( "Streaming output format. 'audio' streams raw pcm/wav bytes; " - "'sse' streams OpenAI speech.audio.* SSE events. Omit for non-streaming." + "'sse' streams OpenAI speech.audio.* SSE events. If omitted, stream=true " + "selects SSE and stream=false remains non-streaming." ), ) stream: bool = Field( default=False, description=( - "Legacy streaming switch; equivalent to stream_format='audio'. " + "Streaming switch; defaults to OpenAI speech.audio.* SSE events. " + "Set stream_format='audio' to opt into raw pcm/wav byte streaming. " "Requires response_format='pcm' or 'wav'. Speed adjustment is not supported when streaming." ), ) @@ -292,13 +294,13 @@ class OpenAICreateSpeechRequest(BaseModel): return self def is_raw_audio_stream(self) -> bool: - return self.stream or self.stream_format == "audio" + return self.stream_format == "audio" def is_sse_stream(self) -> bool: - return self.stream_format == "sse" and not self.is_raw_audio_stream() + return (self.stream or self.stream_format == "sse") and not self.is_raw_audio_stream() def is_streaming(self) -> bool: - return self.is_raw_audio_stream() or self.stream_format == "sse" + return self.is_raw_audio_stream() or self.is_sse_stream() @model_validator(mode="after") def validate_streaming_constraints(self) -> "OpenAICreateSpeechRequest": diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 21555ee7a..ac37a63a2 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -4038,11 +4038,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): - ref_text: Transcript of reference audio (Base task) - x_vector_only_mode: Use speaker embedding only (Base task) - Streaming is supported via ``stream_format='audio'`` or the legacy - ``stream=True`` switch, with ``response_format='pcm'`` or ``'wav'``. - ``stream_format='sse'`` returns OpenAI ``speech.audio.*`` SSE events instead. - Each Code2Wav chunk is yielded as raw audio bytes as soon as it is decoded. - For WAV format, a header with placeholder size values is emitted first. + Streaming is supported via the ``stream=True`` switch or ``stream_format='sse'``, + which return OpenAI ``speech.audio.*`` SSE events. ``stream_format='audio'`` + opts into raw audio streaming with ``response_format='pcm'`` or ``'wav'``. + Raw audio streaming yields each Code2Wav chunk as raw bytes as soon as it is + decoded. Raw WAV streaming emits a header with placeholder size values first. """ if self._diffusion_mode: return await self._create_diffusion_speech(request) @@ -4060,14 +4060,14 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ) try: - if request.is_raw_audio_stream(): - if request.word_timestamps: - return self.create_error_response( - "word_timestamps=true is currently supported by the WebSocket " - "/v1/audio/speech/stream path. Use session.config with " - "stream_audio=true and response_format='pcm'." - ) + if request.is_streaming() and request.word_timestamps: + return self.create_error_response( + "word_timestamps=true is currently supported by the WebSocket " + "/v1/audio/speech/stream path. Use session.config with " + "stream_audio=true and response_format='pcm'." + ) + if request.is_raw_audio_stream(): response_format, error = self._validate_speech_streaming_request( request, mode_label="Streaming",