From 49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd Mon Sep 17 00:00:00 2001 From: AuFlow <73925903+AuFlow@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:25:09 +0800 Subject: [PATCH] [Frontend] Add configurable output sample rate to Speech API (#6553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 郑倩倩 <2458983186@qq.com> Co-authored-by: Hongsheng Liu --- docs/serving/speech_api.md | 3 +- .../qwen3_tts/openai_speech_client.py | 12 ++ .../test_qwen3_tts_customvoice_expansion.py | 17 +++ .../openai_api/test_audio_format.py | 83 +++++++++++- .../openai_api/test_serving_speech.py | 124 ++++++++++++++++++ tests/helpers/assertions.py | 7 + tests/helpers/client.py | 1 + .../entrypoints/openai/audio_utils_mixin.py | 102 ++++++++++++++ .../entrypoints/openai/protocol/audio.py | 10 +- .../entrypoints/openai/serving_speech.py | 104 +++++++++++++-- .../entrypoints/openai/tts_adapters/base.py | 3 + .../openai/tts_adapters/qwen3_tts.py | 1 + 12 files changed, 453 insertions(+), 14 deletions(-) diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 98fb7f380..0527ab443 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -2,7 +2,7 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. Supported TTS models include: -- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz. +- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Native output: 24 kHz; API output can be resampled to 8 kHz. - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz. - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz. - **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz. @@ -116,6 +116,7 @@ Content-Type: application/json | Parameter | Type | Default | Description | |-----------|------|---------|-------------| +| `sample_rate` | integer | model native | Target output sample rate. Qwen3-TTS supports 8000 or 24000 Hz; the model remains native 24 kHz internally and is resampled before encoding. | | `task_type` | string | "CustomVoice" | TTS task type: CustomVoice, VoiceDesign, or Base | | `language` | string | "Auto" | Language (see supported languages below) | | `instructions` | string | "" | Voice style/emotion instructions | diff --git a/examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py b/examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py index 77e13b08e..7b2fecef5 100644 --- a/examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py +++ b/examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech endpoint. This script demonstrates how to use the OpenAI-compatible speech API @@ -74,6 +77,8 @@ def run_tts_generation(args) -> None: "voice": args.speaker, "response_format": args.response_format, } + if args.sample_rate is not None: + payload["sample_rate"] = args.sample_rate # Add optional parameters if args.instructions: @@ -246,6 +251,13 @@ def parse_args(): choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], help="Audio output format (default: wav)", ) + parser.add_argument( + "--sample-rate", + type=int, + default=None, + choices=[8000, 24000], + help="Output sample rate in Hz (default: model native)", + ) parser.add_argument( "--output", "-o", diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py index 07808af27..15198870e 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py @@ -138,6 +138,23 @@ def test_voice_003(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config) +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_sample_rate_001(omni_server, online_client) -> None: + """The speech API resamples Qwen3-TTS output from native 24 kHz to 8 kHz.""" + request_config = { + "model": omni_server.model, + "input": "This response should be encoded as eight kilohertz audio.", + "stream": False, + "response_format": "wav", + "sample_rate": 8000, + "task_type": "CustomVoice", + "voice": "vivian", + } + + online_client.send_audio_speech_request(request_config) + + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_language_001(omni_server, online_client) -> None: diff --git a/tests/entrypoints/openai_api/test_audio_format.py b/tests/entrypoints/openai_api/test_audio_format.py index 2f1f487d9..e47885d9b 100644 --- a/tests/entrypoints/openai_api/test_audio_format.py +++ b/tests/entrypoints/openai_api/test_audio_format.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Tests for audio output format handling in chat completions. Covers: @@ -11,10 +12,15 @@ Covers: from __future__ import annotations +from io import BytesIO + import numpy as np import pytest +import soundfile +import torch +import torchaudio -from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin +from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler from vllm_omni.entrypoints.openai.protocol.audio import ( DEFAULT_AUDIO_FORMAT, SUPPORTED_AUDIO_FORMATS, @@ -122,6 +128,81 @@ class TestCreateAudio: decoded = base64.b64decode(response.audio_data) assert decoded[:4] == b"RIFF" + def test_resamples_wav_to_requested_output_rate(self, mixin, audio_tensor): + response = mixin.create_audio( + CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + output_sample_rate=8000, + response_format="wav", + speed=1.0, + base64_encode=False, + ) + ) + + with soundfile.SoundFile(BytesIO(response.audio_data)) as audio_file: + assert audio_file.samplerate == 8000 + assert audio_file.frames == 8000 + + +class TestStreamingAudioResampler: + @staticmethod + def _resample(waveform): + resampler = StreamingAudioResampler(24000, 8000) + return np.concatenate( + ( + resampler.process(waveform), + resampler.process(np.empty(0), final=True), + ) + ) + + def test_chunk_boundaries_do_not_change_output(self): + waveform = np.sin(np.linspace(0, 200 * np.pi, 24000, endpoint=False)).astype(np.float32) + + expected = self._resample(waveform) + + chunked = StreamingAudioResampler(24000, 8000) + pieces = [chunked.process(chunk) for chunk in np.split(waveform, [137, 2048, 9001, 17003])] + pieces.append(chunked.process(np.empty(0), final=True)) + actual = np.concatenate(pieces) + + assert actual.shape == (8000,) + np.testing.assert_allclose(actual, expected, atol=1e-6) + + reference = torchaudio.functional.resample(torch.from_numpy(waveform), 24000, 8000).numpy() + np.testing.assert_allclose(actual[100:-100], reference[100:-100], atol=2e-3, rtol=1e-3) + + def test_preserves_passband_signal(self): + samples = np.arange(24000, dtype=np.float32) + waveform = np.sin(2 * np.pi * 3000 * samples / 24000).astype(np.float32) + + output = self._resample(waveform) + + input_rms = np.sqrt(np.mean(waveform**2)) + output_rms = np.sqrt(np.mean(output[100:-100] ** 2)) + assert output_rms == pytest.approx(input_rms, rel=0.02) + + @pytest.mark.parametrize("frequency", [6000, 10000]) + def test_attenuates_aliasing_frequencies(self, frequency): + samples = np.arange(24000, dtype=np.float32) + waveform = np.sin(2 * np.pi * frequency * samples / 24000).astype(np.float32) + + output = self._resample(waveform) + + input_rms = np.sqrt(np.mean(waveform**2)) + output_rms = np.sqrt(np.mean(output[100:-100] ** 2)) + assert output_rms < input_rms * 0.01 + + def test_rejects_non_integer_downsampling_ratio(self): + with pytest.raises(ValueError, match="integer downsampling ratio"): + StreamingAudioResampler(24000, 16000) + + def test_rejects_multichannel_audio(self): + resampler = StreamingAudioResampler(24000, 8000) + + with pytest.raises(ValueError, match="only supports mono audio"): + resampler.process(np.zeros((2, 240), dtype=np.float32)) + class TestResolveAudioFormat: """Test _resolve_audio_format via the serving chat class.""" diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 19d383504..4852f678d 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -2953,8 +2953,10 @@ class TestStreamingResponse: usage_acc=None, tts_params=None, collect=None, + target_sample_rate=None, ): captured["tts_params"] = tts_params + captured["target_sample_rate"] = target_sample_rate yield b"\x00\x00" monkeypatch.setattr(speech_server, "_prepare_speech_generation", prepare) @@ -2967,6 +2969,7 @@ class TestStreamingResponse: assert response.status_code == 200 assert captured["tts_params"] is finalized_tts_params + assert captured["target_sample_rate"] is None def test_sse_streaming(self, streaming_app): """stream_format=sse without stream=True returns audio deltas as SSE.""" @@ -3292,6 +3295,28 @@ class TestMergeBatchItem: assert merged.non_streaming_mode is False + def test_sample_rate_batch_default_used(self): + """Batch-level sample_rate should be used when the item omits it.""" + batch = BatchSpeechRequest( + items=[SpeechBatchItem(input="hi")], + sample_rate=8000, + ) + + merged = OmniOpenAIServingSpeech._merge_batch_item(batch, batch.items[0]) + + assert merged.sample_rate == 8000 + + def test_sample_rate_item_override_wins(self): + """Per-item sample_rate should override the batch-level default.""" + batch = BatchSpeechRequest( + items=[SpeechBatchItem(input="hi", sample_rate=24000)], + sample_rate=8000, + ) + + merged = OmniOpenAIServingSpeech._merge_batch_item(batch, batch.items[0]) + + assert merged.sample_rate == 24000 + def test_streaming_speech_session_config_accepts_non_streaming_mode(): config = StreamingSpeechSessionConfig(non_streaming_mode=True) @@ -3299,6 +3324,33 @@ def test_streaming_speech_session_config_accepts_non_streaming_mode(): assert config.non_streaming_mode is True +@pytest.mark.parametrize("sample_rate", [8000, 24000]) +def test_speech_requests_accept_supported_sample_rates(sample_rate): + assert OpenAICreateSpeechRequest(input="hello", sample_rate=sample_rate).sample_rate == sample_rate + + +@pytest.mark.parametrize("sample_rate", [0, -8000]) +def test_speech_request_rejects_non_positive_sample_rate(sample_rate): + with pytest.raises(ValidationError): + OpenAICreateSpeechRequest(input="hello", sample_rate=sample_rate) + + +def test_sample_rate_uses_adapter_capabilities(): + server = object.__new__(OmniOpenAIServingSpeech) + request = OpenAICreateSpeechRequest(input="hello", sample_rate=8000) + + server._tts_model_type = "qwen3_tts" + assert server._validate_speech_sample_rate(request) is None + + request.sample_rate = 16000 + assert server._validate_speech_sample_rate(request) == ( + "sample_rate=16000 is not supported by the current TTS model; supported rates: 8000, 24000" + ) + + server._tts_model_type = "fish_speech" + assert server._validate_speech_sample_rate(request) == ("sample_rate is not supported by the current TTS model") + + def test_streaming_speech_session_config_scopes_native_speed_to_http(): with pytest.raises( ValidationError, @@ -5027,6 +5079,78 @@ class TestTTSAsyncOffloading: assert "partial_audio" not in payload["error"] assert "action" not in payload["error"] + @pytest.mark.asyncio + async def test_generate_audio_chunks_resamples_pcm_to_8khz(self, qwen3_tts_server): + async def pcm_generator(): + output = create_mock_audio_output_for_test() + output.multimodal_output.update( + audio=torch.linspace(-0.5, 0.5, 2400, dtype=torch.float32), + sr=24000, + ) + yield output + + chunks = [ + chunk + async for chunk in qwen3_tts_server._generate_audio_chunks( + pcm_generator(), + "req-8khz", + target_sample_rate=8000, + ) + ] + + assert len(chunks) >= 2 + assert len(b"".join(chunks)) == 800 * 2 + + @pytest.mark.asyncio + async def test_generate_audio_chunks_wav_header_uses_target_sample_rate(self, qwen3_tts_server): + async def pcm_generator(): + output = create_mock_audio_output_for_test() + output.multimodal_output.update( + audio=torch.zeros(2400, dtype=torch.float32), + sr=24000, + ) + yield output + + chunks = [ + chunk + async for chunk in qwen3_tts_server._generate_audio_chunks( + pcm_generator(), + "req-8khz-wav", + response_format="wav", + target_sample_rate=8000, + ) + ] + + assert struct.unpack(" np.ndarray: + chunk = np.asarray(audio, dtype=np.float32) + if chunk.ndim != 1: + raise ValueError(f"Streaming audio resampling only supports mono audio, got shape {chunk.shape}") + + if self._ratio == 1: + return chunk + + if chunk.size: + self._buffer = np.concatenate((self._buffer, chunk)) + self._total_samples += int(chunk.size) + + half = self._taps.size // 2 + max_center = self._total_samples - 1 if final else self._total_samples - 1 - half + if self._next_center > max_center: + return np.empty((0,), dtype=np.float32) + + centers = np.arange(self._next_center, max_center + 1, self._ratio, dtype=np.int64) + first_center = int(centers[0]) + last_center = int(centers[-1]) + window_start = first_center - half + window_end = last_center + half + segment = np.zeros((window_end - window_start + 1,), dtype=np.float32) + + copy_start = max(window_start, self._buffer_start, 0) + copy_end = min(window_end + 1, self._buffer_start + self._buffer.size) + if copy_end > copy_start: + dst_start = copy_start - window_start + src_start = copy_start - self._buffer_start + segment[dst_start : dst_start + copy_end - copy_start] = self._buffer[ + src_start : src_start + copy_end - copy_start + ] + + windows = np.lib.stride_tricks.sliding_window_view(segment, self._taps.size)[:: self._ratio] + output = windows @ self._taps[::-1] + self._next_center = last_center + self._ratio + + keep_from = max(0, self._next_center - half) + drop = min(max(keep_from - self._buffer_start, 0), self._buffer.size) + if drop: + self._buffer = self._buffer[drop:] + self._buffer_start += int(drop) + + return np.asarray(output, dtype=np.float32) + + class AudioMixin: """Mixin class to add audio-related utilities.""" @@ -44,6 +129,10 @@ class AudioMixin: audio_tensor, sample_rate = self._apply_speed_adjustment(audio_tensor, speed, sample_rate) + if audio_obj.output_sample_rate is not None and audio_obj.output_sample_rate != sample_rate: + audio_tensor = self._resample_audio(audio_tensor, sample_rate, audio_obj.output_sample_rate) + sample_rate = audio_obj.output_sample_rate + supported_formats = { "wav": ("WAV", "audio/wav", {}), "pcm": ("RAW", "audio/pcm", {"subtype": "PCM_16"}), @@ -69,6 +158,19 @@ class AudioMixin: return AudioResponse(audio_data=audio_data, media_type=media_type) + @staticmethod + def _resample_audio(audio_tensor: np.ndarray, source_rate: int, target_rate: int) -> np.ndarray: + """Resample complete audio while preserving soundfile's channels-last layout.""" + if source_rate == target_rate: + return audio_tensor + + audio_array = np.asarray(audio_tensor) + if not np.issubdtype(audio_array.dtype, np.floating): + audio_array = audio_array.astype(np.float32) + waveform = torch.from_numpy(audio_array.T.copy() if audio_array.ndim == 2 else audio_array.copy()) + resampled = torchaudio.functional.resample(waveform, source_rate, target_rate).cpu().numpy() + return resampled.T if audio_array.ndim == 2 else resampled + def _apply_speed_adjustment(self, audio_tensor: np.ndarray, speed: float, sample_rate: int): """Apply speed adjustment to the audio tensor while preserving pitch. diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 1c556fc79..f204ae209 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import math -from typing import Any, Literal +from typing import Annotated, Any, Literal import numpy as np from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator @@ -15,6 +15,7 @@ _MAX_EMBEDDING_DIM = 8192 SUPPORTED_AUDIO_FORMATS: frozenset[str] = frozenset({"wav", "pcm", "flac", "mp3", "opus"}) SUPPORTED_CHAT_AUDIO_FORMATS: frozenset[str] = SUPPORTED_AUDIO_FORMATS | {"pcm16"} DEFAULT_AUDIO_FORMAT: str = "wav" +SpeechSampleRate = Annotated[int, Field(gt=0)] def _normalize_ref_audio_value(value): @@ -80,6 +81,10 @@ class OpenAICreateSpeechRequest(BaseModel): description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)", ) response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT + sample_rate: SpeechSampleRate | None = Field( + default=None, + description="Target sample rate of the returned audio. If omitted, use the model's native sample rate.", + ) speed: float | None = Field( default=1.0, ge=0.25, @@ -396,6 +401,7 @@ class OpenAICreateAudioGenerateRequest(BaseModel): class CreateAudio(BaseModel): audio_tensor: np.ndarray sample_rate: int = 24000 + output_sample_rate: int | None = None response_format: str = "wav" speed: float = 1.0 base64_encode: bool = True @@ -420,6 +426,7 @@ class SpeechBatchItem(BaseModel): voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker")) instructions: str | None = None response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] | None = None + sample_rate: SpeechSampleRate | None = None speed: float | None = Field(default=None, ge=0.25, le=4.0) task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None @@ -440,6 +447,7 @@ class BatchSpeechRequest(BaseModel): voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker")) instructions: str | None = None response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT + sample_rate: SpeechSampleRate | None = None speed: float | None = Field(default=1.0, ge=0.25, le=4.0) task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 9ed81a862..224485cbd 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -37,7 +37,7 @@ from vllm.utils import random_uuid from vllm.utils.async_utils import make_async from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError -from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin +from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler from vllm_omni.entrypoints.openai.protocol.audio import ( AudioResponse, BatchSpeechRequest, @@ -1296,6 +1296,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: """Validate TTS request parameters. Returns error message or None.""" + sample_rate_error = self._validate_speech_sample_rate(request) + if sample_rate_error is not None: + return sample_rate_error + adapter = self._get_tts_adapter() if adapter is not None: return adapter.validate(request) @@ -1310,6 +1314,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ) return adapter_cls(ctx).validate(request) + def _validate_speech_sample_rate(self, request: OpenAICreateSpeechRequest) -> str | None: + if request.sample_rate is None: + return None + + adapter_cls = resolve_adapter(self._tts_model_type) + supported_rates = adapter_cls.supported_output_sample_rates if adapter_cls is not None else frozenset() + if request.sample_rate not in supported_rates: + if supported_rates: + rates = ", ".join(str(rate) for rate in sorted(supported_rates)) + return ( + f"sample_rate={request.sample_rate} is not supported by the current TTS model; " + f"supported rates: {rates}" + ) + return "sample_rate is not supported by the current TTS model" + return None + def _voxcpm2_encode(self, text: str) -> list[int]: """Tokenize text for VoxCPM2, splitting multichar Chinese tokens.""" from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import ( @@ -2191,6 +2211,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): usage_acc: SpeechOutputTokenCounter | None = None, tts_params: dict[str, Any] | None = None, collect: dict | None = None, + target_sample_rate: int | None = None, ): """Generate audio chunks for streaming response. @@ -2214,6 +2235,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): first_audio_chunk_s: float | None = None stream_start_s = request_start_s if request_start_s is not None else time.perf_counter() artifact_ready = False + source_sample_rate: int | None = None + resampler: StreamingAudioResampler | None = None # SSE supplies an accumulator for usage output. Raw-audio and WebSocket # streams retain terminal metrics only when their model adapter needs @@ -2239,7 +2262,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if sr_raw is not None: sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw sample_rate_val = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) - + if source_sample_rate is not None and sample_rate_val != source_sample_rate: + raise ValueError( + "Audio sample rate changed during streaming: " + f"{source_sample_rate} Hz to {sample_rate_val} Hz" + ) audio_val = audio_output[audio_key] if isinstance(audio_val, list): # Cumulative mode: each update grows the list; emit only new tail. @@ -2253,12 +2280,25 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): else: new_chunks = [] + if new_chunks and source_sample_rate is None: + if response_format == "wav" and sr_raw is None: + raise ValueError("First audio output must include sample rate metadata for WAV streaming") + source_sample_rate = sample_rate_val + if target_sample_rate is not None and target_sample_rate != source_sample_rate: + resampler = StreamingAudioResampler(source_sample_rate, target_sample_rate) + + output_sample_rate = target_sample_rate or sample_rate_val + for chunk_tensor in new_chunks: chunk_np = ( chunk_tensor.float().detach().cpu().numpy() if hasattr(chunk_tensor, "float") else chunk_tensor ) if chunk_np.ndim > 1: chunk_np = chunk_np.squeeze() + if resampler is not None: + chunk_np = resampler.process(chunk_np) + if chunk_np.size == 0: + continue if self._tts_model_type in _AUDEX_NO_AUDIO_GUARD_MODEL_TYPES and int(np.size(chunk_np)) == 0: # Zero-size chunks must not emit a WAV header or count # as first audio; the post-loop guard below needs to @@ -2266,14 +2306,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): continue # For WAV format, emit header before first audio chunk if response_format == "wav" and first_chunk: - # Assert that sample rate has been set from chunk metadata (not just default) - # This ensures the WAV header contains the correct sample rate - assert sr_raw is not None, ( - "First audio chunk must include sample rate metadata for WAV streaming" - ) num_channels = _infer_audio_num_channels(np.asarray(chunk_np)) wav_header = _create_wav_header( - sample_rate=sample_rate_val, + sample_rate=output_sample_rate, num_channels=num_channels, bits_per_sample=16, ) @@ -2283,7 +2318,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Convert audio to PCM bytes audio_obj = CreateAudio( audio_tensor=chunk_np, - sample_rate=sample_rate_val, + sample_rate=output_sample_rate, response_format="pcm", speed=1.0, base64_encode=False, @@ -2292,7 +2327,33 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): first_audio_chunk_s = time.perf_counter() audio_bytes = self.create_audio(audio_obj).audio_data if include_sample_rate: - yield audio_bytes, sample_rate_val + yield audio_bytes, output_sample_rate + else: + yield audio_bytes + + if resampler is not None: + final_chunk = resampler.process(np.empty((0,), dtype=np.float32), final=True) + if final_chunk.size: + output_sample_rate = target_sample_rate or sample_rate_val + if response_format == "wav" and first_chunk: + yield _create_wav_header( + sample_rate=output_sample_rate, + num_channels=1, + bits_per_sample=16, + ) + first_chunk = False + audio_obj = CreateAudio( + audio_tensor=final_chunk, + sample_rate=output_sample_rate, + response_format="pcm", + speed=1.0, + base64_encode=False, + ) + if first_audio_chunk_s is None: + first_audio_chunk_s = time.perf_counter() + audio_bytes = self.create_audio(audio_obj).audio_data + if include_sample_rate: + yield audio_bytes, output_sample_rate else: yield audio_bytes if self._tts_model_type in _AUDEX_NO_AUDIO_GUARD_MODEL_TYPES and first_audio_chunk_s is None: @@ -2398,6 +2459,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request_start_s=request_start_s, usage_acc=usage_acc, tts_params=tts_params, + target_sample_rate=request.sample_rate if request is not None else None, ): payload = { "type": "speech.audio.delta", @@ -3003,6 +3065,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if self.engine_client.errored: raise self.engine_client.dead_error + sample_rate_error = self._validate_speech_sample_rate(request) + if sample_rate_error is not None: + raise ValueError(sample_rate_error) + request_id = request_id or f"speech-{random_uuid()}" qwen3_ref_audio_warmup_artifact_key: str | None = None @@ -3151,6 +3217,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): include_sample_rate: bool = False, tts_params: dict[str, Any] | None = None, collect: dict | None = None, + target_sample_rate: int | None = None, ): """Yield raw PCM byte chunks from the engine generator. @@ -3166,6 +3233,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): include_sample_rate=include_sample_rate, tts_params=tts_params, collect=collect, + target_sample_rate=target_sample_rate, ): yield chunk @@ -3173,7 +3241,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): """Yield raw PCM bytes for a speech request as soon as chunks are decoded.""" request_id, generator, tts_params = await self._prepare_speech_generation(request) try: - async for chunk in self._generate_pcm_chunks(generator, request_id, tts_params=tts_params): + async for chunk in self._generate_pcm_chunks( + generator, + request_id, + tts_params=tts_params, + target_sample_rate=request.sample_rate, + ): yield chunk finally: self._discard_ref_audio_artifact_warmup(request_id) @@ -3323,6 +3396,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): audio_obj = CreateAudio( audio_tensor=audio_tensor, sample_rate=sample_rate, + output_sample_rate=request.sample_rate, response_format=request.response_format or "wav", speed=self._audio_encode_speed(request), base64_encode=base64_encode, @@ -3453,6 +3527,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): audio_obj = CreateAudio( audio_tensor=audio_tensor, sample_rate=sample_rate, + output_sample_rate=request.sample_rate, response_format=request.response_format or "wav", speed=self._audio_encode_speed(request), base64_encode=False, @@ -3523,6 +3598,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): - ref_audio: Reference audio for voice cloning (Base task) - ref_text: Transcript of reference audio (Base task) - x_vector_only_mode: Use speaker embedding only (Base task) + - sample_rate: Target output sample rate (8000 or 24000 Hz) Streaming is supported via the ``stream=True`` switch or ``stream_format='sse'``, which return OpenAI ``speech.audio.*`` SSE events. ``stream_format='audio'`` @@ -3534,6 +3610,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if _is_default_voice(request.voice.lower(), self._get_available_speakers()): request.voice = None + sample_rate_error = self._validate_speech_sample_rate(request) + if sample_rate_error is not None: + return self.create_error_response(sample_rate_error) + if self._diffusion_mode: return await self._create_diffusion_speech(request) @@ -3580,6 +3660,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): raw_request=raw_request, request_start_s=request_start_s, tts_params=raw_tts_params, + target_sample_rate=request.sample_rate, ), media_type=media_type, ) @@ -3730,6 +3811,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): voice=_pick("voice"), instructions=_pick("instructions"), response_format=_pick("response_format") or "wav", + sample_rate=_pick("sample_rate"), speed=picked_speed if picked_speed is not None else 1.0, stream=False, task_type=_pick("task_type"), diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index 4842acfd7..d4316a2fe 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -166,6 +166,9 @@ class TTSModelAdapter(ABC): validates_generation: ClassVar[bool] = False #: Whether the model consumes ``request.speed`` in its native parameters. native_speed_control: ClassVar[bool] = False + #: Target sample rates validated for this adapter's output path. An empty + #: set means that the adapter does not expose per-request resampling. + supported_output_sample_rates: ClassVar[frozenset[int]] = frozenset() max_new_tokens_min = 1 diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index c2fd4faf8..6e9a89293 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -42,6 +42,7 @@ class Qwen3TTSAdapter(ARTTSAdapter): validates_generation = True stage_keys = frozenset({"qwen3_tts"}) name = "qwen3_tts" + supported_output_sample_rates = frozenset({8000, 24000}) def normalize(self, request: "OpenAICreateSpeechRequest") -> None: """Qwen3-TTS normalization (Base-task inference, voice lowercasing) is