From 68a992060b34e9617da6311416a3eafce61d2214 Mon Sep 17 00:00:00 2001 From: Ricardo Noriega Date: Thu, 16 Jul 2026 08:17:02 +0200 Subject: [PATCH] [Bugfix] Fix audio.format parameter silently ignored in chat completions (#4718) Signed-off-by: Ricardo Noriega De Soto --- docs/serving/audio_generate_api.md | 5 +- docs/serving/speech_api.md | 2 +- .../examples/online_serving/text_to_audio.md | 2 +- .../test_qwen3_omni_expansion.py | 19 ++ .../openai_api/test_audio_format.py | 175 ++++++++++++++++++ .../entrypoints/openai/audio_utils_mixin.py | 7 +- .../entrypoints/openai/protocol/audio.py | 14 +- vllm_omni/entrypoints/openai/serving_chat.py | 39 +++- 8 files changed, 246 insertions(+), 17 deletions(-) create mode 100644 tests/entrypoints/openai_api/test_audio_format.py diff --git a/docs/serving/audio_generate_api.md b/docs/serving/audio_generate_api.md index afe693ae4..3109446c1 100644 --- a/docs/serving/audio_generate_api.md +++ b/docs/serving/audio_generate_api.md @@ -66,7 +66,7 @@ Content-Type: application/json |-----------|------|---------|-------------| | `input` | string | **required** | Text prompt describing the audio to generate | | `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | | `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | #### Diffusion Parameters @@ -90,7 +90,6 @@ Returns binary audio data with the appropriate `Content-Type` header: | `mp3` | `audio/mpeg` | | `flac` | `audio/flac` | | `pcm` | `audio/pcm` | -| `aac` | `audio/aac` | | `opus` | `audio/opus` | ## Examples @@ -285,7 +284,7 @@ Pydantic validation failure (e.g. invalid `response_format`, `speed` out of rang "detail": [ { "type": "literal_error", - "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3', 'aac' or 'opus'", + "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3' or 'opus'", ... } ] diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 27dfda95f..2bf630476 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -109,7 +109,7 @@ Content-Type: application/json | `input` | string | **required** | The text to synthesize into speech | | `model` | string | server's model | Model to use (optional, should match server if specified) | | `voice` | string | "vivian" | Speaker name (e.g., vivian, ryan, aiden) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | | `speed` | float | 1.0 | Playback speed (0.25-4.0) | #### vLLM-Omni Extension Parameters diff --git a/docs/user_guide/examples/online_serving/text_to_audio.md b/docs/user_guide/examples/online_serving/text_to_audio.md index b1ec33a20..a4833aef6 100644 --- a/docs/user_guide/examples/online_serving/text_to_audio.md +++ b/docs/user_guide/examples/online_serving/text_to_audio.md @@ -139,7 +139,7 @@ Content-Type: application/json |-----------|------|---------|-------------| | `input` | string | **required** | Text prompt describing the audio to generate | | `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | | `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | | `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) | | `audio_start` | float | 0.0 | Audio start time in seconds | diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 25ba4f284..3ded285ce 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -7,6 +7,7 @@ E2E Online tests for Qwen3-Omni model. import os import pytest +from openai import BadRequestError from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video @@ -610,3 +611,21 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: text = responses[0].text_content if responses else "" word_count = len(text.split()) assert word_count >= 200, f"Expected at least 200 words in long output, got {word_count}" + + +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params[:1], indirect=True) +def test_invalid_audio_format_rejected(omni_server, openai_client) -> None: + """Ensure invalid audio format is rejected with 400 before streaming starts.""" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text=get_prompt(), + ) + with pytest.raises(BadRequestError, match="audio format"): + openai_client.client.chat.completions.create( + model=omni_server.model, + messages=messages, + modalities=["text", "audio"], + audio={"voice": "alloy", "format": "aac"}, + stream=True, + ) diff --git a/tests/entrypoints/openai_api/test_audio_format.py b/tests/entrypoints/openai_api/test_audio_format.py new file mode 100644 index 000000000..2f1f487d9 --- /dev/null +++ b/tests/entrypoints/openai_api/test_audio_format.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for audio output format handling in chat completions. + +Covers: +- #4716: audio.format parameter must be respected, not hardcoded to WAV +- Format validation rejects unsupported formats +- pcm16 is mapped to pcm for soundfile compatibility +- Default format is WAV when not specified +- create_audio encodes correctly for each supported format +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin +from vllm_omni.entrypoints.openai.protocol.audio import ( + DEFAULT_AUDIO_FORMAT, + SUPPORTED_AUDIO_FORMATS, + SUPPORTED_CHAT_AUDIO_FORMATS, + CreateAudio, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestAudioFormatConstants: + def test_default_format_is_wav(self): + assert DEFAULT_AUDIO_FORMAT == "wav" + + def test_supported_formats_no_aac(self): + assert "aac" not in SUPPORTED_AUDIO_FORMATS + + def test_chat_formats_include_pcm16(self): + assert "pcm16" in SUPPORTED_CHAT_AUDIO_FORMATS + + def test_chat_formats_superset_of_audio_formats(self): + assert SUPPORTED_AUDIO_FORMATS <= SUPPORTED_CHAT_AUDIO_FORMATS + + +class TestCreateAudio: + @pytest.fixture + def mixin(self): + return AudioMixin() + + @pytest.fixture + def audio_tensor(self): + return np.sin(np.linspace(0, 2 * np.pi, 24000)).astype(np.float32) + + @pytest.mark.parametrize("fmt", ["wav", "mp3", "flac", "opus", "pcm"]) + def test_create_audio_supported_formats(self, mixin, audio_tensor, fmt): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format=fmt, + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert len(response.audio_data) > 0 + + def test_wav_magic_bytes(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="wav", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.audio_data[:4] == b"RIFF" + assert response.media_type == "audio/wav" + + def test_mp3_encoding(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="mp3", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.media_type == "audio/mpeg" + assert response.audio_data[:4] != b"RIFF" + + def test_flac_magic_bytes(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="flac", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.audio_data[:4] == b"fLaC" + assert response.media_type == "audio/flac" + + def test_unsupported_format_falls_back_to_default(self, mixin, audio_tensor): + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="aac", + speed=1.0, + base64_encode=False, + ) + response = mixin.create_audio(audio_obj) + assert response.audio_data[:4] == b"RIFF" + assert response.media_type == "audio/wav" + + def test_base64_encoding(self, mixin, audio_tensor): + import base64 + + audio_obj = CreateAudio( + audio_tensor=audio_tensor, + sample_rate=24000, + response_format="wav", + speed=1.0, + base64_encode=True, + ) + response = mixin.create_audio(audio_obj) + decoded = base64.b64decode(response.audio_data) + assert decoded[:4] == b"RIFF" + + +class TestResolveAudioFormat: + """Test _resolve_audio_format via the serving chat class.""" + + @pytest.fixture + def serving_chat(self): + from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat + + return object.__new__(OmniOpenAIServingChat) + + def _make_request(self, audio_params=None): + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + + req = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + ) + if audio_params is not None: + req.audio = audio_params + return req + + def test_default_format_when_no_audio_params(self, serving_chat): + request = self._make_request() + result = serving_chat._resolve_audio_format(request) + assert result == "wav" + + def test_extracts_mp3_format(self, serving_chat): + request = self._make_request({"format": "mp3", "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert result == "mp3" + + def test_pcm16_mapped_to_pcm(self, serving_chat): + request = self._make_request({"format": "pcm16", "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert result == "pcm" + + def test_invalid_format_returns_error(self, serving_chat): + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + request = self._make_request({"format": "aac", "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert isinstance(result, ErrorResponse) + assert "aac" in result.error.message + + def test_all_supported_formats_accepted(self, serving_chat): + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + for fmt in SUPPORTED_CHAT_AUDIO_FORMATS: + request = self._make_request({"format": fmt, "voice": "alloy"}) + result = serving_chat._resolve_audio_format(request) + assert not isinstance(result, ErrorResponse), f"Format {fmt} should be accepted" diff --git a/vllm_omni/entrypoints/openai/audio_utils_mixin.py b/vllm_omni/entrypoints/openai/audio_utils_mixin.py index 12024013d..6c7043201 100644 --- a/vllm_omni/entrypoints/openai/audio_utils_mixin.py +++ b/vllm_omni/entrypoints/openai/audio_utils_mixin.py @@ -5,7 +5,7 @@ import torch import torchaudio from vllm.logger import init_logger -from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio +from vllm_omni.entrypoints.openai.protocol.audio import DEFAULT_AUDIO_FORMAT, AudioResponse, CreateAudio try: import soundfile @@ -49,13 +49,12 @@ class AudioMixin: "pcm": ("RAW", "audio/pcm", {"subtype": "PCM_16"}), "flac": ("FLAC", "audio/flac", {}), "mp3": ("MP3", "audio/mpeg", {}), - "aac": ("AAC", "audio/aac", {}), "opus": ("OGG", "audio/ogg", {"subtype": "OPUS"}), } if response_format not in supported_formats: - logger.warning(f"Unsupported response format '{response_format}', defaulting to 'wav'.") - response_format = "wav" + logger.warning(f"Unsupported response format '{response_format}', defaulting to '{DEFAULT_AUDIO_FORMAT}'.") + response_format = DEFAULT_AUDIO_FORMAT soundfile_format, media_type, kwargs = supported_formats[response_format] diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index ad7bf8fae..92b2efd14 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -6,6 +6,10 @@ from pydantic import AliasChoices, BaseModel, Field, field_validator, model_vali _MAX_EMBEDDING_DIM = 8192 +SUPPORTED_AUDIO_FORMATS: frozenset[str] = frozenset({"wav", "pcm", "flac", "mp3", "opus"}) +SUPPORTED_CHAT_AUDIO_FORMATS: frozenset[str] = SUPPORTED_AUDIO_FORMATS | {"pcm16"} +DEFAULT_AUDIO_FORMAT: str = "wav" + def _normalize_ref_audio_value(value): if value is None: @@ -59,7 +63,7 @@ class OpenAICreateSpeechRequest(BaseModel): default=None, description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)", ) - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field( default=1.0, ge=0.25, @@ -325,7 +329,7 @@ class OpenAICreateAudioGenerateRequest(BaseModel): description="Text prompt describing the audio to generate", ) model: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field( default=1.0, ge=0.25, @@ -391,7 +395,7 @@ class SpeechBatchItem(BaseModel): input: str voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker")) instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] | None = None + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] | None = None speed: float | None = Field(default=None, ge=0.25, le=4.0) task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None @@ -411,7 +415,7 @@ class BatchSpeechRequest(BaseModel): items: list[SpeechBatchItem] = Field(..., min_length=1) voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker")) instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field(default=1.0, ge=0.25, le=4.0) task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None @@ -500,7 +504,7 @@ class StreamingSpeechSessionConfig(BaseModel): task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None language: str | None = None instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" + response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT speed: float | None = Field(default=1.0, ge=0.25, le=4.0) max_new_tokens: int | None = Field(default=None, ge=1) initial_codec_chunk_frames: int | None = Field( diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index a1a51c303..eec6a03b9 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -97,7 +97,12 @@ from vllm.v1.engine.exceptions import EngineDeadError from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin from vllm_omni.entrypoints.openai.image_api_utils import encode_image_base64_with_compression, validate_layered_layers from vllm_omni.entrypoints.openai.protocol import OmniChatCompletionStreamResponse -from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio +from vllm_omni.entrypoints.openai.protocol.audio import ( + DEFAULT_AUDIO_FORMAT, + SUPPORTED_CHAT_AUDIO_FORMATS, + AudioResponse, + CreateAudio, +) from vllm_omni.entrypoints.openai.protocol.images import ( ImageData, ImageEditARDeltaChunk, @@ -450,6 +455,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): output_modalities = getattr(request, "modalities", engine_output_modalities) request.modalities = output_modalities if output_modalities is not None else engine_output_modalities + if request.modalities and "audio" in request.modalities: + audio_format_check = self._resolve_audio_format(request) + if isinstance(audio_format_check, ErrorResponse): + return audio_format_check + num_inference_steps = None extra_body: dict[str, Any] = {} # Omni multistage image generation: Stage-0 (AR) should receive a clean @@ -2561,10 +2571,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): else: sample_rate = int(sr_raw) + audio_format = self._resolve_audio_format(request) + if isinstance(audio_format, ErrorResponse): + return audio_format + audio_obj = CreateAudio( audio_tensor=audio_tensor, sample_rate=sample_rate, - response_format="wav", + response_format=audio_format, speed=1.0, base64_encode=True, ) @@ -3515,10 +3529,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): raise ValueError(f"Unexpected audio tensor rank {audio_tensor.ndim}; expected 1-3 dims.") audio_array = audio_tensor.numpy() + audio_format = self._resolve_audio_format(request) + if isinstance(audio_format, ErrorResponse): + return audio_format + audio_obj = CreateAudio( audio_tensor=audio_array, sample_rate=sample_rate, - response_format="wav", + response_format=audio_format, speed=1.0, base64_encode=True, ) @@ -3734,6 +3752,21 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): return height, width + def _resolve_audio_format(self, request: ChatCompletionRequest) -> str | ErrorResponse: + """Extract and validate the audio output format from a chat completion request.""" + audio_params = getattr(request, "audio", None) + if isinstance(audio_params, dict): + audio_format = audio_params.get("format", DEFAULT_AUDIO_FORMAT) + else: + audio_format = DEFAULT_AUDIO_FORMAT + if audio_format not in SUPPORTED_CHAT_AUDIO_FORMATS: + return self._create_error_response( + f"Invalid audio format '{audio_format}'. Supported formats: {sorted(SUPPORTED_CHAT_AUDIO_FORMATS)}", + ) + if audio_format == "pcm16": + audio_format = "pcm" + return audio_format + def _create_error_response( self, message: str,