[Bugfix] Fix audio.format parameter silently ignored in chat completions (#4718)

Signed-off-by: Ricardo Noriega De Soto <rnoriega@redhat.com>

docs/serving/audio_generate_api.md

@@ -66,7 +66,7 @@ Content-Type: application/json
 |-----------|------|---------|-------------|
 | `input` | string | **required** | Text prompt describing the audio to generate |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
+| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus |
 | `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
 
 #### Diffusion Parameters
@@ -90,7 +90,6 @@ Returns binary audio data with the appropriate `Content-Type` header:
 | `mp3` | `audio/mpeg` |
 | `flac` | `audio/flac` |
 | `pcm` | `audio/pcm` |
-| `aac` | `audio/aac` |
 | `opus` | `audio/opus` |
 
 ## Examples
@@ -285,7 +284,7 @@ Pydantic validation failure (e.g. invalid `response_format`, `speed` out of rang
     "detail": [
         {
             "type": "literal_error",
-            "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3', 'aac' or 'opus'",
+            "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3' or 'opus'",
             ...
         }
     ]

docs/serving/speech_api.md

@@ -109,7 +109,7 @@ Content-Type: application/json
 | `input` | string | **required** | The text to synthesize into speech |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
 | `voice` | string | "vivian" | Speaker name (e.g., vivian, ryan, aiden) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
+| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus |
 | `speed` | float | 1.0 | Playback speed (0.25-4.0) |
 
 #### vLLM-Omni Extension Parameters

docs/user_guide/examples/online_serving/text_to_audio.md

@@ -139,7 +139,7 @@ Content-Type: application/json
 |-----------|------|---------|-------------|
 | `input` | string | **required** | Text prompt describing the audio to generate |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
+| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus |
 | `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
 | `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) |
 | `audio_start` | float | 0.0 | Audio start time in seconds |

tests/e2e/online_serving/test_qwen3_omni_expansion.py

@@ -7,6 +7,7 @@ E2E Online tests for Qwen3-Omni model.
 import os
 
 import pytest
+from openai import BadRequestError
 
 from tests.helpers.mark import hardware_test
 from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
@@ -610,3 +611,21 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
     text = responses[0].text_content if responses else ""
     word_count = len(text.split())
     assert word_count >= 200, f"Expected at least 200 words in long output, got {word_count}"
+
+
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.parametrize("omni_server", test_params[:1], indirect=True)
+def test_invalid_audio_format_rejected(omni_server, openai_client) -> None:
+    """Ensure invalid audio format is rejected with 400 before streaming starts."""
+    messages = dummy_messages_from_mix_data(
+        system_prompt=get_system_prompt(),
+        content_text=get_prompt(),
+    )
+    with pytest.raises(BadRequestError, match="audio format"):
+        openai_client.client.chat.completions.create(
+            model=omni_server.model,
+            messages=messages,
+            modalities=["text", "audio"],
+            audio={"voice": "alloy", "format": "aac"},
+            stream=True,
+        )

tests/entrypoints/openai_api/test_audio_format.py

@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Tests for audio output format handling in chat completions.
+
+Covers:
+- #4716: audio.format parameter must be respected, not hardcoded to WAV
+- Format validation rejects unsupported formats
+- pcm16 is mapped to pcm for soundfile compatibility
+- Default format is WAV when not specified
+- create_audio encodes correctly for each supported format
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
+from vllm_omni.entrypoints.openai.protocol.audio import (
+    DEFAULT_AUDIO_FORMAT,
+    SUPPORTED_AUDIO_FORMATS,
+    SUPPORTED_CHAT_AUDIO_FORMATS,
+    CreateAudio,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class TestAudioFormatConstants:
+    def test_default_format_is_wav(self):
+        assert DEFAULT_AUDIO_FORMAT == "wav"
+
+    def test_supported_formats_no_aac(self):
+        assert "aac" not in SUPPORTED_AUDIO_FORMATS
+
+    def test_chat_formats_include_pcm16(self):
+        assert "pcm16" in SUPPORTED_CHAT_AUDIO_FORMATS
+
+    def test_chat_formats_superset_of_audio_formats(self):
+        assert SUPPORTED_AUDIO_FORMATS <= SUPPORTED_CHAT_AUDIO_FORMATS
+
+
+class TestCreateAudio:
+    @pytest.fixture
+    def mixin(self):
+        return AudioMixin()
+
+    @pytest.fixture
+    def audio_tensor(self):
+        return np.sin(np.linspace(0, 2 * np.pi, 24000)).astype(np.float32)
+
+    @pytest.mark.parametrize("fmt", ["wav", "mp3", "flac", "opus", "pcm"])
+    def test_create_audio_supported_formats(self, mixin, audio_tensor, fmt):
+        audio_obj = CreateAudio(
+            audio_tensor=audio_tensor,
+            sample_rate=24000,
+            response_format=fmt,
+            speed=1.0,
+            base64_encode=False,
+        )
+        response = mixin.create_audio(audio_obj)
+        assert len(response.audio_data) > 0
+
+    def test_wav_magic_bytes(self, mixin, audio_tensor):
+        audio_obj = CreateAudio(
+            audio_tensor=audio_tensor,
+            sample_rate=24000,
+            response_format="wav",
+            speed=1.0,
+            base64_encode=False,
+        )
+        response = mixin.create_audio(audio_obj)
+        assert response.audio_data[:4] == b"RIFF"
+        assert response.media_type == "audio/wav"
+
+    def test_mp3_encoding(self, mixin, audio_tensor):
+        audio_obj = CreateAudio(
+            audio_tensor=audio_tensor,
+            sample_rate=24000,
+            response_format="mp3",
+            speed=1.0,
+            base64_encode=False,
+        )
+        response = mixin.create_audio(audio_obj)
+        assert response.media_type == "audio/mpeg"
+        assert response.audio_data[:4] != b"RIFF"
+
+    def test_flac_magic_bytes(self, mixin, audio_tensor):
+        audio_obj = CreateAudio(
+            audio_tensor=audio_tensor,
+            sample_rate=24000,
+            response_format="flac",
+            speed=1.0,
+            base64_encode=False,
+        )
+        response = mixin.create_audio(audio_obj)
+        assert response.audio_data[:4] == b"fLaC"
+        assert response.media_type == "audio/flac"
+
+    def test_unsupported_format_falls_back_to_default(self, mixin, audio_tensor):
+        audio_obj = CreateAudio(
+            audio_tensor=audio_tensor,
+            sample_rate=24000,
+            response_format="aac",
+            speed=1.0,
+            base64_encode=False,
+        )
+        response = mixin.create_audio(audio_obj)
+        assert response.audio_data[:4] == b"RIFF"
+        assert response.media_type == "audio/wav"
+
+    def test_base64_encoding(self, mixin, audio_tensor):
+        import base64
+
+        audio_obj = CreateAudio(
+            audio_tensor=audio_tensor,
+            sample_rate=24000,
+            response_format="wav",
+            speed=1.0,
+            base64_encode=True,
+        )
+        response = mixin.create_audio(audio_obj)
+        decoded = base64.b64decode(response.audio_data)
+        assert decoded[:4] == b"RIFF"
+
+
+class TestResolveAudioFormat:
+    """Test _resolve_audio_format via the serving chat class."""
+
+    @pytest.fixture
+    def serving_chat(self):
+        from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
+
+        return object.__new__(OmniOpenAIServingChat)
+
+    def _make_request(self, audio_params=None):
+        from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+
+        req = ChatCompletionRequest(
+            model="test-model",
+            messages=[{"role": "user", "content": "hello"}],
+        )
+        if audio_params is not None:
+            req.audio = audio_params
+        return req
+
+    def test_default_format_when_no_audio_params(self, serving_chat):
+        request = self._make_request()
+        result = serving_chat._resolve_audio_format(request)
+        assert result == "wav"
+
+    def test_extracts_mp3_format(self, serving_chat):
+        request = self._make_request({"format": "mp3", "voice": "alloy"})
+        result = serving_chat._resolve_audio_format(request)
+        assert result == "mp3"
+
+    def test_pcm16_mapped_to_pcm(self, serving_chat):
+        request = self._make_request({"format": "pcm16", "voice": "alloy"})
+        result = serving_chat._resolve_audio_format(request)
+        assert result == "pcm"
+
+    def test_invalid_format_returns_error(self, serving_chat):
+        from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+
+        request = self._make_request({"format": "aac", "voice": "alloy"})
+        result = serving_chat._resolve_audio_format(request)
+        assert isinstance(result, ErrorResponse)
+        assert "aac" in result.error.message
+
+    def test_all_supported_formats_accepted(self, serving_chat):
+        from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+
+        for fmt in SUPPORTED_CHAT_AUDIO_FORMATS:
+            request = self._make_request({"format": fmt, "voice": "alloy"})
+            result = serving_chat._resolve_audio_format(request)
+            assert not isinstance(result, ErrorResponse), f"Format {fmt} should be accepted"

vllm_omni/entrypoints/openai/audio_utils_mixin.py

@@ -5,7 +5,7 @@ import torch
 import torchaudio
 from vllm.logger import init_logger
 
-from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio
+from vllm_omni.entrypoints.openai.protocol.audio import DEFAULT_AUDIO_FORMAT, AudioResponse, CreateAudio
 
 try:
     import soundfile
@@ -49,13 +49,12 @@ class AudioMixin:
             "pcm": ("RAW", "audio/pcm", {"subtype": "PCM_16"}),
             "flac": ("FLAC", "audio/flac", {}),
             "mp3": ("MP3", "audio/mpeg", {}),
-            "aac": ("AAC", "audio/aac", {}),
             "opus": ("OGG", "audio/ogg", {"subtype": "OPUS"}),
         }
 
         if response_format not in supported_formats:
-            logger.warning(f"Unsupported response format '{response_format}', defaulting to 'wav'.")
-            response_format = "wav"
+            logger.warning(f"Unsupported response format '{response_format}', defaulting to '{DEFAULT_AUDIO_FORMAT}'.")
+            response_format = DEFAULT_AUDIO_FORMAT
 
         soundfile_format, media_type, kwargs = supported_formats[response_format]
 

vllm_omni/entrypoints/openai/protocol/audio.py

@@ -6,6 +6,10 @@ from pydantic import AliasChoices, BaseModel, Field, field_validator, model_vali
 
 _MAX_EMBEDDING_DIM = 8192
 
+SUPPORTED_AUDIO_FORMATS: frozenset[str] = frozenset({"wav", "pcm", "flac", "mp3", "opus"})
+SUPPORTED_CHAT_AUDIO_FORMATS: frozenset[str] = SUPPORTED_AUDIO_FORMATS | {"pcm16"}
+DEFAULT_AUDIO_FORMAT: str = "wav"
+
 
 def _normalize_ref_audio_value(value):
     if value is None:
@@ -59,7 +63,7 @@ class OpenAICreateSpeechRequest(BaseModel):
         default=None,
         description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)",
     )
-    response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav"
+    response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT
     speed: float | None = Field(
         default=1.0,
         ge=0.25,
@@ -325,7 +329,7 @@ class OpenAICreateAudioGenerateRequest(BaseModel):
         description="Text prompt describing the audio to generate",
     )
     model: str | None = None
-    response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav"
+    response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT
     speed: float | None = Field(
         default=1.0,
         ge=0.25,
@@ -391,7 +395,7 @@ class SpeechBatchItem(BaseModel):
     input: str
     voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker"))
     instructions: str | None = None
-    response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] | None = None
+    response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] | None = None
     speed: float | None = Field(default=None, ge=0.25, le=4.0)
     task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None
     language: str | None = None
@@ -411,7 +415,7 @@ class BatchSpeechRequest(BaseModel):
     items: list[SpeechBatchItem] = Field(..., min_length=1)
     voice: str | None = Field(default=None, validation_alias=AliasChoices("voice", "speaker"))
     instructions: str | None = None
-    response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav"
+    response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT
     speed: float | None = Field(default=1.0, ge=0.25, le=4.0)
     task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None
     language: str | None = None
@@ -500,7 +504,7 @@ class StreamingSpeechSessionConfig(BaseModel):
     task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = None
     language: str | None = None
     instructions: str | None = None
-    response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav"
+    response_format: Literal["wav", "pcm", "flac", "mp3", "opus"] = DEFAULT_AUDIO_FORMAT
     speed: float | None = Field(default=1.0, ge=0.25, le=4.0)
     max_new_tokens: int | None = Field(default=None, ge=1)
     initial_codec_chunk_frames: int | None = Field(

vllm_omni/entrypoints/openai/serving_chat.py

@@ -97,7 +97,12 @@ from vllm.v1.engine.exceptions import EngineDeadError
 from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
 from vllm_omni.entrypoints.openai.image_api_utils import encode_image_base64_with_compression, validate_layered_layers
 from vllm_omni.entrypoints.openai.protocol import OmniChatCompletionStreamResponse
-from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio
+from vllm_omni.entrypoints.openai.protocol.audio import (
+    DEFAULT_AUDIO_FORMAT,
+    SUPPORTED_CHAT_AUDIO_FORMATS,
+    AudioResponse,
+    CreateAudio,
+)
 from vllm_omni.entrypoints.openai.protocol.images import (
     ImageData,
     ImageEditARDeltaChunk,
@@ -450,6 +455,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
         output_modalities = getattr(request, "modalities", engine_output_modalities)
         request.modalities = output_modalities if output_modalities is not None else engine_output_modalities
 
+        if request.modalities and "audio" in request.modalities:
+            audio_format_check = self._resolve_audio_format(request)
+            if isinstance(audio_format_check, ErrorResponse):
+                return audio_format_check
+
         num_inference_steps = None
         extra_body: dict[str, Any] = {}
         # Omni multistage image generation: Stage-0 (AR) should receive a clean
@@ -2561,10 +2571,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
         else:
             sample_rate = int(sr_raw)
 
+        audio_format = self._resolve_audio_format(request)
+        if isinstance(audio_format, ErrorResponse):
+            return audio_format
+
         audio_obj = CreateAudio(
             audio_tensor=audio_tensor,
             sample_rate=sample_rate,
-            response_format="wav",
+            response_format=audio_format,
             speed=1.0,
             base64_encode=True,
         )
@@ -3515,10 +3529,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
                     raise ValueError(f"Unexpected audio tensor rank {audio_tensor.ndim}; expected 1-3 dims.")
                 audio_array = audio_tensor.numpy()
 
+                audio_format = self._resolve_audio_format(request)
+                if isinstance(audio_format, ErrorResponse):
+                    return audio_format
+
                 audio_obj = CreateAudio(
                     audio_tensor=audio_array,
                     sample_rate=sample_rate,
-                    response_format="wav",
+                    response_format=audio_format,
                     speed=1.0,
                     base64_encode=True,
                 )
@@ -3734,6 +3752,21 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
 
         return height, width
 
+    def _resolve_audio_format(self, request: ChatCompletionRequest) -> str | ErrorResponse:
+        """Extract and validate the audio output format from a chat completion request."""
+        audio_params = getattr(request, "audio", None)
+        if isinstance(audio_params, dict):
+            audio_format = audio_params.get("format", DEFAULT_AUDIO_FORMAT)
+        else:
+            audio_format = DEFAULT_AUDIO_FORMAT
+        if audio_format not in SUPPORTED_CHAT_AUDIO_FORMATS:
+            return self._create_error_response(
+                f"Invalid audio format '{audio_format}'. Supported formats: {sorted(SUPPORTED_CHAT_AUDIO_FORMATS)}",
+            )
+        if audio_format == "pcm16":
+            audio_format = "pcm"
+        return audio_format
+
     def _create_error_response(
         self,
         message: str,