From d3493384cfc6c4b9ba1fc4e534e91fe588ecb4f1 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Fri, 11 Sep 2026 10:28:55 -0400 Subject: [PATCH] [BugFix] Fix leftovers of the legacy OpenAI realtime API event names (#7426) Signed-off-by: Nick Cao Co-authored-by: Codex --- docs/serving/realtime_api.md | 6 +-- docs/serving/video_stream_api.md | 4 +- examples/online_serving/qwen3_omni/README.md | 36 ++++++++------ .../qwen3_omni/openai_realtime_client.py | 14 +++--- .../openai_api/test_duplex_handler.py | 4 +- .../test_qwen3_omni_realtime_websocket.py | 16 +++--- .../openai_api/test_serving_video_stream.py | 49 +++++++++++++++++++ .../entrypoints/openai/realtime_connection.py | 8 +-- .../openai/serving_video_stream.py | 4 +- .../entrypoints/openai/video_stream_base.py | 10 ++-- 10 files changed, 103 insertions(+), 48 deletions(-) diff --git a/docs/serving/realtime_api.md b/docs/serving/realtime_api.md index 153564140..e44ff7ed0 100644 --- a/docs/serving/realtime_api.md +++ b/docs/serving/realtime_api.md @@ -43,8 +43,8 @@ event is base64-encoded raw PCM16. | Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input | | Server to client | `transcription.delta` | Carries incremental response text | | Server to client | `transcription.done` | Carries final text and token usage | -| Server to client | `response.audio.delta` | Carries incremental PCM16 response audio | -| Server to client | `response.audio.done` | Marks the end of response audio | +| Server to client | `response.output_audio.delta` | Carries incremental PCM16 response audio | +| Server to client | `response.output_audio.done` | Marks the end of response audio | | Server to client | `error` | Reports an invalid event, model, or audio payload | A minimal client sends events in this order: @@ -63,7 +63,7 @@ engine is consuming the stream. ## Audio Handling - Input is mono PCM16 at 16 kHz for the Qwen3-Omni example. -- `response.audio.delta.audio` contains base64-encoded PCM16 bytes. +- `response.output_audio.delta.audio` contains base64-encoded PCM16 bytes. - Read `sample_rate_hz` from each audio event instead of assuming an output rate. Qwen3-Omni output is typically 24 kHz. - Concatenate audio deltas in receive order to construct the output waveform. diff --git a/docs/serving/video_stream_api.md b/docs/serving/video_stream_api.md index e1f583991..78575e0fc 100644 --- a/docs/serving/video_stream_api.md +++ b/docs/serving/video_stream_api.md @@ -45,8 +45,8 @@ WebSocket /v1/video/chat/stream | Server -> Client | `response.start` | none | Query generation started. | | Server -> Client | `response.text.delta` | `delta` | Incremental text output. | | Server -> Client | `response.text.done` | `text` | Final text output for the query. | -| Server -> Client | `response.audio.delta` | `data`, `format` | Incremental generated audio, base64 WAV. | -| Server -> Client | `response.audio.done` | none | Audio output finished. | +| Server -> Client | `response.output_audio.delta` | `data`, `format` | Incremental generated audio, base64 WAV. | +| Server -> Client | `response.output_audio.done` | none | Audio output finished. | | Server -> Client | `session.done` | none | Session closed. | | Server -> Client | `error` | `message` | Recoverable protocol or generation error. | diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md index ef2367412..107e1f098 100644 --- a/examples/online_serving/qwen3_omni/README.md +++ b/examples/online_serving/qwen3_omni/README.md @@ -1,5 +1,7 @@ # Qwen3-Omni: Online serving + + ## 🛠️ Installation Please refer to [README.md](../../../README.md) @@ -19,6 +21,7 @@ Additionally, NPU, ROCm, and XPU per-platform configuration deltas are determini `platforms`: section of the corresponding YAML. To explicitly utilize a custom deployment YAML, mandate the configuration path accordingly: + ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \ --deploy-config /path/to/your_deploy_config.yaml @@ -53,7 +56,7 @@ Use the stage-based CLI when you want to run one stage per process. The example below pins Stage 0 to GPU 0 and Stage 1/2 to GPU 1 via `CUDA_VISIBLE_DEVICES`. -**1. Stage 0 (Thinker + API server)** +#### 1. Stage 0 (Thinker + API server) ```bash CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \ @@ -63,7 +66,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \ --omni-master-port 26000 ``` -**2. Stage 1 (Talker)** +#### 2. Stage 1 (Talker) ```bash CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \ @@ -73,7 +76,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \ --omni-master-port 26000 ``` -**3. Stage 2 (Code2Wav)** +#### 3. Stage 2 (Code2Wav) ```bash CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \ @@ -252,11 +255,12 @@ pattern works for Qwen2.5-Omni — replace `base_config:` with the path to ### Send Multi-modal Request Get into the example folder + ```bash cd examples/online_serving/qwen3_omni ``` -#### Send request via python +#### Send request via python ```bash python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type use_image --port 8091 --host "localhost" @@ -264,7 +268,7 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene #### Realtime WebSocket client (`openai_realtime_client.py`) -[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://:/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging. +[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://:/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.output_audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging. Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation. @@ -289,7 +293,7 @@ python openai_realtime_client.py \ **Arguments:** | Flag | Default | Description | -|------|---------|-------------| +| ------ | --------- | ------------- | | `--url` | `ws://localhost:8091/v1/realtime` | Full WebSocket URL including path | | `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (sent in `session.update`) | | `--input-wav` | *(required)* | Input WAV: mono, 16-bit PCM, **16 kHz** | @@ -297,7 +301,7 @@ python openai_realtime_client.py \ | `--output-text` | *(optional)* | If set, write final transcription text to this path | | `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) | | `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) | -| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files | +| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.output_audio.delta` WAV files | | `--server-vad` | disabled | Enable server-side Silero VAD instead of sending explicit commits | | `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) | | `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 | @@ -334,7 +338,6 @@ The Python client supports the following command-line arguments: - `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"` - `--speaker`: TTS speaker/voice for audio output when requesting audio (e.g. `ethan`, `chelsie`, `aiden`). Omit to use the model default. Example: `--speaker "chelsie"` - For example, to use a local video file with custom prompt: ```bash @@ -345,22 +348,22 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene --prompt "What are the main activities shown in this video?" ``` -#### Send request via curl +#### Send request via curl ```bash bash run_curl_multimodal_generation.sh use_image ``` - ### FAQ ## Modality control + You can control output modalities to specify which types of output the model should generate. This is useful when you only need text output and want to skip audio generation stages for better performance. ### Supported modalities | Modalities | Output | -|------------|--------| +| ------------ | -------- | | `["text"]` | Text only | | `["audio"]` | Audio only | | `["text", "audio"]` | Text + Audio | @@ -496,7 +499,9 @@ print(response.choices[0].message.audio) Supported speaker names depend on the model (e.g. `Ethan`, `Chelsie`, `Aiden`). Omit `speaker` to use the default. ## Streaming Output + If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. We support both text streaming output and audio streaming output. Other modalities can output normally. + ```bash python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py \ --query-type use_image \ @@ -521,12 +526,14 @@ The convenience script launches both the vLLM server and Gradio demo together: ``` This script will: + 1. Start the vLLM server in the background 2. Wait for the server to be ready 3. Launch the Gradio demo 4. Handle cleanup when you press Ctrl+C The script supports the following arguments: + - `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct) - `--server-port`: Port for vLLM server (default: 8091) - `--gradio-port`: Port for Gradio demo (default: 7861) @@ -537,18 +544,19 @@ The script supports the following arguments: #### Option 2: Manual Launch (Two-Step Process) -**Step 1: Launch the vLLM API server** +##### Step 1: Launch the vLLM API server ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` If you have a custom deploy config file: + ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file ``` -**Step 2: Run the Gradio demo** +##### Step 2: Run the Gradio demo In a separate terminal: @@ -561,7 +569,7 @@ Then open `http://localhost:7861/` on your local browser to interact with the we The gradio script supports the following arguments: - `--model`: Model name/path (should match the server model) -- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1) +- `--api-base`: Base URL for the vLLM API server (default: ) - `--ip`: Host/IP for Gradio server (default: 127.0.0.1) - `--port`: Port for Gradio server (default: 7861) - `--share`: Share the Gradio demo publicly (creates a public link) diff --git a/examples/online_serving/qwen3_omni/openai_realtime_client.py b/examples/online_serving/qwen3_omni/openai_realtime_client.py index 8a6871580..0a98d69a0 100644 --- a/examples/online_serving/qwen3_omni/openai_realtime_client.py +++ b/examples/online_serving/qwen3_omni/openai_realtime_client.py @@ -6,14 +6,14 @@ This client: 1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz), 2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events, -3) Receives response.audio.*, response.output_text.*, and transcript events, +3) Receives response.output_audio.*, response.output_text.*, and transcript events, 4) Saves synthesized audio to an output WAV file and optional text file. -By default each ``response.audio.delta`` is treated as an **incremental PCM** +By default each ``response.output_audio.delta`` is treated as an **incremental PCM** chunk and all chunks are concatenated into the final ``--output-wav``. Optional debugging: pass ``--delta-dump-dir DIR`` to write every -``response.audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, … +``response.output_audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, … Usage: python openai_realtime_client.py \ @@ -187,7 +187,7 @@ async def run_client( if event_type == "session.created": continue - if event_type == "response.audio.delta": + if event_type == "response.output_audio.delta": sr = event.get("sample_rate_hz") if isinstance(sr, int) and sr > 0: output_sample_rate = sr @@ -223,11 +223,11 @@ async def run_client( print(f"{log_prefix}text usage: {usage}") continue - if event_type == "response.audio_transcript.done": + if event_type == "response.output_audio_transcript.done": final_audio_transcript = event.get("transcript", "") continue - if event_type == "response.audio.done": + if event_type == "response.output_audio.done": if not server_vad: break continue @@ -339,7 +339,7 @@ def main() -> None: "--delta-dump-dir", type=Path, default=None, - help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory", + help="If set, each response.output_audio.delta is saved as delta_NNNNNN.wav under this directory", ) parser.add_argument( "--server-vad", diff --git a/tests/entrypoints/openai_api/test_duplex_handler.py b/tests/entrypoints/openai_api/test_duplex_handler.py index 0f9698457..206a38917 100644 --- a/tests/entrypoints/openai_api/test_duplex_handler.py +++ b/tests/entrypoints/openai_api/test_duplex_handler.py @@ -2197,9 +2197,7 @@ def test_native_realtime_protocol_audio_delta_preserves_sample_rate_hz(): ) audio_events = [ - payload - for payload in payloads - if payload["type"] in {"response.output_audio.delta", "response.output_audio.delta"} + payload for payload in payloads if payload["type"] in {"response.output_audio.delta", "response.audio.delta"} ] assert {payload["type"] for payload in audio_events} == {"response.output_audio.delta"} assert {payload["format"] for payload in audio_events} == {"pcm16"} diff --git a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py index 406668a4c..be83ea9db 100644 --- a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py @@ -175,7 +175,7 @@ async def _run_realtime_audio_roundtrip( if event_type == "session.created": continue - if event_type == "response.audio.delta": + if event_type == "response.output_audio.delta": delta_events += 1 sr = event.get("sample_rate_hz") if isinstance(sr, int) and sr > 0: @@ -195,7 +195,7 @@ async def _run_realtime_audio_roundtrip( final_text = event.get("text", "") or "".join(text_chunks) continue - if event_type == "response.audio.done": + if event_type == "response.output_audio.done": break if event_type == "error": @@ -315,7 +315,7 @@ def _server_vad_pcm16_input() -> bytes: def _assert_realtime_smoke(result: dict) -> None: out_pcm = result["output_pcm"] assert result["delta_events"] >= 1 - assert out_pcm, "No output PCM from response.audio.delta" + assert out_pcm, "No output PCM from response.output_audio.delta" assert len(out_pcm) % 2 == 0 assert len(out_pcm) >= 4096, "Output audio unexpectedly small" assert result["output_sample_rate"] > 0 @@ -388,7 +388,7 @@ class TestQwen3OmniRealtimeWebSocket: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True) def test_long_audio_response_completes_async_chunk(self, omni_server) -> None: - """Regression for #6474: long async-chunk audio must emit response.audio.done.""" + """Regression for #6474: long async-chunk audio must emit response.output_audio.done.""" issue_6474_pcm16 = _synthetic_pcm16_input( phrase_text=ISSUE_6474_SYNTH_PHRASE_TEXT, ) @@ -450,8 +450,8 @@ class TestQwen3OmniRealtimeWebSocket: "input_audio_buffer.speech_stopped", "input_audio_buffer.committed", "response.created", - "response.audio.delta", - "response.audio.done", + "response.output_audio.delta", + "response.output_audio.done", "response.done", ] input_item_ids: list[str] = [] @@ -459,7 +459,7 @@ class TestQwen3OmniRealtimeWebSocket: for events in turns: event_types = [event["type"] for event in events] for event_type in required_sequence: - if event_type != "response.audio.delta": + if event_type != "response.output_audio.delta": assert event_types.count(event_type) == 1, event_types positions = [event_types.index(event_type) for event_type in required_sequence] assert positions == sorted(positions) @@ -486,7 +486,7 @@ class TestQwen3OmniRealtimeWebSocket: output_pcm = b"".join( base64.b64decode(event["delta"]) for event in events - if event["type"] == "response.audio.delta" and event.get("delta") + if event["type"] == "response.output_audio.delta" and event.get("delta") ) assert output_pcm assert created["id"] == done["id"] diff --git a/tests/entrypoints/openai_api/test_serving_video_stream.py b/tests/entrypoints/openai_api/test_serving_video_stream.py index b534d6c67..58ea2b57f 100644 --- a/tests/entrypoints/openai_api/test_serving_video_stream.py +++ b/tests/entrypoints/openai_api/test_serving_video_stream.py @@ -14,6 +14,7 @@ import weakref from typing import Any import pytest +import torch from PIL import Image from vllm_omni.entrypoints.openai import video_stream_base, video_stream_envs @@ -457,6 +458,54 @@ async def test_async_chunk_mode_is_read_by_engine_path_at_runtime(monkeypatch): assert not any(m.get("type") == "response.text.delta" for m in ws_off.sent) +@pytest.mark.asyncio +@pytest.mark.parametrize("async_chunk_mode", ["on", "off"]) +async def test_audio_events_use_output_audio_names(monkeypatch, async_chunk_mode): + class AudioEngine: + def generate(self, **_kwargs): + async def _gen(): + yield _audio_result([torch.zeros(1)]) + + return _gen() + + class AudioHandler(QwenOmniStreamingVideoHandler): + async def _preprocess_to_engine_prompt(self, request): + return {"prompt": "x"} + + monkeypatch.setenv("VLLM_VIDEO_ASYNC_CHUNK", async_chunk_mode) + monkeypatch.setattr( + OmniStreamingVideoHandler, + "_encode_audio_wav_b64", + staticmethod(lambda _audio: "audio-b64"), + ) + + ws = MockWebSocket() + handler = AudioHandler(chat_service=object(), engine_client=AudioEngine()) + config = StreamingVideoSessionConfig(model="test", modalities=["text", "audio"]) + + await handler._process_query_engine( + ws, + config, + [_b64(_make_jpeg())], + bytearray(), + [], + "describe", + "req-audio-events", + asyncio.Event(), + {}, + ) + + audio_events = [message for message in ws.sent if "audio" in message.get("type", "")] + assert audio_events == [ + { + "type": "response.output_audio.delta", + "data": "audio-b64", + "format": "wav", + }, + {"type": "response.output_audio.done"}, + ] + + @pytest.mark.asyncio async def test_query_without_engine_client_sends_error(): ws = MockWebSocket() diff --git a/vllm_omni/entrypoints/openai/realtime_connection.py b/vllm_omni/entrypoints/openai/realtime_connection.py index 86a8ba031..3d37fbe37 100644 --- a/vllm_omni/entrypoints/openai/realtime_connection.py +++ b/vllm_omni/entrypoints/openai/realtime_connection.py @@ -176,7 +176,7 @@ class RealtimeConnection(VllmRealtimeConnection): sent_audio = True await self.send_json( { - "type": "response.audio.delta", + "type": "response.output_audio.delta", "audio": self._pcm16_b64(chunk), "format": "pcm16", "sample_rate_hz": sample_rate, @@ -194,7 +194,7 @@ class RealtimeConnection(VllmRealtimeConnection): await self.send(TranscriptionDone(text=full_text, usage=usage)) if sent_audio: - await self.send_json({"type": "response.audio.done", "has_audio": True}) + await self.send_json({"type": "response.output_audio.done", "has_audio": True}) audio_done_sent = True except Exception as e: logger.exception("Error in generation: %s", e) @@ -213,9 +213,9 @@ class RealtimeConnection(VllmRealtimeConnection): # Always send terminal event so clients don't hang forever. if self._is_connected and not audio_done_sent: try: - await self.send_json({"type": "response.audio.done", "has_audio": sent_audio}) + await self.send_json({"type": "response.output_audio.done", "has_audio": sent_audio}) except Exception: - logger.exception("Failed to send response.audio.done") + logger.exception("Failed to send response.output_audio.done") while not self.audio_queue.empty(): self.audio_queue.get_nowait() diff --git a/vllm_omni/entrypoints/openai/serving_video_stream.py b/vllm_omni/entrypoints/openai/serving_video_stream.py index e05a40eef..5b30ff0fa 100644 --- a/vllm_omni/entrypoints/openai/serving_video_stream.py +++ b/vllm_omni/entrypoints/openai/serving_video_stream.py @@ -18,8 +18,8 @@ Protocol: {"type": "response.start"} {"type": "response.text.delta", "delta": "..."} {"type": "response.text.done", "text": "..."} - {"type": "response.audio.delta", "data": "...", "format": "wav"} - {"type": "response.audio.done"} + {"type": "response.output_audio.delta", "data": "...", "format": "wav"} + {"type": "response.output_audio.done"} {"type": "session.done"} {"type": "error", "message": "..."} """ diff --git a/vllm_omni/entrypoints/openai/video_stream_base.py b/vllm_omni/entrypoints/openai/video_stream_base.py index ab5d51c33..977a0b065 100644 --- a/vllm_omni/entrypoints/openai/video_stream_base.py +++ b/vllm_omni/entrypoints/openai/video_stream_base.py @@ -21,8 +21,8 @@ Protocol: {"type": "response.start"} {"type": "response.text.delta", "delta": "..."} {"type": "response.text.done", "text": "..."} - {"type": "response.audio.delta", "data": "...", "format": "wav"} - {"type": "response.audio.done"} + {"type": "response.output_audio.delta", "data": "...", "format": "wav"} + {"type": "response.output_audio.done"} {"type": "session.done"} {"type": "error", "message": "..."} """ @@ -763,7 +763,7 @@ class OmniStreamingVideoHandler: if b64: await websocket.send_json( { - "type": "response.audio.delta", + "type": "response.output_audio.delta", "data": b64, "format": "wav", } @@ -807,7 +807,7 @@ class OmniStreamingVideoHandler: if b64: await websocket.send_json( { - "type": "response.audio.delta", + "type": "response.output_audio.delta", "data": b64, "format": "wav", } @@ -816,7 +816,7 @@ class OmniStreamingVideoHandler: logger.exception("Failed to coalesce off-path audio") if audio_chunk_count > 0: - await websocket.send_json({"type": "response.audio.done"}) + await websocket.send_json({"type": "response.output_audio.done"}) response_text = "".join(text_parts) self.on_turn_complete(message_history, user_message, response_text)