diff --git a/docs/serving/videos_api.md b/docs/serving/videos_api.md index 3825cc1ec..4d156bdf0 100644 --- a/docs/serving/videos_api.md +++ b/docs/serving/videos_api.md @@ -70,6 +70,7 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4 | `input_reference` | file | null | Uploaded reference image or video for image-to-video/video-to-video requests | | `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` | | `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` | +| `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | | `width` | integer | model default | Output video width | | `height` | integer | model default | Output video height | | `num_frames` | integer | model default | Number of generated frames | @@ -161,6 +162,25 @@ are not implemented yet. Models may expose additional V2V controls through `condition_frame_indexes_vision` and `condition_video_keep` to select which decoded reference frames are used as clean conditioning. +### Speech-to-Video + +For models that support audio-driven generation (e.g., Wan2.2-S2V), pass both +an image reference and an audio reference. The `audio_reference` field accepts a +JSON string with `audio_url` pointing to an HTTP(s) URL or base64 data URL. + +```bash +curl -s http://localhost:8091/v1/videos \ + -F "prompt=A person singing" \ + -F 'image_reference={"image_url": "https://example.com/face.png"}' \ + -F 'audio_reference={"audio_url": "https://example.com/speech.mp3"}' \ + -F "width=832" \ + -F "height=480" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.5" \ + -F "fps=16" +``` + + ### Synchronous Generation ```bash @@ -189,5 +209,7 @@ walkthroughs, see: - [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md) - [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md) +- [Speech-to-Video](../user_guide/examples/online_serving/speech_to_video.md) + for Wan2.2-S2V audio-driven lip-sync generation - [Cosmos3 recipes](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-Nano.md) for model-specific video-to-video examples and conditioning controls diff --git a/docs/user_guide/examples/online_serving/speech_to_video.md b/docs/user_guide/examples/online_serving/speech_to_video.md new file mode 100644 index 000000000..80788258c --- /dev/null +++ b/docs/user_guide/examples/online_serving/speech_to_video.md @@ -0,0 +1,128 @@ +# Speech-To-Video + +Source . + +This example demonstrates how to deploy the Wan2.2 speech-to-video (S2V) model for online video generation using vLLM-Omni. + +## Supported Models + +| Model | Model ID | +|-------|----------| +| Wan2.2 S2V (14B) | `Wan-AI/Wan2.2-S2V-14B` | + +## Start Server + +### Basic Start + +```bash +VLLM_WORKER_MULTIPROC_METHOD=spawn \ +vllm serve Wan-AI/Wan2.2-S2V-14B --omni \ + --model-class-name WanS2VPipeline \ + --tensor-parallel-size 2 \ + --flow-shift 3.0 \ + --vae-use-slicing --vae-use-tiling \ + --cache-backend cache_dit \ + --port 8091 +``` + +### Start with Script + +```bash +bash run_server.sh +``` + +The script allows overriding: +- `MODEL` (default: `Wan-AI/Wan2.2-S2V-14B`) +- `PORT` (default: `8091`) +- `FLOW_SHIFT` (default: `3.0`) +- `TP` (default: `2`) +- `CACHE_BACKEND` (default: `cache_dit`) + +### Start with 4 GPUs + +```bash +TP=4 bash run_server.sh +``` + +## API Endpoints + +The server provides OpenAI-compatible video generation endpoints: + +- `POST /v1/videos`: create a video generation job (async) +- `POST /v1/videos/sync`: generate a video and return raw bytes (sync, for benchmarks) +- `GET /v1/videos/{video_id}`: retrieve the current job status and metadata +- `GET /v1/videos`: list stored video jobs +- `GET /v1/videos/{video_id}/content`: download the generated video file +- `DELETE /v1/videos/{video_id}`: delete the job and any stored output + +## Sync API (Recommended for Testing) + +`POST /v1/videos/sync` blocks until generation completes and returns the raw +video bytes (`video/mp4`) directly in the response body. + +```bash +bash run_curl_speech_to_video.sh +``` + +## Async Job API + +`POST /v1/videos` creates an asynchronous job and returns immediately. + +```bash +no_proxy=127.0.0.1 \ +create_response=$(curl -sS -X POST http://127.0.0.1:8091/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A person singing" \ + -F 'image_reference={"image_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png"}' \ + -F 'audio_reference={"audio_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3"}' \ + -F "width=832" -F "height=480" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.5" \ + -F "fps=16") + +video_id=$(echo "$create_response" | jq -r '.id') + +# Poll until complete +while true; do + status=$(curl -s "http://127.0.0.1:8091/v1/videos/${video_id}" | jq -r '.status') + if [ "$status" = "completed" ]; then + break + fi + if [ "$status" = "failed" ]; then + echo "Video generation failed" + exit 1 + fi + sleep 2 +done + +# Download +curl -L "http://127.0.0.1:8091/v1/videos/${video_id}/content" -o s2v_output.mp4 +``` + +## Request Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `prompt` | str | - | Text description of the desired video | +| `image_reference` | JSON str | - | Image reference: `{"image_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | +| `audio_reference` | JSON str | - | Audio reference: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | +| `width` | int | None | Video width in pixels | +| `height` | int | None | Video height in pixels | +| `fps` | int | None | Output video frame rate | +| `num_inference_steps` | int | None | Number of denoising steps (40 recommended) | +| `guidance_scale` | float | None | CFG guidance scale | +| `seed` | int | None | Random seed for reproducibility | + +## Notes + +- S2V requires both a reference image (`image_reference`) and an audio reference + (`audio_reference`). The generated video will show a person matching the reference + image with lip movements synchronized to the audio. +- `audio_reference` accepts a JSON string: `{"audio_url": "..."}` where the URL can be: + - An HTTP/HTTPS URL (e.g., `https://example.com/audio.mp3`) + - A base64 data URL (e.g., `data:audio/mp3;base64,...`) +- `--model-class-name WanS2VPipeline` is required on the server to select the + S2V pipeline (distinct from the T2V/I2V pipelines). +- `--cache-backend cache_dit` enables DiT caching for ~2x speedup on cached steps. +- Audio is muxed into the output MP4 automatically. +- Always pass `fps=16` to ensure correct video/audio alignment. diff --git a/examples/online_serving/speech_to_video/README.md b/examples/online_serving/speech_to_video/README.md new file mode 100644 index 000000000..b074a63ce --- /dev/null +++ b/examples/online_serving/speech_to_video/README.md @@ -0,0 +1,115 @@ +# Speech-To-Video + +This example demonstrates how to deploy the Wan2.2 speech-to-video (S2V) model for online video generation using vLLM-Omni. + +## Supported Models + +| Model | Model ID | +|-------|----------| +| Wan2.2 S2V (14B) | `Wan-AI/Wan2.2-S2V-14B` | + +## Start Server + +### Basic Start + +```bash +VLLM_WORKER_MULTIPROC_METHOD=spawn \ +vllm serve Wan-AI/Wan2.2-S2V-14B --omni \ + --model-class-name WanS2VPipeline \ + --tensor-parallel-size 2 \ + --flow-shift 3.0 \ + --vae-use-slicing --vae-use-tiling \ + --cache-backend cache_dit \ + --port 8091 +``` + +### Start with Script + +```bash +bash run_server.sh +``` + +The script allows overriding: +- `MODEL` (default: `Wan-AI/Wan2.2-S2V-14B`) +- `PORT` (default: `8091`) +- `FLOW_SHIFT` (default: `3.0`) +- `TP` (default: `2`) +- `CACHE_BACKEND` (default: `cache_dit`) + +### Start with 4 GPUs + +```bash +TP=4 bash run_server.sh +``` + +## Sync API (Recommended for Testing) + +`POST /v1/videos/sync` blocks until generation completes and returns the raw +video bytes (`video/mp4`) directly in the response body. + +```bash +bash run_curl_speech_to_video.sh +``` + +## Async Job API + +`POST /v1/videos` creates an asynchronous job and returns immediately. + +```bash +no_proxy=127.0.0.1 \ +create_response=$(curl -sS -X POST http://127.0.0.1:8091/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A person singing" \ + -F 'image_reference={"image_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png"}' \ + -F 'audio_reference={"audio_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3"}' \ + -F "width=832" -F "height=480" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.5" \ + -F "fps=16") + +video_id=$(echo "$create_response" | jq -r '.id') + +# Poll until complete +while true; do + status=$(curl -s "http://127.0.0.1:8091/v1/videos/${video_id}" | jq -r '.status') + if [ "$status" = "completed" ]; then + break + fi + if [ "$status" = "failed" ]; then + echo "Video generation failed" + exit 1 + fi + sleep 2 +done + +# Download +curl -L "http://127.0.0.1:8091/v1/videos/${video_id}/content" -o s2v_output.mp4 +``` + +## Request Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `prompt` | str | - | Text description of the desired video | +| `image_reference` | JSON str | - | Image reference: `{"image_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | +| `audio_reference` | JSON str | - | Audio reference: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | +| `width` | int | None | Video width in pixels | +| `height` | int | None | Video height in pixels | +| `fps` | int | None | Output video frame rate | +| `num_inference_steps` | int | None | Number of denoising steps (40 recommended) | +| `guidance_scale` | float | None | CFG guidance scale | +| `seed` | int | None | Random seed for reproducibility | + +## Notes + +- S2V requires both a reference image (`image_reference`) and an audio reference + (`audio_reference`). The generated video will show a person matching the reference + image with lip movements synchronized to the audio. +- `audio_reference` accepts a JSON string: `{"audio_url": "..."}` where the URL can be: + - An HTTP/HTTPS URL (e.g., `https://example.com/audio.mp3`) + - A base64 data URL (e.g., `data:audio/mp3;base64,...`) +- `--model-class-name WanS2VPipeline` is required on the server to select the + S2V pipeline (distinct from the T2V/I2V pipelines). +- `--cache-backend cache_dit` enables DiT caching for ~2x speedup on cached steps. +- Audio is muxed into the output MP4 automatically. +- Always pass `fps=16` to ensure correct video/audio alignment. diff --git a/examples/online_serving/speech_to_video/run_curl_speech_to_video.sh b/examples/online_serving/speech_to_video/run_curl_speech_to_video.sh new file mode 100755 index 000000000..dee0d0fac --- /dev/null +++ b/examples/online_serving/speech_to_video/run_curl_speech_to_video.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Wan2.2 S2V (speech-to-video) curl example using the sync video API. + +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8091}" +OUTPUT_PATH="${OUTPUT_PATH:-s2v_480p_serve.mp4}" +IMAGE_URL="${IMAGE_URL:-https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png}" +AUDIO_URL="${AUDIO_URL:-https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3}" +PROMPT="${PROMPT:-A person singing}" +WIDTH="${WIDTH:-832}" +HEIGHT="${HEIGHT:-480}" +NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-40}" +GUIDANCE_SCALE="${GUIDANCE_SCALE:-4.5}" +FPS="${FPS:-16}" + +echo "Sending S2V request..." +echo " Image URL: $IMAGE_URL" +echo " Audio URL: $AUDIO_URL" +echo " Prompt: $PROMPT" +echo " Resolution: ${WIDTH}x${HEIGHT}" +echo " Steps: $NUM_INFERENCE_STEPS" +echo " FPS: $FPS" + +IMAGE_REF_JSON="{\"image_url\": \"${IMAGE_URL}\"}" +AUDIO_REF_JSON="{\"audio_url\": \"${AUDIO_URL}\"}" + +no_proxy=127.0.0.1 \ +curl -X POST "${BASE_URL}/v1/videos/sync" \ + -F "prompt=${PROMPT}" \ + -F "image_reference=${IMAGE_REF_JSON}" \ + -F "audio_reference=${AUDIO_REF_JSON}" \ + -F "width=${WIDTH}" -F "height=${HEIGHT}" \ + -F "num_inference_steps=${NUM_INFERENCE_STEPS}" \ + -F "guidance_scale=${GUIDANCE_SCALE}" \ + -F "fps=${FPS}" \ + --output "${OUTPUT_PATH}" + +if [ -f "$OUTPUT_PATH" ] && [ -s "$OUTPUT_PATH" ]; then + echo "Saved video to ${OUTPUT_PATH} ($(du -h "$OUTPUT_PATH" | cut -f1))" +else + echo "ERROR: Output file is empty or missing" + exit 1 +fi diff --git a/examples/online_serving/speech_to_video/run_server.sh b/examples/online_serving/speech_to_video/run_server.sh new file mode 100755 index 000000000..1f20a5b72 --- /dev/null +++ b/examples/online_serving/speech_to_video/run_server.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Wan2.2 S2V (speech-to-video) online serving startup script + +MODEL="${MODEL:-Wan-AI/Wan2.2-S2V-14B}" +PORT="${PORT:-8091}" +FLOW_SHIFT="${FLOW_SHIFT:-3.0}" +TP="${TP:-2}" +CACHE_BACKEND="${CACHE_BACKEND:-cache_dit}" + +echo "Starting Wan2.2 S2V server..." +echo "Model: $MODEL" +echo "Port: $PORT" +echo "Flow shift: $FLOW_SHIFT" +echo "Tensor parallel size: $TP" +echo "Cache backend: $CACHE_BACKEND" + +CACHE_BACKEND_FLAG="" +if [ "$CACHE_BACKEND" != "none" ]; then + CACHE_BACKEND_FLAG="--cache-backend $CACHE_BACKEND" +fi + +VLLM_WORKER_MULTIPROC_METHOD=spawn \ +vllm serve "$MODEL" --omni \ + --model-class-name WanS2VPipeline \ + --tensor-parallel-size "$TP" \ + --flow-shift "$FLOW_SHIFT" \ + --vae-use-slicing --vae-use-tiling \ + --port "$PORT" \ + $CACHE_BACKEND_FLAG diff --git a/recipes/Wan-AI/Wan2.2-S2V.md b/recipes/Wan-AI/Wan2.2-S2V.md index 9f34509d4..a91e768d0 100644 --- a/recipes/Wan-AI/Wan2.2-S2V.md +++ b/recipes/Wan-AI/Wan2.2-S2V.md @@ -171,6 +171,34 @@ python examples/offline_inference/speech_to_video/speech_to_video.py \ ``` +### Online Serving + +#### Server + +```bash +vllm serve Wan-AI/Wan2.2-S2V-14B --omni \ + --model-class-name WanS2VPipeline \ + --tensor-parallel-size 2 \ + --flow-shift 3.0 \ + --vae-use-slicing --vae-use-tiling \ + --port 8091 +``` + +#### Client + +```bash +no_proxy=127.0.0.1 \ +curl -X POST http://127.0.0.1:8091/v1/videos/sync \ + -F "prompt=A person singing" \ + -F 'image_reference={"image_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png"}' \ + -F 'audio_reference={"audio_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3"}' \ + -F "width=832" -F "height=480" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.5" \ + -F "fps=16" \ + --output s2v_480p_serve.mp4 +``` + #### Verification The output video should show a talking/singing person matching the reference diff --git a/tests/entrypoints/openai_api/test_decode_audio_url.py b/tests/entrypoints/openai_api/test_decode_audio_url.py new file mode 100644 index 000000000..2a5c5be47 --- /dev/null +++ b/tests/entrypoints/openai_api/test_decode_audio_url.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for decode_audio_url helper.""" + +import asyncio +import base64 +import os + +import httpx +import pytest + +from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError +from vllm_omni.entrypoints.openai.video_api_utils import decode_audio_url + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _run(coro): + return asyncio.run(coro) + + +def _cleanup(path): + if path and os.path.exists(path): + os.unlink(path) + + +class TestDecodeAudioUrlDataUrl: + """Tests for base64 data URL decoding.""" + + def test_valid_mp3_data_url(self): + audio_bytes = b"\xff\xfb\x90\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/mp3;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert os.path.isfile(path) + assert path.endswith(".mp3") + with open(path, "rb") as f: + assert f.read() == audio_bytes + finally: + _cleanup(path) + + def test_valid_wav_data_url(self): + audio_bytes = b"RIFF" + b"\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/wav;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert path.endswith(".wav") + with open(path, "rb") as f: + assert f.read() == audio_bytes + finally: + _cleanup(path) + + def test_valid_mpeg_data_url_uses_mp3_suffix(self): + audio_bytes = b"\xff\xfb\x90\x00" * 50 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/mpeg;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert path.endswith(".mp3") + finally: + _cleanup(path) + + def test_valid_ogg_data_url(self): + audio_bytes = b"OggS" + b"\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/ogg;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert path.endswith(".ogg") + finally: + _cleanup(path) + + def test_invalid_base64_raises(self): + url = "data:audio/mp3;base64,!!!not-valid-base64!!!" + + with pytest.raises(InvalidInputReferenceError, match="not valid base64"): + _run(decode_audio_url(url)) + + def test_empty_audio_data_raises(self): + url = "data:audio/mp3;base64," + + with pytest.raises(InvalidInputReferenceError, match="audio data is empty"): + _run(decode_audio_url(url)) + + +class TestDecodeAudioUrlSuffixSanitization: + """Tests for MIME extension sanitization (path traversal prevention).""" + + def test_path_traversal_in_mime_is_neutralized(self): + audio_bytes = b"\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/../..;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + # ".." contains non-alnum dots, so falls back to .wav + assert path.endswith(".wav") + assert os.path.dirname(path) == "/tmp" + finally: + _cleanup(path) + + def test_non_alnum_extension_falls_back_to_wav(self): + audio_bytes = b"\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/foo-bar;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert path.endswith(".wav") + finally: + _cleanup(path) + + def test_too_long_extension_falls_back_to_wav(self): + audio_bytes = b"\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/verylongextension;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert path.endswith(".wav") + finally: + _cleanup(path) + + def test_valid_short_alnum_extension_is_used(self): + audio_bytes = b"\x00" * 100 + b64 = base64.b64encode(audio_bytes).decode() + url = f"data:audio/flac;base64,{b64}" + + path = _run(decode_audio_url(url)) + try: + assert path.endswith(".flac") + finally: + _cleanup(path) + + +class TestDecodeAudioUrlHttp: + """Tests for HTTP URL decoding.""" + + def test_valid_http_url(self, monkeypatch): + audio_bytes = b"\xff\xfb\x90\x00" * 50 + fake_resp = httpx.Response(200, content=audio_bytes) + + async def _mock_get(self, url): + return fake_resp + + monkeypatch.setattr(httpx.AsyncClient, "get", _mock_get) + monkeypatch.setattr(httpx.Response, "raise_for_status", lambda self: None) + + path = _run(decode_audio_url("https://example.com/audio.mp3")) + try: + assert os.path.isfile(path) + assert path.endswith(".wav") + with open(path, "rb") as f: + assert f.read() == audio_bytes + finally: + _cleanup(path) + + def test_http_error_raises(self, monkeypatch): + async def _mock_get(self, url): + raise httpx.HTTPStatusError("Not Found", request=httpx.Request("GET", url), response=httpx.Response(404)) + + monkeypatch.setattr(httpx.AsyncClient, "get", _mock_get) + + with pytest.raises(InvalidInputReferenceError, match="failed to download audio"): + _run(decode_audio_url("https://example.com/missing.mp3")) + + +class TestDecodeAudioUrlInvalid: + """Tests for invalid URL schemes.""" + + def test_local_file_path_raises(self): + with pytest.raises(InvalidInputReferenceError, match="must be an http.*URL or data URL"): + _run(decode_audio_url("/path/to/local/file.mp3")) + + def test_ftp_url_raises(self): + with pytest.raises(InvalidInputReferenceError, match="must be an http.*URL or data URL"): + _run(decode_audio_url("ftp://example.com/file.mp3")) + + def test_empty_string_raises(self): + with pytest.raises(InvalidInputReferenceError, match="must be an http.*URL or data URL"): + _run(decode_audio_url("")) diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index b2dc32c6e..1bd353841 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -86,8 +86,10 @@ class BlockingVideoHandler: if self.stage_configs is None: self.stage_configs = stage_configs - async def generate_video_bytes(self, request, reference_id, *, reference_image=None, reference_video=None): - del request, reference_id, reference_image, reference_video + async def generate_video_bytes( + self, request, reference_id, *, reference_image=None, reference_video=None, reference_audio=None + ): + del request, reference_id, reference_image, reference_video, reference_audio self.started.set() try: await asyncio.Future() @@ -457,6 +459,41 @@ def test_cosmos3_reference_video_limit_caps_condition_frames_to_output_frames(): assert api_server._reference_video_decode_spec(request, _cosmos3_stage_configs()).max_frames == 5 +def test_s2v_video_generation_with_audio_reference_form(test_client, mocker: MockerFixture): + """Speech-to-video: image + audio_reference (base64 data URL) passes audio path to multi_modal_data.""" + audio_bytes = b"\xff\xfb\x90\x00" * 50 + audio_b64 = base64.b64encode(audio_bytes).decode() + audio_ref = json.dumps({"audio_url": f"data:audio/mp3;base64,{audio_b64}"}) + + mocker.patch( + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", + ) + response = test_client.post( + "/v1/videos", + data={ + "prompt": "A person singing", + "audio_reference": audio_ref, + "width": "832", + "height": "480", + }, + files={"input_reference": ("face.png", _make_test_image_bytes((64, 64)), "image/png")}, + ) + + assert response.status_code == 200 + video_id = response.json()["id"] + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + + engine = test_client.app.state.openai_serving_video._engine_client + prompt = engine.captured_prompt + assert "multi_modal_data" in prompt + assert "image" in prompt["multi_modal_data"] + assert "audio" in prompt["multi_modal_data"] + audio_path = prompt["multi_modal_data"]["audio"] + assert isinstance(audio_path, str) + assert audio_path.endswith(".mp3") + + def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture): fps_values = [] diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py index 81889607a..8edf7ec19 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py @@ -69,8 +69,7 @@ class DistributedRMSNorm(nn.Module): local_count = x.shape[-1] if tp_size > 1: - global_sum_sq = local_sum_sq.clone() - tensor_model_parallel_all_reduce(global_sum_sq) + global_sum_sq = tensor_model_parallel_all_reduce(local_sum_sq) global_count = local_count * tp_size else: global_sum_sq = local_sum_sq diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 0164e584a..0b419c8ed 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -126,7 +126,12 @@ from vllm_omni.entrypoints.openai.serving_audio_generate import OmniOpenAIServin from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler -from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo, ReferenceImage, ReferenceVideo +from vllm_omni.entrypoints.openai.serving_video import ( + OmniOpenAIServingVideo, + ReferenceAudio, + ReferenceImage, + ReferenceVideo, +) from vllm_omni.entrypoints.openai.serving_video_stream import OmniStreamingVideoHandler from vllm_omni.entrypoints.openai.stage_params import ( build_stage_sampling_params_list, @@ -135,7 +140,7 @@ from vllm_omni.entrypoints.openai.stage_params import ( from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER from vllm_omni.entrypoints.openai.stores import VIDEO_STORE, VIDEO_TASKS from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request -from vllm_omni.entrypoints.openai.video_api_utils import decode_input_reference +from vllm_omni.entrypoints.openai.video_api_utils import decode_audio_url, decode_input_reference from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace @@ -2686,6 +2691,7 @@ async def _run_video_generation_job( video_id: str, reference_image: ReferenceImage | None = None, reference_video: ReferenceVideo | None = None, + reference_audio: ReferenceAudio | None = None, app_state: Any | None = None, ) -> None: job = await VIDEO_STORE.get(video_id) @@ -2702,6 +2708,7 @@ async def _run_video_generation_job( video_id, reference_image=reference_image, reference_video=reference_video, + reference_audio=reference_audio, ) file_name = f"{video_id}.{job.file_extension}" @@ -2758,6 +2765,9 @@ async def _run_video_generation_job( _cleanup_video(video_id, output_path) await VIDEO_STORE.pop(video_id) raise + finally: + if reference_audio is not None and os.path.exists(reference_audio.path): + os.unlink(reference_audio.path) VIDEO_SYNC_TIMEOUT_S = 600.0 @@ -2769,6 +2779,7 @@ async def _parse_video_form( input_reference: UploadFile | None = File(default=None), image_reference: str | None = Form(default=None), video_reference: str | None = Form(default=None), + audio_reference: str | None = Form(default=None), model: str | None = Form(default=None), seconds: SecondStr | None = Form(default=None), size: SizeStr | None = Form(default=None), @@ -2793,7 +2804,14 @@ async def _parse_video_form( frame_interpolation_model_path: str | None = Form(default=None), lora: str | None = Form(default=None), extra_params: str | None = Form(default=None), -) -> tuple[VideoGenerationRequest, "OmniOpenAIServingVideo", str, ReferenceImage | None, ReferenceVideo | None]: +) -> tuple[ + VideoGenerationRequest, + "OmniOpenAIServingVideo", + str, + ReferenceImage | None, + ReferenceVideo | None, + ReferenceAudio | None, +]: """FastAPI dependency that parses video form data, validates inputs, resolves the handler, and decodes any reference image. @@ -2802,6 +2820,7 @@ async def _parse_video_form( input_reference_bytes = await input_reference.read() if input_reference is not None else None parsed_image_reference = _parse_form_json(image_reference) parsed_video_reference = _parse_form_json(video_reference) + parsed_audio_reference = _parse_form_json(audio_reference) provided_references = sum( item is not None for item in (parsed_image_reference, parsed_video_reference, input_reference_bytes) @@ -2819,6 +2838,7 @@ async def _parse_video_form( "size": size, "image_reference": parsed_image_reference, "video_reference": parsed_video_reference, + "audio_reference": parsed_audio_reference, "user": user, "width": width, "height": height, @@ -2893,7 +2913,16 @@ async def _parse_video_form( reference_image = ReferenceImage(data=media_data) if isinstance(media_data, Image.Image) else None reference_video = ReferenceVideo(data=media_data) if isinstance(media_data, list) else None - return request, handler, effective_model_name, reference_image, reference_video + + reference_audio: ReferenceAudio | None = None + if request.audio_reference is not None: + try: + audio_path = await decode_audio_url(request.audio_reference.audio_url) + except InvalidInputReferenceError as exc: + raise HTTPException(400, detail=str(exc)) from exc + reference_audio = ReferenceAudio(path=audio_path) + + return request, handler, effective_model_name, reference_image, reference_video, reference_audio @router.post( @@ -2913,6 +2942,7 @@ async def create_video( str, ReferenceImage | None, ReferenceVideo | None, + ReferenceAudio | None, ] = Depends(_parse_video_form), ) -> VideoResponse: """Create an asynchronous video generation job. @@ -2920,7 +2950,7 @@ async def create_video( Accepts multipart form-data (see ``_parse_video_form`` for parameters), persists a queued job record, and starts generation in the background. """ - request, handler, effective_model_name, reference_image, reference_video = ctx + request, handler, effective_model_name, reference_image, reference_video, reference_audio = ctx ref = video_response_from_request(effective_model_name, request) await VIDEO_STORE.upsert(ref.id, ref) task = asyncio.create_task( @@ -2930,6 +2960,7 @@ async def create_video( ref.id, reference_image, reference_video, + reference_audio, app_state=raw_request.app.state, ) ) @@ -2954,6 +2985,7 @@ async def create_video_sync( str, ReferenceImage | None, ReferenceVideo | None, + ReferenceAudio | None, ] = Depends(_parse_video_form), ) -> Response: """Synchronous video generation endpoint. @@ -2965,7 +2997,7 @@ async def create_video_sync( Metadata is returned via response headers ``X-Request-Id``, ``X-Model``, and ``X-Inference-Time-S``. """ - request, handler, effective_model_name, reference_image, reference_video = ctx + request, handler, effective_model_name, reference_image, reference_video, reference_audio = ctx request_id = f"video_sync-{random_uuid()}" raw_request.state.request_metadata = RequestResponseMetadata(request_id=request_id) started_at = time.perf_counter() @@ -2976,6 +3008,7 @@ async def create_video_sync( request_id, reference_image=reference_image, reference_video=reference_video, + reference_audio=reference_audio, ), timeout=VIDEO_SYNC_TIMEOUT_S, ) @@ -2994,6 +3027,9 @@ async def create_video_sync( status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=f"Video generation failed: {str(exc)}", ) from exc + finally: + if reference_audio is not None and os.path.exists(reference_audio.path): + os.unlink(reference_audio.path) inference_time_s = time.perf_counter() - started_at return Response( diff --git a/vllm_omni/entrypoints/openai/protocol/videos.py b/vllm_omni/entrypoints/openai/protocol/videos.py index 7cca71ad1..8773dd2d8 100644 --- a/vllm_omni/entrypoints/openai/protocol/videos.py +++ b/vllm_omni/entrypoints/openai/protocol/videos.py @@ -86,6 +86,14 @@ class UrlVideoReference(BaseModel): VideoReference = UrlVideoReference | FileVideoReference +class UrlAudioReference(BaseModel): + model_config = ConfigDict(extra="forbid") + audio_url: str + + +AudioReference = UrlAudioReference + + class VideoGenerationRequest(BaseModel): """ OpenAI-style video generation request. @@ -116,6 +124,10 @@ class VideoGenerationRequest(BaseModel): default=None, description="Optional JSON-safe video reference that guides generation. Provide either video_url or file_id.", ) + audio_reference: AudioReference | None = Field( + default=None, + description="Optional audio reference for speech-to-video. Provide audio_url (http(s) or data URL).", + ) # Video params block for extensibility video_params: VideoParams | None = Field(default=None, description="Optional video-specific parameters") diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index f1cd32a76..f05ce204b 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -46,6 +46,13 @@ class ReferenceVideo: data: list[Image.Image] +@dataclass +class ReferenceAudio: + """Reference audio file path for speech-to-video generation.""" + + path: str + + @dataclass class VideoGenerationArtifacts: """Normalized outputs and profiler metadata extracted from one request.""" @@ -104,6 +111,7 @@ class OmniOpenAIServingVideo: *, reference_image: ReferenceImage | None = None, reference_video: ReferenceVideo | None = None, + reference_audio: ReferenceAudio | None = None, ) -> VideoGenerationArtifacts: """Run the generation pipeline and extract video/audio/profiler outputs.""" prompt: OmniTextPrompt = OmniTextPrompt(prompt=request.prompt, modalities=["video"]) @@ -124,10 +132,15 @@ class OmniOpenAIServingVideo: target_size = (vp.width, vp.height) if input_image.size != target_size: input_image = input_image.resize(target_size, Image.Resampling.LANCZOS) + multi_modal_data: dict[str, Any] = {} if input_image is not None: - prompt["multi_modal_data"] = {"image": input_image} + multi_modal_data["image"] = input_image elif input_video is not None: - prompt["multi_modal_data"] = {"video": input_video} + multi_modal_data["video"] = input_video + if reference_audio is not None: + multi_modal_data["audio"] = reference_audio.path + if multi_modal_data: + prompt["multi_modal_data"] = multi_modal_data if vp.width is not None and vp.height is not None: gen_params.width = vp.width gen_params.height = vp.height @@ -228,12 +241,14 @@ class OmniOpenAIServingVideo: *, reference_image: ReferenceImage | None = None, reference_video: ReferenceVideo | None = None, + reference_audio: ReferenceAudio | None = None, ) -> VideoGenerationResponse: artifacts = await self._run_and_extract( request, reference_id, reference_image=reference_image, reference_video=reference_video, + reference_audio=reference_audio, ) video_codec_options = {"preset": "ultrafast", "threads": "0"} @@ -275,6 +290,7 @@ class OmniOpenAIServingVideo: *, reference_image: ReferenceImage | None = None, reference_video: ReferenceVideo | None = None, + reference_audio: ReferenceAudio | None = None, ) -> tuple[bytes, dict[str, float], float, VideoAction | None]: """Generate a video and return raw MP4 bytes, bypassing base64 encoding.""" artifacts = await self._run_and_extract( @@ -282,6 +298,7 @@ class OmniOpenAIServingVideo: reference_id, reference_image=reference_image, reference_video=reference_video, + reference_audio=reference_audio, ) if len(artifacts.videos) > 1: logger.warning( diff --git a/vllm_omni/entrypoints/openai/video_api_utils.py b/vllm_omni/entrypoints/openai/video_api_utils.py index d4cb38ea3..aa4eeaba5 100644 --- a/vllm_omni/entrypoints/openai/video_api_utils.py +++ b/vllm_omni/entrypoints/openai/video_api_utils.py @@ -186,6 +186,53 @@ async def decode_video_url( raise InvalidInputReferenceError("Invalid video_reference.video_url: must be an http(s) URL or data URL.") +async def decode_audio_url(audio_url: str) -> str: + """Decode an audio URL or data-URL to a temporary file path.""" + import tempfile + + audio_bytes: bytes | None = None + + if audio_url.startswith("data:audio"): + _, b64_data = audio_url.split(",", 1) + try: + audio_bytes = base64.b64decode(b64_data) + except (binascii.Error, ValueError) as exc: + raise InvalidInputReferenceError( + "Invalid audio_reference.audio_url: audio data is not valid base64." + ) from exc + elif audio_url.startswith(("http://", "https://")): + async with httpx.AsyncClient(timeout=60) as client: + try: + response = await client.get(audio_url) + response.raise_for_status() + except httpx.HTTPError as exc: + raise InvalidInputReferenceError( + "Invalid audio_reference.audio_url: failed to download audio." + ) from exc + audio_bytes = response.content + else: + raise InvalidInputReferenceError("Invalid audio_reference.audio_url: must be an http(s) URL or data URL.") + + if not audio_bytes: + raise InvalidInputReferenceError("Invalid audio_reference: audio data is empty.") + + suffix = ".wav" + if audio_url.startswith("data:audio/"): + mime = audio_url.split(";")[0].removeprefix("data:") + ext = mime.split("/")[-1] + if ext in ("mpeg", "mp3"): + suffix = ".mp3" + elif ext == "wav": + suffix = ".wav" + elif ext.isalnum() and len(ext) <= 8: + suffix = f".{ext}" + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) + tmp.write(audio_bytes) + tmp.close() + return tmp.name + + async def decode_input_reference( image_reference: ImageReference | None, video_reference: VideoReference | None,