From 9c00852d85810aba72eadb679d0f88c8eb56466f Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Thu, 12 Feb 2026 14:28:32 +0800 Subject: [PATCH] [Feature] Support Wan2.2 T2V and I2V Online Serving with OpenAI /v1/videos API (#1073) Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> Signed-off-by: SamitHuang <285365963@qq.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../online_serving/image_to_video/README.md | 83 +++++ .../image_to_video/run_curl_image_to_video.sh | 33 ++ .../image_to_video/run_server.sh | 25 ++ .../online_serving/text_to_video/README.md | 116 +++++++ .../text_to_video/run_curl_text_to_video.sh | 20 ++ .../text_to_video/run_server.sh | 31 ++ .../openai_api/test_video_server.py | 317 +++++++++++++++++ .../models/wan2_2/pipeline_wan2_2.py | 17 +- .../models/wan2_2/pipeline_wan2_2_i2v.py | 15 +- vllm_omni/entrypoints/openai/api_server.py | 123 +++++++ .../entrypoints/openai/protocol/__init__.py | 10 + .../entrypoints/openai/protocol/videos.py | 168 +++++++++ vllm_omni/entrypoints/openai/serving_video.py | 325 ++++++++++++++++++ .../entrypoints/openai/video_api_utils.py | 161 +++++++++ 14 files changed, 1436 insertions(+), 8 deletions(-) create mode 100644 examples/online_serving/image_to_video/README.md create mode 100644 examples/online_serving/image_to_video/run_curl_image_to_video.sh create mode 100644 examples/online_serving/image_to_video/run_server.sh create mode 100644 examples/online_serving/text_to_video/README.md create mode 100644 examples/online_serving/text_to_video/run_curl_text_to_video.sh create mode 100644 examples/online_serving/text_to_video/run_server.sh create mode 100644 tests/entrypoints/openai_api/test_video_server.py create mode 100644 vllm_omni/entrypoints/openai/protocol/videos.py create mode 100644 vllm_omni/entrypoints/openai/serving_video.py create mode 100644 vllm_omni/entrypoints/openai/video_api_utils.py diff --git a/examples/online_serving/image_to_video/README.md b/examples/online_serving/image_to_video/README.md new file mode 100644 index 000000000..7e67a15e4 --- /dev/null +++ b/examples/online_serving/image_to_video/README.md @@ -0,0 +1,83 @@ +# Image-To-Video + +This example demonstrates how to deploy the Wan2.2 image-to-video model for online video generation using vLLM-Omni. + +## Start Server + +### Basic Start + +```bash +vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni --port 8091 +``` + +### Start with Parameters + +Or use the startup script: + +```bash +bash run_server.sh +``` + +The script allows overriding: +- `MODEL` (default: `Wan-AI/Wan2.2-I2V-A14B-Diffusers`) +- `PORT` (default: `8091`) +- `BOUNDARY_RATIO` (default: `0.875`) +- `FLOW_SHIFT` (default: `12.0`) +- `CACHE_BACKEND` (default: `none`) +- `ENABLE_CACHE_DIT_SUMMARY` (default: `0`) + +## API Calls + +### Method 1: Using curl + +```bash +# Basic image-to-video generation +bash run_curl_image_to_video.sh + +# Or execute directly (OpenAI-style multipart) +curl -X POST http://localhost:8091/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A bear playing with yarn, smooth motion" \ + -F "negative_prompt=low quality, blurry, static" \ + -F "input_reference=@/path/to/qwen-bear.png" \ + -F "width=832" \ + -F "height=480" \ + -F "num_frames=33" \ + -F "fps=16" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=1.0" \ + -F "guidance_scale_2=1.0" \ + -F "boundary_ratio=0.875" \ + -F "flow_shift=12.0" \ + -F "seed=42" | jq -r '.data[0].b64_json' | base64 -d > wan22_i2v_output.mp4 +``` + +## Request Format + +### Required Fields + +```bash +curl -X POST http://localhost:8091/v1/videos \ + -F "prompt=A bear playing with yarn, smooth motion" \ + -F "negative_prompt=low quality, blurry, static" \ + -F "input_reference=@/path/to/qwen-bear.png" +``` + +### Generation with Parameters + +```bash +curl -X POST http://localhost:8091/v1/videos \ + -F "prompt=A bear playing with yarn, smooth motion" \ + -F "negative_prompt=low quality, blurry, static" \ + -F "input_reference=@/path/to/qwen-bear.png" \ + -F "width=832" \ + -F "height=480" \ + -F "num_frames=33" \ + -F "fps=16" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=1.0" \ + -F "guidance_scale_2=1.0" \ + -F "boundary_ratio=0.875" \ + -F "flow_shift=12.0" \ + -F "seed=42" +``` diff --git a/examples/online_serving/image_to_video/run_curl_image_to_video.sh b/examples/online_serving/image_to_video/run_curl_image_to_video.sh new file mode 100644 index 000000000..5e6faf477 --- /dev/null +++ b/examples/online_serving/image_to_video/run_curl_image_to_video.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Wan2.2 image-to-video curl example (OpenAI-style multipart) + +INPUT_IMAGE="${INPUT_IMAGE:-../../offline_inference/image_to_video/qwen-bear.png}" +OUTPUT_PATH="${OUTPUT_PATH:-wan22_i2v_output.mp4}" +NEGATIVE_PROMPT="${NEGATIVE_PROMPT:-}" + +if [ ! -f "$INPUT_IMAGE" ]; then + echo "Input image not found: $INPUT_IMAGE" + exit 1 +fi + +NEGATIVE_PROMPT_FLAG="" +if [ -n "$NEGATIVE_PROMPT" ]; then + NEGATIVE_PROMPT_FLAG="-F negative_prompt=${NEGATIVE_PROMPT}" +fi + +curl -X POST http://localhost:8099/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A bear playing with yarn, smooth motion" \ + $NEGATIVE_PROMPT_FLAG \ + -F "input_reference=@${INPUT_IMAGE}" \ + -F "seconds=2" \ + -F "size=832x480" \ + -F "fps=16" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=1.0" \ + -F "guidance_scale_2=1.0" \ + -F "boundary_ratio=0.875" \ + -F "flow_shift=12.0" \ + -F "seed=42" | jq -r '.data[0].b64_json' | base64 -d > "${OUTPUT_PATH}" + +echo "Saved video to ${OUTPUT_PATH}" diff --git a/examples/online_serving/image_to_video/run_server.sh b/examples/online_serving/image_to_video/run_server.sh new file mode 100644 index 000000000..a488f5340 --- /dev/null +++ b/examples/online_serving/image_to_video/run_server.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Wan2.2 image-to-video server start script + +MODEL="${MODEL:-Wan-AI/Wan2.2-I2V-A14B-Diffusers}" +PORT="${PORT:-8099}" +CACHE_BACKEND="${CACHE_BACKEND:-none}" +ENABLE_CACHE_DIT_SUMMARY="${ENABLE_CACHE_DIT_SUMMARY:-0}" + +echo "Starting Wan2.2 I2V server..." +echo "Model: $MODEL" +echo "Port: $PORT" +echo "Cache backend: $CACHE_BACKEND" +if [ "$ENABLE_CACHE_DIT_SUMMARY" != "0" ]; then + echo "Cache-DiT summary: enabled" +fi + +CACHE_BACKEND_FLAG="" +if [ "$CACHE_BACKEND" != "none" ]; then + CACHE_BACKEND_FLAG="--cache-backend $CACHE_BACKEND" +fi + +vllm serve "$MODEL" --omni \ + --port "$PORT" \ + $CACHE_BACKEND_FLAG \ + $(if [ "$ENABLE_CACHE_DIT_SUMMARY" != "0" ]; then echo "--enable-cache-dit-summary"; fi) diff --git a/examples/online_serving/text_to_video/README.md b/examples/online_serving/text_to_video/README.md new file mode 100644 index 000000000..a8c69e87b --- /dev/null +++ b/examples/online_serving/text_to_video/README.md @@ -0,0 +1,116 @@ +# Text-To-Video + +This example demonstrates how to deploy the Wan2.2 text-to-video model for online video generation using vLLM-Omni. + +## Start Server + +### Basic Start + +```bash +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091 +``` + +### Start with Parameters + +Or use the startup script: + +```bash +bash run_server.sh +``` + +The script allows overriding: +- `MODEL` (default: `Wan-AI/Wan2.2-T2V-A14B-Diffusers`) +- `PORT` (default: `8091`) +- `BOUNDARY_RATIO` (default: `0.875`) +- `FLOW_SHIFT` (default: `5.0`) +- `CACHE_BACKEND` (default: `none`) +- `ENABLE_CACHE_DIT_SUMMARY` (default: `0`) + +## API Calls + +### Method 1: Using curl + +```bash +# Basic text-to-video generation +bash run_curl_text_to_video.sh + +# Or execute directly (OpenAI-style multipart) +curl -s http://localhost:8091/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \ + -F "width=832" \ + -F "height=480" \ + -F "num_frames=33" \ + -F "negative_prompt=色调艳丽 ，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" \ + -F "fps=16" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.0" \ + -F "guidance_scale_2=4.0" \ + -F "boundary_ratio=0.875" \ + -F "seed=42" | jq -r '.data[0].b64_json' | base64 -d > wan22_output.mp4 +``` + +## Request Format + +### Simple Text-to-Video Generation + +```bash +curl -X POST http://localhost:8091/v1/videos \ + -F "prompt=A cinematic view of a futuristic city at sunset" +``` + +### Generation with Parameters + +```bash +curl -X POST http://localhost:8091/v1/videos \ + -F "prompt=A cinematic view of a futuristic city at sunset" \ + -F "width=832" \ + -F "height=480" \ + -F "num_frames=33" \ + -F "negative_prompt=low quality, blurry, static" \ + -F "fps=16" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.0" \ + -F "guidance_scale_2=4.0" \ + -F "boundary_ratio=0.875" \ + -F "flow_shift=5.0" \ + -F "seed=42" +``` + +## Generation Parameters + +| Parameter | Type | Default | Description | +| --------------------- | ------ | ------- | ------------------------------------------------ | +| `prompt` | str | - | Text description of the desired video | +| `negative_prompt` | str | None | Negative prompt | +| `n` | int | 1 | Number of videos to generate | +| `width` | int | None | Video width in pixels | +| `height` | int | None | Video height in pixels | +| `num_frames` | int | None | Number of frames to generate | +| `fps` | int | None | Frames per second for output video | +| `num_inference_steps` | int | None | Number of denoising steps | +| `guidance_scale` | float | None | CFG guidance scale (low-noise stage) | +| `guidance_scale_2` | float | None | CFG guidance scale (high-noise stage, Wan2.2) | +| `boundary_ratio` | float | None | Boundary split ratio for low/high DiT (Wan2.2) | +| `flow_shift` | float | None | Scheduler flow shift (Wan2.2) | +| `seed` | int | None | Random seed (reproducible) | +| `lora` | object | None | LoRA configuration | +| `extra_body` | object | None | Model-specific extra parameters | + +## Response Format + +```json +{ + "created": 1234567890, + "data": [ + { "b64_json": "" } + ] +} +``` + +## Extract Video + +```bash +# Extract base64 from response and decode to video +cat response.json | jq -r '.data[0].b64_json' | base64 -d > wan22_output.mp4 +``` diff --git a/examples/online_serving/text_to_video/run_curl_text_to_video.sh b/examples/online_serving/text_to_video/run_curl_text_to_video.sh new file mode 100644 index 000000000..43889c398 --- /dev/null +++ b/examples/online_serving/text_to_video/run_curl_text_to_video.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Wan2.2 text-to-video curl example + +OUTPUT_PATH="wan22_output.mp4" + +curl -X POST http://localhost:8098/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \ + -F "seconds=2" \ + -F "size=832x480" \ + -F "negative_prompt=色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" \ + -F "fps=16" \ + -F "num_inference_steps=40" \ + -F "guidance_scale=4.0" \ + -F "guidance_scale_2=4.0" \ + -F "boundary_ratio=0.875" \ + -F "flow_shift=5.0" \ + -F "seed=42" | jq -r '.data[0].b64_json' | base64 -d > "${OUTPUT_PATH}" + +echo "Saved video to ${OUTPUT_PATH}" diff --git a/examples/online_serving/text_to_video/run_server.sh b/examples/online_serving/text_to_video/run_server.sh new file mode 100644 index 000000000..2d7ce1d7a --- /dev/null +++ b/examples/online_serving/text_to_video/run_server.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Wan2.2 online serving startup script + +MODEL="${MODEL:-Wan-AI/Wan2.2-T2V-A14B-Diffusers}" +PORT="${PORT:-8098}" +BOUNDARY_RATIO="${BOUNDARY_RATIO:-0.875}" +FLOW_SHIFT="${FLOW_SHIFT:-5.0}" +CACHE_BACKEND="${CACHE_BACKEND:-none}" +ENABLE_CACHE_DIT_SUMMARY="${ENABLE_CACHE_DIT_SUMMARY:-0}" + +echo "Starting Wan2.2 server..." +echo "Model: $MODEL" +echo "Port: $PORT" +echo "Boundary ratio: $BOUNDARY_RATIO" +echo "Flow shift: $FLOW_SHIFT" +echo "Cache backend: $CACHE_BACKEND" +if [ "$ENABLE_CACHE_DIT_SUMMARY" != "0" ]; then + echo "Cache-DiT summary: enabled" +fi + +CACHE_BACKEND_FLAG="" +if [ "$CACHE_BACKEND" != "none" ]; then + CACHE_BACKEND_FLAG="--cache-backend $CACHE_BACKEND" +fi + +vllm serve "$MODEL" --omni \ + --port "$PORT" \ + --boundary-ratio "$BOUNDARY_RATIO" \ + --flow-shift "$FLOW_SHIFT" \ + $CACHE_BACKEND_FLAG \ + $(if [ "$ENABLE_CACHE_DIT_SUMMARY" != "0" ]; then echo "--enable-cache-dit-summary"; fi) diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py new file mode 100644 index 000000000..6df304ce5 --- /dev/null +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -0,0 +1,317 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Unit tests for OpenAI-compatible video generation endpoints. +""" + +import io +from unittest.mock import patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from PIL import Image +from pydantic import ValidationError + +from vllm_omni.entrypoints.openai.api_server import router +from vllm_omni.entrypoints.openai.protocol.videos import VideoGenerationRequest, VideoResponseFormat +from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo + + +class MockVideoResult: + def __init__(self, videos): + self.multimodal_output = {"video": videos} + + +class FakeAsyncOmni: + def __init__(self): + self.stage_list = ["diffusion"] + self.captured_prompt = None + self.captured_sampling_params_list = None + + async def generate(self, prompt, request_id, sampling_params_list): + self.captured_prompt = prompt + self.captured_sampling_params_list = sampling_params_list + num_outputs = sampling_params_list[0].num_outputs_per_prompt + videos = [object() for _ in range(num_outputs)] + yield MockVideoResult(videos) + + +@pytest.fixture +def test_client(): + app = FastAPI() + app.include_router(router) + app.state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion( + diffusion_engine=FakeAsyncOmni(), + model_name="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + ) + return TestClient(app) + + +def _make_test_image_bytes(size=(64, 64)) -> bytes: + image = Image.new("RGB", size, color="blue") + buf = io.BytesIO() + image.save(buf, format="PNG") + return buf.getvalue() + + +def test_t2v_video_generation_form(test_client): + fps_values = [] + + def _fake_encode(video, fps): + fps_values.append(fps) + return "Zg==" + + with patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + side_effect=_fake_encode, + ): + response = test_client.post( + "/v1/videos", + data={ + "prompt": "A cat runs across the street.", + "size": "640x360", + "seconds": "2", + "fps": "12", + "n": "2", + }, + ) + + assert response.status_code == 200 + data = response.json() + assert "data" in data and len(data["data"]) == 2 + assert all(item["b64_json"] == "Zg==" for item in data["data"]) + + engine = test_client.app.state.openai_serving_video._engine_client + captured = engine.captured_sampling_params_list[0] + assert captured.num_outputs_per_prompt == 2 + assert captured.width == 640 + assert captured.height == 360 + assert captured.num_frames == 24 + assert captured.fps == 12 + assert fps_values == [12, 12] + + +def test_i2v_video_generation_form(test_client): + image_bytes = _make_test_image_bytes((48, 32)) + + with patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + return_value="Zg==", + ): + response = test_client.post( + "/v1/videos", + data={"prompt": "A bear playing with yarn."}, + files={"input_reference": ("input.png", image_bytes, "image/png")}, + ) + + assert response.status_code == 200 + data = response.json() + assert "data" in data and len(data["data"]) == 1 + assert data["data"][0]["b64_json"] == "Zg==" + + engine = test_client.app.state.openai_serving_video._engine_client + prompt = engine.captured_prompt + assert "multi_modal_data" in prompt + assert "image" in prompt["multi_modal_data"] + input_image = prompt["multi_modal_data"]["image"] + assert isinstance(input_image, Image.Image) + assert input_image.size == (48, 32) + + +def test_seconds_defaults_fps_and_frames(test_client): + fps_values = [] + + def _fake_encode(video, fps): + fps_values.append(fps) + return "Zg==" + + with patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + side_effect=_fake_encode, + ): + response = test_client.post( + "/v1/videos", + data={ + "prompt": "A bird flying.", + "seconds": "3", + }, + ) + + assert response.status_code == 200 + engine = test_client.app.state.openai_serving_video._engine_client + captured = engine.captured_sampling_params_list[0] + assert captured.num_frames == 72 + assert captured.fps == 24 + assert fps_values == [24] + + +def test_size_param_sets_width_height(test_client): + with patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + return_value="Zg==", + ): + response = test_client.post( + "/v1/videos", + data={ + "prompt": "size test", + "size": "320x240", + }, + ) + + assert response.status_code == 200 + engine = test_client.app.state.openai_serving_video._engine_client + captured = engine.captured_sampling_params_list[0] + assert captured.width == 320 + assert captured.height == 240 + + +def test_sampling_params_pass_through(test_client): + with patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + return_value="Zg==", + ): + response = test_client.post( + "/v1/videos", + data={ + "prompt": "param pass", + "num_inference_steps": "30", + "guidance_scale": "6.5", + "guidance_scale_2": "8.0", + "true_cfg_scale": "4.0", + "boundary_ratio": "0.7", + "flow_shift": "0.25", + }, + ) + + assert response.status_code == 200 + engine = test_client.app.state.openai_serving_video._engine_client + captured = engine.captured_sampling_params_list[0] + assert captured.num_inference_steps == 30 + assert captured.guidance_scale == 6.5 + assert captured.guidance_scale_2 == 8.0 + assert captured.true_cfg_scale == 4.0 + assert captured.boundary_ratio == 0.7 + assert captured.extra_args["flow_shift"] == 0.25 + + +def test_missing_handler_returns_503(): + app = FastAPI() + app.include_router(router) + app.state.openai_serving_video = None + client = TestClient(app) + + response = client.post( + "/v1/videos", + data={"prompt": "no handler"}, + ) + assert response.status_code == 503 + assert "not initialized" in response.json()["detail"].lower() + + +def test_missing_prompt_returns_422(test_client): + response = test_client.post( + "/v1/videos", + data={"size": "320x240"}, + ) + assert response.status_code == 422 + + +def test_invalid_size_format_raises_validation_error(test_client): + with pytest.raises(ValidationError): + test_client.post( + "/v1/videos", + data={"prompt": "bad size", "size": "invalid"}, + ) + + +def test_invalid_size_parse_returns_500(test_client): + response = test_client.post( + "/v1/videos", + data={"prompt": "bad size", "size": "640x"}, + ) + assert response.status_code == 500 + assert "video generation failed" in response.json()["detail"].lower() + + +def test_invalid_response_format_raises_validation_error(test_client): + with pytest.raises(ValidationError): + test_client.post( + "/v1/videos", + data={"prompt": "bad format", "response_format": "url"}, + ) + + +def test_invalid_seconds_returns_422(test_client): + response = test_client.post( + "/v1/videos", + data={"prompt": "bad seconds", "seconds": "abc"}, + ) + assert response.status_code == 422 + + +def test_invalid_n_raises_validation_error(test_client): + with pytest.raises(ValidationError): + test_client.post( + "/v1/videos", + data={"prompt": "bad n", "n": "0"}, + ) + + with pytest.raises(ValidationError): + test_client.post( + "/v1/videos", + data={"prompt": "bad n", "n": "5"}, + ) + + +def test_negative_prompt_and_seed_pass_through(test_client): + with patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + return_value="Zg==", + ): + response = test_client.post( + "/v1/videos", + data={ + "prompt": "snowy mountain", + "negative_prompt": "blurry", + "seed": "123", + }, + ) + + assert response.status_code == 200 + engine = test_client.app.state.openai_serving_video._engine_client + captured_prompt = engine.captured_prompt + captured_params = engine.captured_sampling_params_list[0] + assert captured_prompt["negative_prompt"] == "blurry" + assert captured_params.seed == 123 + + +def test_invalid_lora_returns_400(test_client): + response = test_client.post( + "/v1/videos", + data={ + "prompt": "lora test", + "lora": '{"name": "bad-lora"}', + }, + ) + assert response.status_code == 400 + assert "lora" in response.json()["detail"].lower() + + +def test_video_request_validation(): + req = VideoGenerationRequest(prompt="test") + assert req.prompt == "test" + assert req.n == 1 + assert req.response_format == VideoResponseFormat.B64_JSON + + with pytest.raises(ValueError): + VideoGenerationRequest(prompt="test", response_format="url") + + with pytest.raises(ValueError): + VideoGenerationRequest(prompt="test", size="invalid") + + with pytest.raises(ValueError): + VideoGenerationRequest(prompt="test", seconds="abc") + + with pytest.raises(ValueError): + VideoGenerationRequest(prompt="test", n=0) diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index b902bc692..23eebc7fd 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -384,6 +384,13 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin): self._guidance_scale = guidance_low self._guidance_scale_2 = guidance_high + # Prefer engine-configured boundary_ratio, but allow per-request fallback. + boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else req.sampling_params.boundary_ratio + + if boundary_ratio is None: + boundary_ratio = 0.875 + logger.warning("boundary_ratio is required for T2V generation. using default value 0.875") + # validate shapes self.check_inputs( prompt=prompt, @@ -392,7 +399,8 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin): width=width, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, - guidance_scale_2=guidance_high if self.boundary_ratio is not None else None, + guidance_scale_2=guidance_high if boundary_ratio is not None else None, + boundary_ratio=boundary_ratio, ) if num_frames % self.vae_scale_factor_temporal != 1: @@ -440,8 +448,8 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin): timesteps = self.scheduler.timesteps self._num_timesteps = len(timesteps) boundary_timestep = None - if self.boundary_ratio is not None: - boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps + if boundary_ratio is not None: + boundary_timestep = boundary_ratio * self.scheduler.config.num_train_timesteps # Handle I2V mode when expand_timesteps=True and image is provided multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None @@ -780,6 +788,7 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin): prompt_embeds=None, negative_prompt_embeds=None, guidance_scale_2=None, + boundary_ratio=None, ): if height % 16 != 0 or width % 16 != 0: raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") @@ -806,5 +815,5 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin): ): raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}") - if self.boundary_ratio is None and guidance_scale_2 is not None: + if boundary_ratio is None and guidance_scale_2 is not None: raise ValueError("`guidance_scale_2` is only supported when `boundary_ratio` is set.") diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index 1aed9b75d..4b32d5a5f 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -354,6 +354,11 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): self._guidance_scale = guidance_low self._guidance_scale_2 = guidance_high + boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else req.sampling_params.boundary_ratio + if boundary_ratio is None: + boundary_ratio = 0.875 + logger.warning("boundary_ratio is required for I2V generation. using default value 0.875") + # Validate inputs self.check_inputs( prompt=prompt, @@ -364,7 +369,8 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, image_embeds=image_embeds, - guidance_scale_2=guidance_high if self.boundary_ratio is not None else None, + guidance_scale_2=guidance_high if boundary_ratio is not None else None, + boundary_ratio=boundary_ratio, ) # Adjust num_frames to be compatible with VAE temporal scaling @@ -417,8 +423,8 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): self._num_timesteps = len(timesteps) boundary_timestep = None - if self.boundary_ratio is not None: - boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps + if boundary_ratio is not None: + boundary_timestep = boundary_ratio * self.scheduler.config.num_train_timesteps # Prepare latents (use out_channels=16 for VAE latent, not in_channels=36) num_channels_latents = self.transformer.config.out_channels @@ -752,6 +758,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): negative_prompt_embeds=None, image_embeds=None, guidance_scale_2=None, + boundary_ratio=None, ): if image is None and image_embeds is None: raise ValueError("Provide either `image` or `image_embeds`. Cannot leave both undefined.") @@ -773,7 +780,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): if prompt is None and prompt_embeds is None: raise ValueError("Provide either `prompt` or `prompt_embeds`.") - if self.boundary_ratio is None and guidance_scale_2 is not None: + if boundary_ratio is None and guidance_scale_2 is not None: raise ValueError("`guidance_scale_2` is only supported when `boundary_ratio` is set.") def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index c4fc970c4..521bdf0f3 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -85,8 +85,13 @@ from vllm_omni.entrypoints.openai.protocol.images import ( ImageGenerationRequest, ImageGenerationResponse, ) +from vllm_omni.entrypoints.openai.protocol.videos import ( + VideoGenerationRequest, + VideoGenerationResponse, +) from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.utils import stable_lora_int_id @@ -406,6 +411,13 @@ async def omni_init_app_state( model_name=model_name, ) + diffusion_stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None + state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion( + diffusion_engine=engine_client, # type: ignore + model_name=model_name, + stage_configs=diffusion_stage_configs, + ) + state.enable_server_load_tracking = getattr(args, "enable_server_load_tracking", False) state.server_load_metrics = 0 logger.info("Pure diffusion API server initialized for model: %s", model_name) @@ -684,10 +696,20 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger ) + state.openai_serving_video = OmniOpenAIServingVideo( + engine_client, + model_name=served_model_names[0] if served_model_names else None, + stage_configs=state.stage_configs, + ) + state.enable_server_load_tracking = args.enable_server_load_tracking state.server_load_metrics = 0 +def Omnivideo(request: Request) -> OmniOpenAIServingVideo | None: + return request.app.state.openai_serving_video + + def Omnichat(request: Request) -> OmniOpenAIServingChat | None: return request.app.state.openai_serving_chat @@ -1456,3 +1478,104 @@ def apply_stage_default_sampling_params( for param_name, param_value in stage_defaults.items(): if hasattr(sampling_params, param_name): setattr(sampling_params, param_name, param_value) + + +async def _run_video_generation( + request: VideoGenerationRequest, + raw_request: Request, + *, + input_reference_bytes: bytes | None = None, +) -> VideoGenerationResponse: + handler = Omnivideo(raw_request) + if handler is None: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, + detail="Video generation handler not initialized.", + ) + logger.info("Video generation handler: %s", type(handler).__name__) + try: + return await handler.generate_videos(request, raw_request, input_reference_bytes=input_reference_bytes) + except HTTPException: + raise + except Exception as e: + logger.exception("Video generation failed: %s", e) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + detail=f"Video generation failed: {str(e)}", + ) + + +def _parse_form_json(value: str | None) -> Any: + if value is None or value == "": + return None + try: + return json.loads(value) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Invalid JSON in form field.", + ) from exc + + +@router.post( + "/v1/videos", + responses={ + HTTPStatus.OK.value: {"model": VideoGenerationResponse}, + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.SERVICE_UNAVAILABLE.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def create_video( + raw_request: Request, + prompt: str = Form(...), + input_reference: UploadFile | None = File(default=None), + model: str | None = Form(default=None), + n: int | None = Form(default=None), + seconds: int | None = Form(default=None), + size: str | None = Form(default=None), + response_format: str | None = Form(default=None), + user: str | None = Form(default=None), + width: int | None = Form(default=None), + height: int | None = Form(default=None), + num_frames: int | None = Form(default=None), + fps: int | None = Form(default=None), + num_inference_steps: int | None = Form(default=None), + guidance_scale: float | None = Form(default=None), + guidance_scale_2: float | None = Form(default=None), + boundary_ratio: float | None = Form(default=None), + flow_shift: float | None = Form(default=None), + true_cfg_scale: float | None = Form(default=None), + seed: int | None = Form(default=None), + negative_prompt: str | None = Form(default=None), + lora: str | None = Form(default=None), +) -> VideoGenerationResponse: + """OpenAI-style video create endpoint (multipart form-data).""" + input_reference_bytes = await input_reference.read() if input_reference is not None else None + + request_data: dict[str, Any] = { + "prompt": prompt, + "model": model, + "seconds": seconds, + "size": size, + "user": user, + "response_format": response_format, + "input_reference": None, + "n": n, + "width": width, + "height": height, + "num_frames": num_frames, + "fps": fps, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "guidance_scale_2": guidance_scale_2, + "boundary_ratio": boundary_ratio, + "flow_shift": flow_shift, + "true_cfg_scale": true_cfg_scale, + "seed": seed, + "negative_prompt": negative_prompt, + "lora": _parse_form_json(lora), + } + request_data = {k: v for k, v in request_data.items() if v is not None} + request = VideoGenerationRequest(**request_data) + return await _run_video_generation(request, raw_request, input_reference_bytes=input_reference_bytes) diff --git a/vllm_omni/entrypoints/openai/protocol/__init__.py b/vllm_omni/entrypoints/openai/protocol/__init__.py index da65e1817..3788ee2af 100644 --- a/vllm_omni/entrypoints/openai/protocol/__init__.py +++ b/vllm_omni/entrypoints/openai/protocol/__init__.py @@ -8,11 +8,21 @@ from vllm_omni.entrypoints.openai.protocol.images import ( ImageGenerationResponse, ResponseFormat, ) +from vllm_omni.entrypoints.openai.protocol.videos import ( + VideoData, + VideoGenerationRequest, + VideoGenerationResponse, + VideoResponseFormat, +) __all__ = [ "ImageData", "ImageGenerationRequest", "ImageGenerationResponse", "ResponseFormat", + "VideoData", + "VideoGenerationRequest", + "VideoGenerationResponse", + "VideoResponseFormat", "OmniChatCompletionStreamResponse", ] diff --git a/vllm_omni/entrypoints/openai/protocol/videos.py b/vllm_omni/entrypoints/openai/protocol/videos.py new file mode 100644 index 000000000..49fa43189 --- /dev/null +++ b/vllm_omni/entrypoints/openai/protocol/videos.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +OpenAI-compatible protocol definitions for video generation. + +This module provides Pydantic models for a video generation endpoint that +mirrors the OpenAI Images API shape, with vllm-omni extensions for diffusion +video models (e.g., Wan2.2). +""" + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field, field_validator + + +class VideoResponseFormat(str, Enum): + """Video response format.""" + + B64_JSON = "b64_json" + URL = "url" # Not implemented in PoC + + +class VideoParams(BaseModel): + """Optional block for video-specific parameters.""" + + width: int | None = Field(default=None, ge=1, description="Video width in pixels") + height: int | None = Field(default=None, ge=1, description="Video height in pixels") + num_frames: int | None = Field(default=None, ge=1, description="Number of frames") + fps: int | None = Field(default=None, ge=1, description="Frames per second for output video") + + +class VideoGenerationRequest(BaseModel): + """ + OpenAI-style video generation request. + + Follows the OpenAI Images API conventions with extensions for video. + """ + + prompt: str = Field(..., description="Text description of the desired video(s)") + + # OpenAI standard fields + model: str | None = Field( + default=None, + description="Model to use (optional, uses server's configured model if omitted)", + ) + n: int = Field(default=1, ge=1, le=4, description="Number of videos to generate") + seconds: int | str | None = Field( + default=None, + description="Clip duration in seconds (OpenAI-compatible, e.g., 4, 8, 12)", + ) + size: str | None = Field( + default=None, + description="Video dimensions in WIDTHxHEIGHT format (e.g., '1280x720')", + ) + response_format: VideoResponseFormat = Field( + default=VideoResponseFormat.B64_JSON, description="Format of the returned video" + ) + user: str | None = Field(default=None, description="User identifier for tracking") + stream: bool | None = Field(default=False, description="Whether to stream outputs (not yet supported)") + + # Video params block for extensibility + video_params: VideoParams | None = Field(default=None, description="Optional video-specific parameters") + + # Image-to-video input (OpenAI-style name) + input_reference: str | None = Field( + default=None, + description="Image input for I2V; provided via multipart form-data as a file upload.", + ) + + # Video-specific fields (top-level for OpenAI-style compatibility) + width: int | None = Field(default=None, ge=1, description="Video width in pixels") + height: int | None = Field(default=None, ge=1, description="Video height in pixels") + num_frames: int | None = Field(default=None, ge=1, description="Number of frames to generate") + fps: int | None = Field(default=None, ge=1, description="Frames per second for output video") + + # vllm-omni extensions for diffusion control + negative_prompt: str | None = Field(default=None, description="Text describing what to avoid in the video") + num_inference_steps: int | None = Field( + default=None, + ge=1, + le=200, + description="Number of diffusion sampling steps (uses model defaults if not specified)", + ) + guidance_scale: float | None = Field( + default=None, + ge=0.0, + le=20.0, + description="Classifier-free guidance scale (uses model defaults if not specified)", + ) + guidance_scale_2: float | None = Field( + default=None, + ge=0.0, + le=20.0, + description="High-noise CFG scale for video models (Wan2.2)", + ) + boundary_ratio: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description="Boundary split ratio for low/high DiT (Wan2.2)", + ) + flow_shift: float | None = Field( + default=None, + description="Scheduler flow_shift for video models (Wan2.2)", + ) + true_cfg_scale: float | None = Field( + default=None, + ge=0.0, + le=20.0, + description="True CFG scale (model-specific parameter, may be ignored if not supported)", + ) + seed: int | None = Field(default=None, description="Random seed for reproducibility") + + # vllm-omni extension for per-request LoRA. + lora: dict[str, Any] | None = Field( + default=None, + description=( + "Optional LoRA adapter for this request. Expected shape: " + "{name/path/scale/int_id}. Field names are flexible " + "(e.g. name|lora_name|adapter, path|lora_path|local_path, " + "scale|lora_scale, int_id|lora_int_id)." + ), + ) + + @field_validator("size") + @classmethod + def validate_size(cls, v): + if v is None: + return None + if not isinstance(v, str) or "x" not in v: + raise ValueError("size must be in format 'WIDTHxHEIGHT' (e.g., '1280x720')") + return v + + @field_validator("response_format") + @classmethod + def validate_response_format(cls, v): + if v is not None and v != VideoResponseFormat.B64_JSON: + raise ValueError(f"Only 'b64_json' response format is supported, got: {v}") + return v + + @field_validator("seconds") + @classmethod + def validate_seconds(cls, v): + if v is None: + return None + if isinstance(v, str): + if not v.isdigit(): + raise ValueError("seconds must be an integer or numeric string") + return int(v) + if isinstance(v, int): + return v + raise ValueError("seconds must be an integer or numeric string") + + +class VideoData(BaseModel): + """Single generated video data.""" + + b64_json: str | None = Field(default=None, description="Base64-encoded MP4 video") + url: str | None = Field(default=None, description="Video URL (not implemented)") + revised_prompt: str | None = Field(default=None, description="Revised prompt (OpenAI compatibility, always null)") + + +class VideoGenerationResponse(BaseModel): + """OpenAI-style video generation response.""" + + created: int = Field(..., description="Unix timestamp of when the generation completed") + data: list[VideoData] = Field(..., description="Array of generated videos") diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py new file mode 100644 index 000000000..c5140f24a --- /dev/null +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import time +import uuid +from http import HTTPStatus +from typing import Any, cast + +from fastapi import HTTPException, Request +from PIL import Image +from vllm.engine.protocol import EngineClient +from vllm.logger import init_logger + +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.openai.image_api_utils import parse_size +from vllm_omni.entrypoints.openai.protocol.videos import ( + VideoData, + VideoGenerationRequest, + VideoGenerationResponse, +) +from vllm_omni.entrypoints.openai.video_api_utils import decode_input_reference, encode_video_base64 +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt +from vllm_omni.lora.request import LoRARequest +from vllm_omni.lora.utils import stable_lora_int_id + +logger = init_logger(__name__) + + +class OmniOpenAIServingVideo: + """OpenAI-style video generation handler for omni diffusion models.""" + + def __init__( + self, + engine_client: EngineClient, + model_name: str | None = None, + stage_configs: list[Any] | None = None, + ) -> None: + self._engine_client = engine_client + self._model_name = model_name + self._stage_configs = stage_configs + + @classmethod + def for_diffusion( + cls, + diffusion_engine: EngineClient, + model_name: str, + stage_configs: list[Any] | None = None, + ) -> OmniOpenAIServingVideo: + return cls( + diffusion_engine, + model_name=model_name, + stage_configs=stage_configs, + ) + + async def generate_videos( + self, + request: VideoGenerationRequest, + raw_request: Request | None = None, + *, + input_reference_bytes: bytes | None = None, + ) -> VideoGenerationResponse: + if request.stream: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Streaming video generation is not supported yet.", + ) + + request_id = f"video_gen_{uuid.uuid4().hex}" + model_name = self._resolve_model_name(raw_request) + + if request.model is not None and model_name is not None and request.model != model_name: + logger.warning( + "Model mismatch: request specifies '%s' but server is running '%s'. Using server model.", + request.model, + model_name, + ) + + prompt: OmniTextPrompt = {"prompt": request.prompt} + if request.negative_prompt is not None: + prompt["negative_prompt"] = request.negative_prompt + + input_image = None + try: + input_image = decode_input_reference(request.input_reference, input_reference_bytes) + except ValueError as exc: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail=str(exc), + ) from exc + if input_image is not None: + prompt["multi_modal_data"] = {"image": input_image} + + gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n) + + width, height, num_frames, fps = self._resolve_video_params(request) + if width is not None and height is not None: + gen_params.width = width + gen_params.height = height + if num_frames is not None: + gen_params.num_frames = num_frames + if fps is not None: + gen_params.fps = fps + + if request.num_inference_steps is not None: + gen_params.num_inference_steps = request.num_inference_steps + if request.guidance_scale is not None: + gen_params.guidance_scale = request.guidance_scale + if request.guidance_scale_2 is not None: + gen_params.guidance_scale_2 = request.guidance_scale_2 + if request.true_cfg_scale is not None: + gen_params.true_cfg_scale = request.true_cfg_scale + if request.seed is not None: + gen_params.seed = request.seed + if request.boundary_ratio is not None: + gen_params.boundary_ratio = request.boundary_ratio + + logger.info( + "Boundary ratio parse: request=%s gen_params=%s", + request.boundary_ratio, + gen_params.boundary_ratio, + ) + if request.flow_shift is not None: + gen_params.extra_args["flow_shift"] = request.flow_shift + + self._apply_lora(request.lora, gen_params) + + logger.info( + "Video sampling params: steps=%s guidance=%s guidance_2=%s seed=%s", + gen_params.num_inference_steps, + gen_params.guidance_scale, + gen_params.guidance_scale_2, + gen_params.seed, + ) + + result = await self._run_generation(prompt, gen_params, request_id, raw_request) + videos = self._extract_video_outputs(result) + output_fps = fps or 24 + + video_data = [VideoData(b64_json=encode_video_base64(video, fps=output_fps)) for video in videos] + return VideoGenerationResponse(created=int(time.time()), data=video_data) + + def _resolve_model_name(self, raw_request: Request | None) -> str | None: + if self._model_name: + return self._model_name + if raw_request is None: + return None + serving_models = getattr(raw_request.app.state, "openai_serving_models", None) + if serving_models and getattr(serving_models, "base_model_paths", None): + base_paths = serving_models.base_model_paths + if base_paths: + return base_paths[0].name + return None + + @staticmethod + def _resolve_video_params(request: VideoGenerationRequest) -> tuple[int | None, int | None, int | None, int | None]: + width = request.width or (request.video_params.width if request.video_params else None) + height = request.height or (request.video_params.height if request.video_params else None) + num_frames = request.num_frames or (request.video_params.num_frames if request.video_params else None) + fps = request.fps or (request.video_params.fps if request.video_params else None) + seconds = request.seconds + + if request.size: + width, height = parse_size(request.size) + + if fps is None: + fps = 24 # Default FPS if not specified + + if num_frames is None and seconds is not None: + num_frames = int(seconds) * int(fps) + + return width, height, num_frames, fps + + @staticmethod + def _apply_lora(lora_body: Any, gen_params: OmniDiffusionSamplingParams) -> None: + if lora_body is None: + return + if not isinstance(lora_body, dict): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Invalid lora field: expected an object.", + ) + lora_name = lora_body.get("name") or lora_body.get("lora_name") or lora_body.get("adapter") + lora_path = ( + lora_body.get("local_path") + or lora_body.get("path") + or lora_body.get("lora_path") + or lora_body.get("lora_local_path") + ) + lora_scale = lora_body.get("scale") + if lora_scale is None: + lora_scale = lora_body.get("lora_scale") + lora_int_id = lora_body.get("int_id") + if lora_int_id is None: + lora_int_id = lora_body.get("lora_int_id") + if lora_int_id is None and lora_path: + lora_int_id = stable_lora_int_id(str(lora_path)) + + if not lora_name or not lora_path: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Invalid lora object: both name and path are required.", + ) + + gen_params.lora_request = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)) + if lora_scale is not None: + gen_params.lora_scale = float(lora_scale) + + async def _run_generation( + self, + prompt: OmniTextPrompt, + gen_params: OmniDiffusionSamplingParams, + request_id: str, + raw_request: Request | None, + ) -> Any: + has_stage_list = hasattr(self._engine_client, "stage_list") + logger.info( + "Video generation routing: stage_configs=%s, has_stage_list=%s, engine_type=%s", + "present" + if (self._stage_configs or (getattr(raw_request.app.state, "stage_configs", None) if raw_request else None)) + else "missing", + has_stage_list, + type(self._engine_client).__name__, + ) + stage_configs = ( + self._stage_configs + or (getattr(raw_request.app.state, "stage_configs", None) if raw_request else None) + or getattr(self._engine_client, "stage_configs", None) + ) + + if not stage_configs: + if not hasattr(self._engine_client, "stage_list"): + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, + detail="Stage configs not found. Start server with an omni diffusion model.", + ) + + # Video generation endpoint only supports diffusion stages. + if stage_configs: + for stage in stage_configs: + # Extract stage_type: dicts and OmegaConf objects use .get(), others use getattr + if hasattr(stage, "get"): + stage_type = stage.get("stage_type", "llm") + else: + stage_type = getattr(stage, "stage_type", "llm") + + if stage_type != "diffusion": + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, + detail=f"Video generation only supports diffusion stages, found '{stage_type}' stage.", + ) + + # Common generation logic for both paths + engine_client = cast(AsyncOmni, self._engine_client) + stage_list = getattr(engine_client, "stage_list", None) + if isinstance(stage_list, list): + sampling_params_list: list[OmniSamplingParams] = [gen_params for _ in stage_list] + else: + sampling_params_list = [gen_params] + + result = None + async for output in engine_client.generate( + prompt=prompt, + request_id=request_id, + sampling_params_list=sampling_params_list, + ): + result = output + + if result is None: + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + detail="No output generated from video generation pipeline.", + ) + return result + + @staticmethod + def _normalize_video_outputs(videos: Any) -> list[Any]: + if videos is None: + return [] + if hasattr(videos, "ndim") and videos.ndim == 5: + return [videos[i] for i in range(videos.shape[0])] + if isinstance(videos, list): + if not videos: + return [] + first = videos[0] + if hasattr(first, "ndim") and first.ndim == 5: + flattened: list[Any] = [] + for item in videos: + if hasattr(item, "ndim") and item.ndim == 5: + flattened.extend([item[i] for i in range(item.shape[0])]) + else: + flattened.append(item) + return flattened + if isinstance(first, list): + return videos + if hasattr(first, "ndim") and first.ndim == 3: + return [videos] + if isinstance(first, Image.Image): + return [videos] + return videos + return [videos] + + def _extract_video_outputs(self, result: Any) -> list[Any]: + videos = None + if hasattr(result, "images") and result.images: + videos = result.images + elif hasattr(result, "request_output"): + request_output = result.request_output + if isinstance(request_output, dict) and request_output.get("images"): + videos = request_output["images"] + elif hasattr(request_output, "images") and request_output.images: + videos = request_output.images + elif hasattr(request_output, "multimodal_output") and request_output.multimodal_output: + videos = request_output.multimodal_output.get("video") + if videos is None and hasattr(result, "multimodal_output") and result.multimodal_output: + videos = result.multimodal_output.get("video") + + normalized = self._normalize_video_outputs(videos) + if not normalized: + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + detail="No video outputs found in generation result.", + ) + return normalized diff --git a/vllm_omni/entrypoints/openai/video_api_utils.py b/vllm_omni/entrypoints/openai/video_api_utils.py new file mode 100644 index 000000000..865b1a528 --- /dev/null +++ b/vllm_omni/entrypoints/openai/video_api_utils.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Shared helper utilities for OpenAI-compatible video generation API. +""" + +from __future__ import annotations + +import base64 +import os +import tempfile +from io import BytesIO +from typing import Any + +import numpy as np +import torch +from PIL import Image + + +def decode_input_reference(input_reference: str | None, input_reference_bytes: bytes | None) -> Image.Image | None: + """Decode image input from multipart bytes or base64/data URL.""" + if input_reference and input_reference_bytes: + raise ValueError("Provide input_reference either as file upload or base64, not both.") + if input_reference_bytes: + return Image.open(BytesIO(input_reference_bytes)).convert("RGB") + if input_reference: + if input_reference.startswith("data:image"): + _, b64_data = input_reference.split(",", 1) + else: + b64_data = input_reference + try: + image_bytes = base64.b64decode(b64_data) + except Exception as exc: # pragma: no cover - malformed base64 + raise ValueError("Invalid base64 input_reference.") from exc + return Image.open(BytesIO(image_bytes)).convert("RGB") + return None + + +def _normalize_video_tensor(video_tensor: torch.Tensor) -> np.ndarray: + """Normalize a torch video tensor into a numpy array of frames (F, H, W, C).""" + video_tensor = video_tensor.detach().cpu() + if video_tensor.dim() == 5: + raise ValueError("Batched video tensors are not supported for single-video encoding.") + elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4): + # [C, F, H, W] -> [F, H, W, C] + video_tensor = video_tensor.permute(1, 2, 3, 0) + + if video_tensor.is_floating_point(): + video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5 + else: + video_tensor = video_tensor.to(torch.float32) / 255.0 + video_array = video_tensor.numpy() + return _normalize_single_video_array(video_array) + + +def _normalize_single_video_array(video_array: np.ndarray) -> np.ndarray: + """Normalize a single video array into shape (F, H, W, C).""" + if video_array.ndim == 5: + raise ValueError("Batched video arrays are not supported for single-video encoding.") + + if video_array.ndim == 4: + # Convert channel-first layouts to channel-last + if video_array.shape[0] in (3, 4) and video_array.shape[-1] not in (3, 4): + video_array = np.transpose(video_array, (1, 2, 3, 0)) + elif video_array.shape[1] in (3, 4) and video_array.shape[-1] not in (3, 4): + video_array = np.transpose(video_array, (0, 2, 3, 1)) + + if np.issubdtype(video_array.dtype, np.floating): + if video_array.min() < 0.0 or video_array.max() > 1.0: + video_array = np.clip(video_array, -1.0, 1.0) * 0.5 + 0.5 + elif np.issubdtype(video_array.dtype, np.integer): + video_array = video_array.astype(np.float32) / 255.0 + return video_array + + +def _normalize_video_array(video_array: np.ndarray) -> list[np.ndarray] | np.ndarray: + """Normalize a numpy video array into shape (F, H, W, C). + + If a batch dimension is present, returns a list of per-video arrays. + """ + if video_array.ndim == 5: + return [_normalize_single_video_array(video_array[i]) for i in range(video_array.shape[0])] + return _normalize_single_video_array(video_array) + + +def _normalize_frames(frames: list[Any]) -> list[np.ndarray]: + """Normalize a list of frames into numpy arrays with values in [0,1].""" + normalized: list[np.ndarray] = [] + for frame in frames: + if isinstance(frame, torch.Tensor): + frame_array = frame.detach().cpu().numpy() + elif isinstance(frame, Image.Image): + frame_array = np.array(frame) + elif isinstance(frame, np.ndarray): + frame_array = frame + else: + raise ValueError(f"Unsupported frame type: {type(frame)}") + + if frame_array.ndim == 3 and frame_array.shape[0] in (3, 4) and frame_array.shape[-1] not in (3, 4): + frame_array = np.transpose(frame_array, (1, 2, 0)) + + if np.issubdtype(frame_array.dtype, np.floating): + if frame_array.min() < 0.0 or frame_array.max() > 1.0: + frame_array = np.clip(frame_array, -1.0, 1.0) * 0.5 + 0.5 + elif np.issubdtype(frame_array.dtype, np.integer): + frame_array = frame_array.astype(np.float32) / 255.0 + + normalized.append(frame_array) + return normalized + + +def _coerce_video_to_frames(video: Any) -> list[np.ndarray]: + """Convert a video payload into a list of frames for export_to_video.""" + if isinstance(video, torch.Tensor): + video_array = _normalize_video_tensor(video) + return list(video_array) + if isinstance(video, np.ndarray): + video_array = _normalize_video_array(video) + if isinstance(video_array, list): + raise ValueError("Batched video arrays must be split before encoding.") + if video_array.ndim == 4: + return list(video_array) + if video_array.ndim == 3: + return [video_array] + raise ValueError(f"Unsupported video array shape: {video_array.shape}") + if isinstance(video, list): + if not video: + return [] + # If this looks like a list of frames, normalize directly. + if all(isinstance(item, (np.ndarray, torch.Tensor, Image.Image)) for item in video): + # If each item is itself a video (ndim==4), handle elsewhere. + if all(hasattr(item, "ndim") and item.ndim >= 4 for item in video): + raise ValueError("Expected a single video, got a list of video tensors/arrays.") + return _normalize_frames(video) + raise ValueError("Unsupported list contents for video payload.") + raise ValueError(f"Unsupported video payload type: {type(video)}") + + +def encode_video_base64(video: Any, fps: int) -> str: + """Encode a video (frames/array/tensor) to base64 MP4.""" + try: + from diffusers.utils import export_to_video + except ImportError as exc: # pragma: no cover - optional dependency + raise ImportError("diffusers is required for export_to_video.") from exc + + frames = _coerce_video_to_frames(video) + if not frames: + raise ValueError("No frames found to encode.") + + tmp_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) + tmp_file.close() + try: + export_to_video(frames, tmp_file.name, fps=fps) + with open(tmp_file.name, "rb") as f: + video_bytes = f.read() + return base64.b64encode(video_bytes).decode("utf-8") + finally: + try: + os.remove(tmp_file.name) + except OSError: + pass