# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ OpenAI-compatible protocol definitions for video generation. This module provides Pydantic models for a video generation endpoint that mirrors the OpenAI Images API shape, with vllm-omni extensions for diffusion video models (e.g., Wan2.2). """ import mimetypes import time import uuid from enum import Enum from functools import lru_cache from typing import Annotated, Any, Literal from pydantic import BaseModel, ConfigDict, Field, StringConstraints, field_validator from vllm_omni.entrypoints.openai.image_api_utils import parse_size from vllm_omni.inputs.data import DIFFUSION_QUALITY_LEVELS class VideoGenerationStatus(str, Enum): QUEUED = "queued" IN_PROGRESS = "in_progress" COMPLETED = "completed" FAILED = "failed" SizeStr = Annotated[str, StringConstraints(pattern=r"^\d+x\d+$")] SecondStr = Annotated[str, StringConstraints(pattern=r"^[1-9]\d*$")] DEFAULT_FPS = 24 @lru_cache def file_extension(media_type: str): media_type = str(media_type).split(";", 1)[0].strip().lower() ext = mimetypes.guess_extension(media_type, strict=False) if ext is None: raise ValueError(f"No recognized file extension for media_type {media_type}") # Keep naming stable for unknown/unsupported MIME types. return ext.lstrip(".") class VideoParams(BaseModel): """Optional block for video-specific parameters.""" width: int | None = Field(default=None, ge=1, description="Video width in pixels") height: int | None = Field(default=None, ge=1, description="Video height in pixels") num_frames: int | None = Field(default=None, ge=1, description="Number of frames") fps: int | None = Field(default=None, ge=1, description="Frames per second for output video") @property def size(self) -> str | None: if self.width and self.height: return f"{self.width}x{self.height}" else: return None class FileImageReference(BaseModel): model_config = ConfigDict(extra="forbid") file_id: str class UrlImageReference(BaseModel): model_config = ConfigDict(extra="forbid") image_url: str ImageReference = UrlImageReference | FileImageReference class FileVideoReference(BaseModel): model_config = ConfigDict(extra="forbid") file_id: str class UrlVideoReference(BaseModel): model_config = ConfigDict(extra="forbid") video_url: str VideoReference = UrlVideoReference | FileVideoReference class UrlAudioReference(BaseModel): model_config = ConfigDict(extra="forbid") audio_url: str AudioReference = UrlAudioReference class VideoGenerationRequest(BaseModel): """ OpenAI-style video generation request. Follows the OpenAI Images API conventions with extensions for video. """ # OpenAI standard fields model: str | None = Field( default=None, description="Model to use (optional, uses server's configured model if omitted)", ) prompt: str = Field(..., description="Text description of the desired video(s)") seconds: SecondStr | None = Field( default=None, description="Clip duration in seconds (OpenAI-compatible, e.g., 4, 8, 12)", ) size: SizeStr | None = Field( default=None, description="Video dimensions in WIDTHxHEIGHT format (e.g., '1280x720')", ) image_reference: ImageReference | list[ImageReference] | None = Field( default=None, description=( "Optional image reference or ordered list of references. MiniMax H3 uses the list order for " "FL2VA first/last frames and Ref2VA image labels." ), ) video_reference: VideoReference | list[VideoReference] | None = Field( default=None, description="Optional video reference or ordered list of Ref2VA video references.", ) audio_reference: AudioReference | list[AudioReference] | None = Field( default=None, description="Optional audio reference or ordered list of Ref2VA audio references.", ) # Video params block for extensibility video_params: VideoParams | None = Field(default=None, description="Optional video-specific parameters") # User specific tracking field user: str | None = Field(default=None, description="User identifier for tracking") # Video-specific fields (top-level for OpenAI-style compatibility) width: int | None = Field(default=None, ge=1, description="Video width in pixels") height: int | None = Field(default=None, ge=1, description="Video height in pixels") fps: int | None = Field(default=None, ge=1, description="Frames per second for output video") num_frames: int | None = Field(default=None, ge=1, description="Number of frames to generate") aspect_ratio: str | None = Field( default=None, description=( "MiniMax H3 output ratio. T2VA requires 21:9, 16:9, 4:3, 1:1, 3:4, or 9:16; " "FL2VA follows the input image; Ref2VA defaults to 16:9." ), ) short_edge: int | None = Field(default=None, ge=1, description="MiniMax H3 output short edge in pixels") num_outputs_per_prompt: int = Field( default=1, ge=1, le=10, description="Number of videos to generate. MiniMax H3 supports 1 through 10.", ) start_time_seconds: float | None = Field( default=None, ge=0.0, description="Start offset for a single MiniMax H3 reference video.", ) # vllm-omni extensions for diffusion control quality: str | None = Field( default=None, description=( "Request-level generation quality intent. Supported values are " "'lossless' and 'high'; exact behavior is model-specific. " "When omitted, the model chooses its default policy." ), ) @field_validator("quality") @classmethod def validate_quality(cls, value: str | None) -> str | None: if value is None: return None if value not in DIFFUSION_QUALITY_LEVELS: raise ValueError(f"quality must be one of {list(DIFFUSION_QUALITY_LEVELS)}, got {value!r}") return value negative_prompt: str | None = Field(default=None, description="Text describing what to avoid in the video") num_inference_steps: int | None = Field( default=None, ge=1, le=200, description="Number of diffusion sampling steps (uses model defaults if not specified)", ) guidance_scale: float | None = Field( default=None, ge=0.0, le=20.0, description="Classifier-free guidance scale (uses model defaults if not specified)", ) guidance_scale_2: float | None = Field( default=None, ge=0.0, le=20.0, description="High-noise CFG scale for video models (Wan2.2)", ) boundary_ratio: float | None = Field( default=None, ge=0.0, le=1.0, description="Boundary split ratio for low/high DiT (Wan2.2)", ) flow_shift: float | None = Field( default=None, description="Scheduler flow_shift for video models (Wan2.2)", ) true_cfg_scale: float | None = Field( default=None, ge=0.0, le=20.0, description="True CFG scale (model-specific parameter, may be ignored if not supported)", ) seed: int | None = Field(default=None, description="Random seed for reproducibility") generate_sound: bool = Field( default=False, description="Request model-generated audio for video models that support sound generation.", ) sound_duration: float | None = Field( default=None, gt=0.0, description="Duration in seconds for model-generated audio. Defaults to the generated video duration.", ) # vllm-omni extensions for post-generation frame interpolation. enable_frame_interpolation: bool = Field( default=False, description="Enable post-generation RIFE frame interpolation before MP4 encoding.", ) frame_interpolation_exp: int = Field( default=1, ge=1, description="Interpolation exponent: 1=2x temporal resolution, 2=4x, etc.", ) frame_interpolation_scale: float = Field( default=1.0, gt=0.0, description="RIFE inference scale. Use 0.5 for high-resolution inputs to save memory.", ) frame_interpolation_model_path: str | None = Field( default=None, description=( "Local directory or Hugging Face repo ID containing RIFE flownet.pkl weights. " "Defaults to elfgum/RIFE-4.22.lite." ), ) # vllm-omni extension for per-request LoRA. lora: dict[str, Any] | None = Field( default=None, description=( "Optional LoRA adapter for this request. Expected shape: " "{name/path/scale/int_id}. Field names are flexible " "(e.g. name|lora_name|adapter, path|lora_path|local_path, " "scale|lora_scale, int_id|lora_int_id)." ), ) # Generic model-specific parameters extra_params: dict[str, Any] | None = Field( default=None, description=("Optional model-specific parameters passed directly to the model's extra_args. "), ) def resolve_video_params(self) -> VideoParams: vp = VideoParams(width=self.width, height=self.height, fps=self.fps, num_frames=self.num_frames) if self.video_params is not None: vp.width = vp.width or self.video_params.width vp.height = vp.height or self.video_params.height vp.fps = vp.fps or self.video_params.fps vp.num_frames = vp.num_frames or self.video_params.num_frames if self.size: vp.width, vp.height = parse_size(self.size) if vp.fps is None: vp.fps = DEFAULT_FPS if vp.num_frames is None and self.seconds is not None: vp.num_frames = int(self.seconds) * int(vp.fps) return vp class VideoAction(BaseModel): """Generated action sequence returned by action-capable video models.""" data: list[Any] = Field(..., description="JSON-serializable nested action values") shape: list[int] = Field(..., description="Shape of the returned action data") dtype: str | None = Field(default=None, description="Source action dtype, if available") raw_action_dim: int | None = Field(default=None, description="Raw action dimension requested by the model") action_mode: str | None = Field(default=None, description="Action generation mode") domain_id: int | None = Field(default=None, description="Action embodiment domain id") class VideoData(BaseModel): """Single generated video data.""" b64_json: str | None = Field(default=None, description="Base64-encoded MP4 video") url: str | None = Field(default=None, description="Video URL (not implemented)") revised_prompt: str | None = Field(default=None, description="Revised prompt (OpenAI compatibility, always null)") action: VideoAction | None = Field(default=None, description="Generated action sequence metadata, if any") class VideoGenerationResponse(BaseModel): """OpenAI-style video generation response.""" created: int = Field(..., description="Unix timestamp of when the generation completed") data: list[VideoData] = Field(..., description="Array of generated videos") stage_durations: dict[str, float] = Field( default_factory=dict, description="Profiler stage durations reported by the diffusion pipeline.", ) peak_memory_mb: float = Field( default=0.0, description="Peak device memory usage in MB reported by the diffusion pipeline.", ) class VideoError(BaseModel): code: int | str = Field(..., description="A machine-readable error code that was returned.") message: str = Field(..., description="A human-readable description of the error that was returned.") class VideoResponse(BaseModel): """Stored metadata for an async video generation job.""" # OpenAI standard fields model: str = Field(..., description="Model name used for video generation.") prompt: str = Field(..., description="The prompt that was used to generate the video.") id: str = Field( default_factory=lambda: f"video_gen_{uuid.uuid4().hex}", description="Unique id for a video request" ) object: Literal["video"] = Field(default="video", description="Object type identifier.") status: VideoGenerationStatus = Field( default=VideoGenerationStatus.QUEUED, description="Current lifecycle status of the video job." ) size: SizeStr | None = Field(default=None, description="Requested output size in WIDTHxHEIGHT format.") progress: int = Field(default=0, description="Best-effort progress indicator from 0 to 100.") seconds: SecondStr = Field(default="4", description="Requested clip length in seconds.") quality: str = Field(default="default", description="Requested quality level for generation.") completed_at: int | None = Field( default=None, description="Unix timestamp (seconds) for when the job completed, if finished." ) created_at: int = Field( default_factory=lambda: int(time.time()), description="Unix timestamp (seconds) for when the job was created.", ) remixed_from_video_id: str | None = Field( default=None, description="Optional source video id for remix/edit flows.", ) error: VideoError | None = Field( default=None, description="Error payload that explains why generation failed, if applicable.", ) # vLLM specific fields media_type: Literal["video/mp4"] = Field(default="video/mp4", description="MIME type of the generated artifact.") expires_at: int | None = Field(default=None, description="Unix timestamp when this record is considered expired.") file_name: str | None = Field( default=None, description="Filename of the saved output video files for this job.", ) inference_time_s: float | None = Field(default=None, description="End-to-end inference time in seconds.") stage_durations: dict[str, float] = Field( default_factory=dict, description="Profiler stage durations reported by the diffusion pipeline.", ) peak_memory_mb: float = Field( default=0.0, description="Peak device memory usage in MB reported by the diffusion pipeline.", ) action: VideoAction | None = Field(default=None, description="Generated action sequence metadata, if any") @property def file_extension(self) -> str: return file_extension(self.media_type) class VideoDeleteResponse(BaseModel): id: str = Field(description="Identifier of the deleted video.") deleted: bool = Field(description="Indicates that the video resource was deleted.") object: Literal["video.deleted"] = Field( default="video.deleted", description="The object type that signals the deletion response." ) class VideoListResponse(BaseModel): """Paginated-style wrapper for listing stored video jobs.""" first_id: str | None = Field(..., description="The ID of the first item in the list.") last_id: str | None = Field(..., description="The ID of the last item in the list.") has_more: bool = Field(..., description="Whether there are more items available.") data: list[VideoResponse] = Field(..., description="Array of video job records.") object: Literal["list"] = Field(default="list", description="Object type identifier for list responses.")