# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """``/v1/videos*`` endpoint helpers peeled from ``api_server.py``. PUT HERE: - Multipart form parsing, upload limits, job run/cleanup, response factories, app-state runtime context, and other route-adjacent ``/v1/videos*`` logic that used to live in ``api_server.py``. - May import ``video_api_utils`` for decode/encode (helpers → utils OK). DO NOT PUT HERE: - Shared media encode/decode backends — keep those in ``video_api_utils``. LONGEVITY: - This package ``helpers.py`` is the **longer home** for video generation endpoint logic through the helper (P0.2) + router (P0.3) stages. - The later P1.3 video modality PR (#5227) may further split this file (e.g. form / jobs) and move ``video_api_utils`` into ``media.py`` here; until then keep growing here, not in root utils. See ``video/generation/README.md`` (utils vs helpers, no overlap). """ import asyncio import io import json import os import tempfile import time from collections.abc import Mapping, Sequence from contextlib import suppress from http import HTTPStatus from numbers import Integral from pathlib import Path from typing import Any, Literal, cast from fastapi import File, Form, HTTPException, Request, UploadFile from PIL import Image from vllm.entrypoints.launchers.launcher import terminate_if_errored from vllm.entrypoints.serve import create_error_response from vllm.logger import init_logger from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError from vllm_omni.diffusion.models.interface import ReferenceVideoDecodeSpec from vllm_omni.entrypoints.openai.app_state import Omnivideo from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError from vllm_omni.entrypoints.openai.protocol.videos import ( SecondStr, SizeStr, VideoAction, VideoError, VideoGenerationRequest, VideoGenerationStatus, VideoResponse, ) from vllm_omni.entrypoints.openai.serving_video import ( LatentEditInput, OmniOpenAIServingVideo, ReferenceAudio, ReferenceImage, ReferenceVideo, _stage_diffusion_model_class_name, ) from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER from vllm_omni.entrypoints.openai.stores import VIDEO_STORE from vllm_omni.entrypoints.openai.utils import get_stage_type from vllm_omni.entrypoints.openai.video_api_utils import ( VideoFrames, _decode_image_bytes, _validate_image_pixel_limit, decode_audio_url, decode_input_reference, ) from vllm_omni.errors import OmniClientError logger = init_logger(__name__) MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024 MINIMAX_H3_MAX_REFERENCE_VIDEO_BYTES = 50 * 1024 * 1024 MINIMAX_H3_MAX_REFERENCE_AUDIO_BYTES = 15 * 1024 * 1024 MINIMAX_H3_MAX_REFERENCE_COUNT = 12 MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"}) MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES = frozenset({".mp4", ".mov"}) MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES = frozenset({".wav", ".mp3"}) CONTROL_REFERENCE_IMAGE_SUFFIXES = frozenset({".bmp", ".gif", ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp"}) CONTROL_REFERENCE_VIDEO_SUFFIXES = frozenset({".mkv", ".mov", ".mp4", ".webm"}) CONTROL_REFERENCE_MAX_BYTES = 512 * 1024 * 1024 LATENT_EDIT_SOURCE_MAX_BYTES = 512 * 1024 * 1024 LATENT_EDIT_MASK_FILE_MAX_BYTES = 8 * 1024 * 1024 VIDEO_SYNC_TIMEOUT_S = float(os.environ.get("VLLM_OMNI_VIDEO_SYNC_TIMEOUT", 600.0)) def _resolve_video_runtime_context(raw_request: Request) -> tuple[str | None, list[Any] | None]: app_model_name = None serving_models = getattr(raw_request.app.state, "openai_serving_models", None) if serving_models and getattr(serving_models, "base_model_paths", None): base_paths = serving_models.base_model_paths if base_paths: app_model_name = base_paths[0].name app_stage_configs = getattr(raw_request.app.state, "stage_configs", None) return app_model_name, app_stage_configs def _parse_form_json(value: str | None, expected_type: type | None = None) -> Any: if value is None or value == "": return None try: parsed = json.loads(value) except json.JSONDecodeError as exc: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Invalid JSON in form field.", ) from exc if expected_type is not None and not isinstance(parsed, expected_type): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"Invalid JSON in form field: expected {expected_type.__name__}, got {type(parsed).__name__}.", ) return parsed def _parse_latent_edit_mask_json( value: str, *, field_name: str, ) -> Any: """Decode one request mask; Stage 0 owns its semantic validation.""" try: parsed = _parse_form_json(value) # Bound container nesting at the parsing boundary without duplicating # Stage 0's numeric, range, or shape validation. pending = [(parsed, 0)] while pending: item, depth = pending.pop() if not isinstance(item, (list, dict)): continue if depth >= 64: raise ValueError("JSON nesting exceeds 64 levels") children = item.values() if isinstance(item, dict) else item pending.extend((child, depth + 1) for child in children) return parsed except (RecursionError, ValueError) as exc: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} contains invalid JSON or is nested too deeply.", ) from exc async def _read_latent_edit_mask_json( value: UploadFile | None, *, field_name: str, ) -> Any | None: """Read a mask from a bounded UTF-8 JSON file part.""" if value is None: return None try: declared_size = value.size if isinstance(declared_size, Integral) and int(declared_size) > LATENT_EDIT_MASK_FILE_MAX_BYTES: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} JSON exceeds the {LATENT_EDIT_MASK_FILE_MAX_BYTES // (1024 * 1024)} MiB limit.", ) payload = await value.read(LATENT_EDIT_MASK_FILE_MAX_BYTES + 1) if len(payload) > LATENT_EDIT_MASK_FILE_MAX_BYTES: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} JSON exceeds the {LATENT_EDIT_MASK_FILE_MAX_BYTES // (1024 * 1024)} MiB limit.", ) try: text = payload.decode("utf-8") except UnicodeDecodeError as exc: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} JSON file must be UTF-8 encoded.", ) from exc if not text: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} JSON file must not be empty.", ) parsed = _parse_latent_edit_mask_json(text, field_name=field_name) if parsed is None: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} JSON file must contain a mask value.", ) return parsed finally: await value.close() def _config_get(config: Any, key: str, default: Any = None) -> Any: if isinstance(config, dict): return config.get(key, default) if hasattr(config, "get"): try: return config.get(key, default) except Exception: pass return getattr(config, key, default) def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]: if not stage_configs: return [] from vllm_omni.diffusion.registry import DiffusionModelRegistry model_classes: list[type] = [] for stage_cfg in stage_configs: if get_stage_type(stage_cfg) != "diffusion": continue model_class_name = _stage_diffusion_model_class_name(stage_cfg) if not model_class_name: continue model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name) if model_cls is not None: model_classes.append(model_cls) return model_classes def _normalize_reference_video_decode_spec(spec: ReferenceVideoDecodeSpec) -> ReferenceVideoDecodeSpec: max_frames = spec.max_frames if max_frames is not None: try: max_frames = int(max_frames) except (TypeError, ValueError) as exc: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Invalid reference video decode spec: max_frames must be an integer.", ) from exc if max_frames <= 0: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Invalid reference video decode spec: max_frames must be positive.", ) keep = str(spec.keep or "first").strip().lower() if keep not in {"first", "last"}: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Invalid reference video decode spec: keep must be either 'first' or 'last'.", ) return ReferenceVideoDecodeSpec(max_frames=max_frames, keep=cast(Literal["first", "last"], keep)) def _reference_video_decode_spec( req: VideoGenerationRequest, stage_configs: list[Any] | None, ) -> ReferenceVideoDecodeSpec: video_params = req.resolve_video_params() extra_params = req.extra_params if isinstance(req.extra_params, dict) else {} for model_cls in _diffusion_model_classes(stage_configs): resolver = getattr(model_cls, "reference_video_decode_spec", None) if resolver is None: continue try: spec = resolver(num_frames=video_params.num_frames, extra_args=extra_params) except (TypeError, ValueError) as exc: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value, detail=str(exc)) from exc if spec is not None: return _normalize_reference_video_decode_spec(spec) return ReferenceVideoDecodeSpec(max_frames=video_params.num_frames, keep="first") def video_response_from_request(model_name: str, req: VideoGenerationRequest) -> VideoResponse: video_params = req.resolve_video_params() duration_s = None if req.seconds is not None: duration_s = float(req.seconds) elif video_params.num_frames is not None and video_params.fps is not None: duration_s = video_params.num_frames / video_params.fps resp = VideoResponse( model=model_name, status=VideoGenerationStatus.QUEUED, size=req.size, prompt=req.prompt, quality=req.quality or "default", fps=video_params.fps, num_frames=video_params.num_frames, duration_s=duration_s, ) resp.seconds = str(req.seconds or resp.seconds) return resp def _status_code_for_video_failure(error: VideoError | None) -> int: if error is None: return HTTPStatus.INTERNAL_SERVER_ERROR.value if isinstance(error.code, int): if 400 <= error.code < 600: return error.code return HTTPStatus.INTERNAL_SERVER_ERROR.value if error.code == "HTTPException": status_text, _, _ = error.message.partition(":") try: status_code = int(status_text) except ValueError: return HTTPStatus.INTERNAL_SERVER_ERROR.value if 400 <= status_code < 600: return status_code return HTTPStatus.INTERNAL_SERVER_ERROR.value if error.code == "EngineDeadError": return HTTPStatus.INTERNAL_SERVER_ERROR.value if error.code == "EngineGenerateError": return HTTPStatus.INTERNAL_SERVER_ERROR.value return HTTPStatus.INTERNAL_SERVER_ERROR.value def _video_error_from_exception(exc: Exception) -> VideoError: if isinstance(exc, HTTPException): message = str(exc.detail) if exc.detail else str(exc) return VideoError(code=exc.status_code, message=message) if isinstance(exc, OmniClientError): return VideoError(code=exc.status_code, message=exc.message) if isinstance(exc, (EngineGenerateError, EngineDeadError)): err = create_error_response(exc) return VideoError(code=err.error.code, message=err.error.message) return VideoError( code=HTTPStatus.INTERNAL_SERVER_ERROR.value, message=str(exc), ) async def _cleanup_video(video_id: str): try: await STORAGE_MANAGER.delete(video_id) except Exception: logger.warning("Failed to cleanup partial video file '%s'", video_id) def _cleanup_video_references( reference_video: ReferenceVideo | None, reference_audio: ReferenceAudio | None, control_path: str | None = None, latent_edit_input: LatentEditInput | None = None, ) -> None: if reference_video is not None: for path in reference_video.cleanup_paths: if os.path.exists(path): os.unlink(path) if reference_audio is not None: cleanup_paths = reference_audio.cleanup_paths or tuple(_reference_list(reference_audio.path)) for path in cleanup_paths: if os.path.exists(path): os.unlink(path) if control_path is not None and os.path.exists(control_path): os.unlink(control_path) if latent_edit_input is not None: for path in latent_edit_input.cleanup_paths: if os.path.exists(path): os.unlink(path) def _unpack_video_generation_result( result: Sequence[object], ) -> tuple[bytes, dict[str, float], float, VideoAction | None, dict[str, object]]: video_metadata: dict[str, object] = {} if len(result) == 5: video_bytes, stage_durations, peak_memory_mb, action, raw_metadata = result if isinstance(raw_metadata, dict): video_metadata = {str(key): value for key, value in raw_metadata.items()} else: video_bytes, stage_durations, peak_memory_mb, action = result return ( cast(bytes, video_bytes), cast(dict[str, float], stage_durations), float(cast(float, peak_memory_mb)), cast(VideoAction | None, action), video_metadata, ) async def _run_video_generation_job( handler: OmniOpenAIServingVideo, request: VideoGenerationRequest, video_id: str, reference_image: ReferenceImage | None = None, reference_video: ReferenceVideo | None = None, reference_audio: ReferenceAudio | None = None, control_path: str | None = None, app_state: Any | None = None, latent_edit_input: LatentEditInput | None = None, ) -> None: job = await VIDEO_STORE.get(video_id) if job is None: logger.warning("Video job %s missing before generation task started; skipping", video_id) _cleanup_video_references(reference_video, reference_audio, control_path, latent_edit_input) return started_at = time.perf_counter() try: async def _mark_started() -> None: await VIDEO_STORE.update_fields(video_id, {"status": VideoGenerationStatus.IN_PROGRESS}) video_bytes, stage_durations, peak_memory_mb, action, video_metadata = _unpack_video_generation_result( await handler.generate_video_bytes( request, video_id, reference_image=reference_image, reference_video=reference_video, reference_audio=reference_audio, on_started=_mark_started, latent_edit_input=latent_edit_input, ) ) save_context = await STORAGE_MANAGER.save(video_bytes, video_id) logger.info("Video request %s persisted %s output file.", video_id, save_context.key) updated_fields = { "status": VideoGenerationStatus.COMPLETED, "progress": 100, "file_name": f"{video_id}.{job.file_extension}", "completed_at": save_context.created_at, "inference_time_s": time.perf_counter() - started_at, "stage_durations": stage_durations, "peak_memory_mb": peak_memory_mb, "action": action, } updated_fields.update({key: value for key, value in video_metadata.items() if value is not None}) if save_context.expires_at is not None: updated_fields["expires_at"] = save_context.expires_at await VIDEO_STORE.update_fields(video_id, updated_fields) except (EngineGenerateError, EngineDeadError) as exc: logger.exception("Video generation failed (engine error) for id=%s", video_id) await _cleanup_video(video_id) await VIDEO_STORE.update_fields( video_id, { "status": VideoGenerationStatus.FAILED, "completed_at": int(time.time()), "error": _video_error_from_exception(exc), "inference_time_s": time.perf_counter() - started_at, }, ) # Background tasks can't propagate exceptions to FastAPI handlers. # Actively signal shutdown when the engine is dead. if app_state is not None and isinstance(exc, EngineDeadError): terminate_if_errored( server=app_state.server, engine=app_state.engine_client, ) except Exception as exc: logger.exception("Video generation failed for id=%s", video_id) await _cleanup_video(video_id) await VIDEO_STORE.update_fields( video_id, { "status": VideoGenerationStatus.FAILED, "completed_at": int(time.time()), "error": _video_error_from_exception(exc), "inference_time_s": time.perf_counter() - started_at, }, ) except asyncio.CancelledError: await _cleanup_video(video_id) await VIDEO_STORE.pop(video_id) raise finally: _cleanup_video_references(reference_video, reference_audio, control_path, latent_edit_input) async def _persist_uploaded_video_references(uploads: list[UploadFile]) -> list[str]: paths: list[str] = [] try: for upload in uploads: suffix = Path(upload.filename or "").suffix.lower() if suffix not in {".mkv", ".mov", ".mp4", ".webm"}: suffix = ".mp4" fd, path = tempfile.mkstemp(prefix="vllm_omni_video_reference_", suffix=suffix) paths.append(path) with os.fdopen(fd, "wb") as output: while chunk := await upload.read(1024 * 1024): output.write(chunk) except Exception: for path in paths: if os.path.exists(path): os.unlink(path) raise return paths async def _persist_uploaded_reference( upload: UploadFile, *, field_name: str, suffix: str, max_bytes: int, ) -> str: """Stream one bounded upload to request-scoped local storage.""" declared_size = upload.size if isinstance(declared_size, Integral) and int(declared_size) > max_bytes: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} exceeds the {max_bytes // (1024 * 1024)} MiB size limit.", ) fd, path = tempfile.mkstemp(prefix=f"vllm_omni_{field_name}_", suffix=suffix) size = 0 persisted = False try: with os.fdopen(fd, "wb") as output: while chunk := await upload.read(min(1024 * 1024, max_bytes - size + 1)): size += len(chunk) if size > max_bytes: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} exceeds the {max_bytes // (1024 * 1024)} MiB size limit.", ) output.write(chunk) if size == 0: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} must not be empty.", ) persisted = True return path finally: if not persisted: with suppress(OSError): os.unlink(path) async def _persist_uploaded_control_reference( upload: UploadFile, *, max_bytes: int = CONTROL_REFERENCE_MAX_BYTES, ) -> str: """Stream one model control upload to request-scoped local storage.""" kind = _uploaded_media_kind(upload) suffix = Path(upload.filename or "").suffix.lower() supported_suffixes = CONTROL_REFERENCE_IMAGE_SUFFIXES | CONTROL_REFERENCE_VIDEO_SUFFIXES if kind == "audio" or (suffix and suffix not in supported_suffixes): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="control_reference must be an image or video file.", ) if not suffix: suffix = ".png" if kind == "image" else ".mp4" return await _persist_uploaded_reference( upload, field_name="control_reference", suffix=suffix, max_bytes=max_bytes, ) async def _persist_latent_edit_source( upload: UploadFile, *, field_name: Literal["source_video", "source_audio"], max_bytes: int = LATENT_EDIT_SOURCE_MAX_BYTES, ) -> str: """Stream one latent-edit source upload to request-scoped storage.""" suffix = Path(upload.filename or "").suffix.lower() if field_name == "source_video": allowed_suffixes = MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES expected = "an MP4 or MOV file" else: allowed_suffixes = MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES expected = "a WAV or MP3 file" if suffix not in allowed_suffixes: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"{field_name} must be {expected}.", ) return await _persist_uploaded_reference( upload, field_name=field_name, suffix=suffix, max_bytes=max_bytes, ) def _validate_latent_edit_inputs( handler: OmniOpenAIServingVideo, *, source_video: UploadFile | None, source_audio: UploadFile | None, video_noise_mask: Any | None, audio_noise_mask: Any | None, ) -> bool: """Validate cross-field latent-edit requirements before persisting files.""" has_any_input = any(value is not None for value in (source_video, source_audio, video_noise_mask, audio_noise_mask)) if not has_any_input: return False if not handler.supports_latent_mask_editing: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Latent-mask editing is not supported by this model.", ) if video_noise_mask is None and audio_noise_mask is None: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Latent-mask editing requires at least one of video_noise_mask or audio_noise_mask.", ) if source_audio is not None and audio_noise_mask is None: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="source_audio requires an audio_noise_mask upload.", ) return True def _validate_control_upload( handler: OmniOpenAIServingVideo, request: VideoGenerationRequest, control_reference: UploadFile | None, control_type: str | None, ) -> str | None: """Validate a generic uploaded control against model capabilities.""" if control_reference is None: if control_type is not None: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="control_type requires a control_reference upload.", ) return None if control_type is None or not control_type.strip(): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="control_reference requires control_type.", ) normalized_type = control_type.strip().lower() supported_types = handler.supported_control_upload_types if normalized_type not in supported_types: supported = ", ".join(sorted(supported_types)) or "none" raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=( f"Control upload type '{normalized_type}' is not supported by this model. " f"Supported control upload types: {supported}." ), ) existing = (request.extra_params or {}).get(normalized_type) if existing is not None: if not isinstance(existing, Mapping) and existing is not True: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"extra_params.{normalized_type} must be an object when control_reference is uploaded.", ) if isinstance(existing, Mapping) and any(existing.get(key) is not None for key in ("control", "control_path")): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=( "Provide either control_reference or " f"extra_params.{normalized_type}.control/control_path, not both." ), ) return normalized_type def _attach_control_upload( request: VideoGenerationRequest, control_type: str, control_path: str | None = None, ) -> None: """Declare an uploaded control and attach its persisted path when available.""" extra_params = dict(request.extra_params or {}) existing = extra_params.get(control_type) control_params = dict(existing) if isinstance(existing, Mapping) else {} if control_path is not None: control_params["control_path"] = control_path extra_params[control_type] = control_params request.extra_params = extra_params def _reference_list(value: Any) -> list[Any]: if value is None: return [] return list(value) if isinstance(value, list) else [value] def _uploaded_media_kind(upload: UploadFile) -> str: content_type = (upload.content_type or "").lower() if content_type.startswith("image/"): return "image" if content_type.startswith("audio/"): return "audio" suffix = Path(upload.filename or "").suffix.lower() if suffix in {".png", ".jpg", ".jpeg", ".webp", ".heic", ".heif"}: return "image" if suffix in {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg"}: return "audio" return "video" def _minimax_h3_upload_limit(upload: UploadFile) -> int: kind = _uploaded_media_kind(upload) if kind == "image": return MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES if kind == "audio": return MINIMAX_H3_MAX_REFERENCE_AUDIO_BYTES return MINIMAX_H3_MAX_REFERENCE_VIDEO_BYTES async def _read_upload_limited(upload: UploadFile, *, max_bytes: int | None = None) -> bytes: """Read an upload with an optional hard byte limit.""" if max_bytes is None: return await upload.read() declared_size = getattr(upload, "size", None) if isinstance(declared_size, Integral) and int(declared_size) > max_bytes: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"Uploaded reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.", ) chunks: list[bytes] = [] total = 0 while True: chunk = await upload.read(min(1024 * 1024, max_bytes - total + 1)) if not chunk: break total += len(chunk) if total > max_bytes: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"Uploaded reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.", ) chunks.append(chunk) return b"".join(chunks) def _validate_minimax_h3_image_payload( payload: bytes, *, filename: str | None, allow_non_image: bool = False, ) -> None: """Validate a H3 image before converting it to a format-less PIL image.""" try: with Image.open(io.BytesIO(payload)) as image: image_format = str(image.format or "").lower() if image_format in MINIMAX_H3_REFERENCE_IMAGE_FORMATS: _validate_image_pixel_limit(image) except InvalidInputReferenceError as exc: raise HTTPException(HTTPStatus.BAD_REQUEST.value, detail=str(exc)) from exc except Image.DecompressionBombError as exc: raise HTTPException( HTTPStatus.BAD_REQUEST.value, detail=f"Invalid uploaded image reference: {filename}; image exceeds the decoder pixel limit.", ) from exc except (OSError, ValueError) as exc: if allow_non_image: return raise HTTPException(400, detail=f"Invalid uploaded image reference: {filename}") from exc if image_format not in MINIMAX_H3_REFERENCE_IMAGE_FORMATS: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=( "MiniMax H3 reference images must use JPG, JPEG, PNG, WEBP, HEIC, or HEIF; " f"got {image_format or 'unknown'}." ), ) async def _persist_uploaded_media_references( uploads: list[UploadFile], ) -> tuple[list[Image.Image], list[str], list[str]]: """Persist a mixed MiniMax H3 multipart reference list. Images are decoded in memory; videos and audio remain files because H3's reference encoders need the original container streams (including video soundtracks). """ images: list[Image.Image] = [] videos: list[str] = [] audios: list[str] = [] paths: list[str] = [] if len(uploads) > MINIMAX_H3_MAX_REFERENCE_COUNT: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=f"MiniMax H3 accepts at most {MINIMAX_H3_MAX_REFERENCE_COUNT} total references.", ) try: for upload in uploads: kind = _uploaded_media_kind(upload) payload = await _read_upload_limited(upload, max_bytes=_minimax_h3_upload_limit(upload)) if kind == "image": try: _validate_minimax_h3_image_payload(payload, filename=upload.filename) images.append( _decode_image_bytes( payload, source=f"uploaded image reference: {upload.filename}", ) ) except InvalidInputReferenceError as exc: raise HTTPException(HTTPStatus.BAD_REQUEST.value, detail=str(exc)) from exc continue suffix = Path(upload.filename or "").suffix.lower() if kind == "video" and suffix and suffix not in MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="MiniMax H3 reference videos must use an MP4 or MOV file.", ) if kind == "audio" and suffix and suffix not in MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="MiniMax H3 reference audio must use a WAV or MP3 file.", ) if not suffix or len(suffix) > 8: suffix = ".mp3" if kind == "audio" else ".mp4" fd, path = tempfile.mkstemp(prefix="vllm_omni_reference_", suffix=suffix) paths.append(path) with os.fdopen(fd, "wb") as output: output.write(payload) if kind == "audio": audios.append(path) else: videos.append(path) except Exception: for path in paths: if os.path.exists(path): os.unlink(path) raise return images, videos, audios async def _parse_video_form( raw_request: Request, prompt: str = Form(...), input_reference: UploadFile | None = File(default=None), input_references: list[UploadFile] | None = File(default=None), control_reference: UploadFile | None = File(default=None), control_type: str | None = Form(default=None), image_reference: str | None = Form(default=None), video_reference: str | None = Form(default=None), audio_reference: str | None = Form(default=None), model: str | None = Form(default=None), seconds: SecondStr | None = Form(default=None), size: SizeStr | None = Form(default=None), user: str | None = Form(default=None), width: int | None = Form(default=None), height: int | None = Form(default=None), num_frames: int | None = Form(default=None), fps: float | None = Form(default=None, ge=1, allow_inf_nan=False), aspect_ratio: str | None = Form(default=None), short_edge: int | None = Form(default=None, ge=1), num_outputs_per_prompt: int = Form(default=1, ge=1, le=10), start_time_seconds: float | None = Form(default=None, ge=0.0), quality: str | None = Form(default=None), num_inference_steps: int | None = Form(default=None), guidance_scale: float | None = Form(default=None), guidance_scale_2: float | None = Form(default=None), boundary_ratio: float | None = Form(default=None), flow_shift: float | None = Form(default=None), true_cfg_scale: float | None = Form(default=None), seed: int | None = Form(default=None), generate_sound: bool | None = Form(default=None), sound_duration: float | None = Form(default=None, gt=0.0), negative_prompt: str | None = Form(default=None), enable_frame_interpolation: bool | None = Form(default=None), frame_interpolation_exp: int | None = Form(default=None, ge=1), frame_interpolation_scale: float | None = Form(default=None, gt=0.0), frame_interpolation_model_path: str | None = Form(default=None), lora: str | None = Form(default=None), extra_params: str | None = Form(default=None), return_stage_metrics: bool | None = Form(default=None), source_video: UploadFile | None = File(default=None), source_audio: UploadFile | None = File(default=None), video_noise_mask: UploadFile | None = File(default=None), audio_noise_mask: UploadFile | None = File(default=None), ) -> tuple[ VideoGenerationRequest, "OmniOpenAIServingVideo", str, ReferenceImage | None, ReferenceVideo | None, ReferenceAudio | None, str | None, LatentEditInput | None, ]: """FastAPI dependency that parses video form data, validates inputs, resolves the handler, and decodes any reference image. Used by both ``POST /v1/videos`` (async) and ``POST /v1/videos/sync``. """ input_references = input_references or [] input_reference_bytes: bytes | None = None parsed_image_reference = _parse_form_json(image_reference) parsed_video_reference = _parse_form_json(video_reference) parsed_audio_reference = _parse_form_json(audio_reference) parsed_video_noise_mask = await _read_latent_edit_mask_json( video_noise_mask, field_name="video_noise_mask", ) parsed_audio_noise_mask = await _read_latent_edit_mask_json( audio_noise_mask, field_name="audio_noise_mask", ) if input_references and any( item is not None for item in (parsed_image_reference, parsed_video_reference, input_reference) ): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail="Provide input_references alone, without input_reference, image_reference, or video_reference.", ) if input_reference is not None and (parsed_image_reference is not None or parsed_video_reference is not None): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=( "Provide only one of input_reference, image_reference, or video_reference when using " "input_reference; image_reference and video_reference may be combined." ), ) request_data: dict[str, Any] = { "prompt": prompt, "model": model, "seconds": seconds, "size": size, "image_reference": parsed_image_reference, "video_reference": parsed_video_reference, "audio_reference": parsed_audio_reference, "user": user, "width": width, "height": height, "num_frames": num_frames, "fps": fps, "aspect_ratio": aspect_ratio, "short_edge": short_edge, "num_outputs_per_prompt": num_outputs_per_prompt, "start_time_seconds": start_time_seconds, "quality": quality, "num_inference_steps": num_inference_steps, "guidance_scale": guidance_scale, "guidance_scale_2": guidance_scale_2, "boundary_ratio": boundary_ratio, "flow_shift": flow_shift, "true_cfg_scale": true_cfg_scale, "seed": seed, "generate_sound": generate_sound, "sound_duration": sound_duration, "negative_prompt": negative_prompt, "enable_frame_interpolation": enable_frame_interpolation, "frame_interpolation_exp": frame_interpolation_exp, "frame_interpolation_scale": frame_interpolation_scale, "frame_interpolation_model_path": frame_interpolation_model_path, "lora": _parse_form_json(lora, expected_type=dict), "extra_params": _parse_form_json(extra_params, expected_type=dict), "return_stage_metrics": return_stage_metrics, } request_data = {k: v for k, v in request_data.items() if v is not None} request = VideoGenerationRequest(**request_data) handler = Omnivideo(raw_request) if handler is None: raise HTTPException( status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, detail="Video generation handler not initialized.", ) logger.info("Video generation handler: %s", type(handler).__name__) try: app_model_name, app_stage_configs = _resolve_video_runtime_context(raw_request) effective_model_name = handler.model_name or app_model_name or request.model or "unknown" if request.model is not None and effective_model_name is not None and request.model != effective_model_name: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST.value, detail=( f"Model mismatch: request specifies '{request.model}' but server is running " f"'{effective_model_name}'." ), ) handler.set_stage_configs_if_missing(app_stage_configs) except HTTPException: raise except Exception as e: logger.exception("Video generation setup failed: %s", e) raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=f"Video generation setup failed: {str(e)}", ) has_latent_edit = _validate_latent_edit_inputs( handler, source_video=source_video, source_audio=source_audio, video_noise_mask=parsed_video_noise_mask, audio_noise_mask=parsed_audio_noise_mask, ) normalized_control_type = _validate_control_upload(handler, request, control_reference, control_type) if normalized_control_type is not None: # Make the selected transfer mode visible while choosing the model's # reference-video decode policy. The upload itself stays unpersisted # until reference parsing succeeds, preserving failure cleanup. _attach_control_upload(request, normalized_control_type) supports_mixed_reference_inputs = handler.supports_mixed_reference_inputs if input_reference is not None: input_reference_bytes = await _read_upload_limited( input_reference, max_bytes=_minimax_h3_upload_limit(input_reference) if supports_mixed_reference_inputs else None, ) if supports_mixed_reference_inputs: input_reference_kind = _uploaded_media_kind(input_reference) _validate_minimax_h3_image_payload( input_reference_bytes, filename=input_reference.filename, allow_non_image=input_reference_kind != "image", ) decode_spec = ReferenceVideoDecodeSpec() if not input_references and (parsed_video_reference is not None or input_reference_bytes is not None): stage_configs = ( handler.stage_configs or app_stage_configs or getattr(getattr(handler, "_engine_client", None), "stage_configs", None) ) decode_spec = _reference_video_decode_spec(request, stage_configs) reference_image = None reference_video = None reference_audio: ReferenceAudio | None = None if input_references: if not supports_mixed_reference_inputs: video_paths = await _persist_uploaded_video_references(input_references) reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths)) images, audio_paths = [], [] else: images, video_paths, audio_paths = await _persist_uploaded_media_references(input_references) if images: reference_image = ReferenceImage(data=images if len(images) > 1 else images[0]) if video_paths: reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths)) if audio_paths: reference_audio = ReferenceAudio(path=audio_paths, cleanup_paths=tuple(audio_paths)) else: video_paths: list[str] = [] try: image_items = _reference_list(request.image_reference) video_items = _reference_list(request.video_reference) image_data = [] for item in image_items: media_data = await decode_input_reference(item, None, None) if not isinstance(media_data, Image.Image): raise InvalidInputReferenceError("image_reference did not decode to an image") image_data.append(media_data) video_frames: list[Image.Image] | None = None for item in video_items: media_data = await decode_input_reference( None, item, None, max_video_frames=decode_spec.max_frames, video_keep=decode_spec.keep, ) if not isinstance(media_data, VideoFrames): raise InvalidInputReferenceError("video_reference did not decode to a video") if media_data.source_path is not None: video_paths.append(media_data.source_path) else: if len(video_items) != 1: raise InvalidInputReferenceError( "multiple video URL references must be downloadable source videos" ) video_frames = list(media_data) if input_reference_bytes is not None: media_data = await decode_input_reference( None, None, input_reference_bytes, max_video_frames=decode_spec.max_frames, video_keep=decode_spec.keep, ) if isinstance(media_data, Image.Image): image_data.append(media_data) elif isinstance(media_data, VideoFrames): if media_data.source_path is not None: video_paths.append(media_data.source_path) else: video_frames = list(media_data) if image_data: reference_image = ReferenceImage(data=image_data if len(image_data) > 1 else image_data[0]) if video_paths: reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths)) elif video_frames is not None: reference_video = ReferenceVideo(data=video_frames) except InvalidInputReferenceError as exc: for path in video_paths: if os.path.exists(path): os.unlink(path) raise HTTPException(400, detail=str(exc) or "Invalid input reference.") from exc audio_paths = [] if reference_audio is None else list(_reference_list(reference_audio.path)) if request.audio_reference is not None: try: for audio_reference in _reference_list(request.audio_reference): audio_paths.append(await decode_audio_url(audio_reference.audio_url)) except InvalidInputReferenceError as exc: _cleanup_video_references(reference_video, reference_audio) cleanup_paths = set(() if reference_audio is None else reference_audio.cleanup_paths) for path in audio_paths: if path not in cleanup_paths and os.path.exists(path): os.unlink(path) raise HTTPException(400, detail=str(exc)) from exc if audio_paths: cleanup_paths = ( tuple(audio_paths) if reference_audio is None else reference_audio.cleanup_paths + tuple(path for path in audio_paths if path not in reference_audio.cleanup_paths) ) reference_audio = ReferenceAudio( path=audio_paths if len(audio_paths) > 1 else audio_paths[0], cleanup_paths=cleanup_paths, ) control_path: str | None = None if control_reference is not None and normalized_control_type is not None: try: control_path = await _persist_uploaded_control_reference( control_reference, max_bytes=CONTROL_REFERENCE_MAX_BYTES, ) _attach_control_upload(request, normalized_control_type, control_path) except (asyncio.CancelledError, HTTPException, OSError, TypeError, ValueError): _cleanup_video_references(reference_video, reference_audio, control_path) raise latent_edit_input: LatentEditInput | None = None if has_latent_edit: source_paths: list[str] = [] try: source_video_path = None source_audio_path = None if source_video is not None: source_video_path = await _persist_latent_edit_source( source_video, field_name="source_video", ) source_paths.append(source_video_path) if source_audio is not None: source_audio_path = await _persist_latent_edit_source( source_audio, field_name="source_audio", ) source_paths.append(source_audio_path) latent_edit_input = LatentEditInput( source_video=source_video_path, source_audio=source_audio_path, video_noise_mask=parsed_video_noise_mask, audio_noise_mask=parsed_audio_noise_mask, cleanup_paths=tuple(source_paths), ) except (asyncio.CancelledError, Exception): for path in source_paths: with suppress(OSError): os.unlink(path) _cleanup_video_references(reference_video, reference_audio, control_path) raise return ( request, handler, effective_model_name, reference_image, reference_video, reference_audio, control_path, latent_edit_input, )